> labels) {
+ this.labels = labels;
+ }
+
+ public String getCreateService() {
+ return createService;
+ }
+
+ public void setCreateService(String createService) {
+ this.createService = createService;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
@Override
public String toString() {
return "RequestPersistTask{" +
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestQueryTask.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestQueryTask.java
similarity index 85%
rename from core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestQueryTask.java
rename to linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestQueryTask.java
index d5323a4d12..08d0059a59 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestQueryTask.java
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestQueryTask.java
@@ -14,11 +14,12 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.protocol.query;
+package com.webank.wedatasphere.linkis.governance.common.entity.task;
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
+
/**
- * Created by enjoyyin on 2018/9/30.
* Need to inherit the relationship, to fill in the rest of the query module restful code
* 需要继承关系,方便query模块restful代码的填写
*/
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestReadAllTask.java
similarity index 86%
rename from core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java
rename to linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestReadAllTask.java
index 886b75ca65..ad82d48b87 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestReadAllTask.java
@@ -14,12 +14,10 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.protocol.query;
+package com.webank.wedatasphere.linkis.governance.common.entity.task;
+
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
-/**
- * created by enjoyyin on 2018/10/9
- * Description:
- */
public class RequestReadAllTask implements QueryProtocol {
/**
* Instance of microservices, through this example, we can take all the tasks below this instance from the database
@@ -27,7 +25,7 @@ public class RequestReadAllTask implements QueryProtocol {
*/
private String instance;
- public RequestReadAllTask(String instance){
+ public RequestReadAllTask(String instance) {
super();
this.instance = instance;
}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestUpdateTask.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestUpdateTask.java
similarity index 82%
rename from core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestUpdateTask.java
rename to linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestUpdateTask.java
index 216001a7e4..1fcb54b03b 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestUpdateTask.java
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/RequestUpdateTask.java
@@ -14,11 +14,10 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.protocol.query;
+package com.webank.wedatasphere.linkis.governance.common.entity.task;
-/**
- * Created by enjoyyin on 2018/9/30.
- */
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
+
public class RequestUpdateTask extends RequestPersistTask implements QueryProtocol {
}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/ResponsePersist.java
similarity index 94%
rename from core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java
rename to linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/ResponsePersist.java
index 39dde90f0c..ab3e1c7991 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/com/webank/wedatasphere/linkis/governance/common/entity/task/ResponsePersist.java
@@ -14,15 +14,15 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.protocol.query;
+package com.webank.wedatasphere.linkis.governance.common.entity.task;
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import java.util.Map;
/**
- * Created by enjoyyin on 2018/9/30.
* Description: ResponsePersist is the content type returned by the query module to the entity
* Description: ResponsePersist 是query模块向entrance返回的内容类型
*/
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/conf/GovernaceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/conf/GovernaceCommonConf.scala
new file mode 100644
index 0000000000..0dd39014d3
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/conf/GovernaceCommonConf.scala
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.conf
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars
+
+object GovernanceCommonConf {
+
+ val CONF_FILTER_RM = "wds.linkis.rm"
+
+ val SPARK_ENGINE_VERSION = CommonVars("wds.linkis.spark.engine.version", "2.4.3")
+
+ val HIVE_ENGINE_VERSION = CommonVars("wds.linkis.hive.engine.version", "1.2.1")
+
+ val PYTHON_ENGINE_VERSION = CommonVars("wds.linkis.python.engine.version", "python2")
+
+ val ENGINE_CONN_SPRING_NAME = CommonVars("wds.linkis.engineconn.name", "EngineConn")
+
+ val ENGINE_CONN_MANAGER_SPRING_NAME = CommonVars("wds.linkis.engineconn.manager.name", "linkis-cg-engineconnmanager")
+
+ val MANAGER_SPRING_NAME = CommonVars("wds.linkis.engineconn.manager.name", "linkis-cg-linkismanager")
+
+ val ENTRANCE_SPRING_NAME = CommonVars("wds.linkis.entrance.name", "linkis-cg-entrance")
+
+
+}
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/DWCJobRetryException.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/DWCJobRetryException.scala
new file mode 100644
index 0000000000..6f12d2f914
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/DWCJobRetryException.scala
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.exception
+
+import com.webank.wedatasphere.linkis.common.exception.DWCRetryException
+
+class DWCJobRetryException(desc: String) extends DWCRetryException(DWCJobRetryException.JOB_RETRY_ERROR_CODE, desc)
+
+object DWCJobRetryException {
+ val JOB_RETRY_ERROR_CODE = 25000
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java
new file mode 100644
index 0000000000..41317efe25
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.exception.engineconn;
+
+/**
+ * ErrorCode of Engine start with 40000
+ *
+ */
+public class EngineConnExecutorErrorCode {
+
+ public static final int INVALID_ENGINE_TYPE = 40100;
+
+ public static final int INVALID_METHOD = 40101;
+
+ public static final int INVALID_PARAMS = 40102;
+
+ public static final int INVALID_LOCK = 40103;
+
+ public static final int INVALID_TASK = 40104;
+
+ public static final int SEND_TO_ENTRANCE_ERROR = 40105;
+
+ public static final int INIT_EXECUTOR_FAILED = 40106;
+
+}
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorException.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorException.java
new file mode 100644
index 0000000000..78a8a07cb8
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorException.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.exception.engineconn;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+
+public class EngineConnExecutorErrorException extends ErrorException {
+
+ public EngineConnExecutorErrorException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+
+ public EngineConnExecutorErrorException(int errCode, String desc, String ip, int port, String serviceKind) {
+ super(errCode, desc, ip, port, serviceKind);
+ }
+
+ public EngineConnExecutorErrorException(int errCode, String desc, Throwable e) {
+ this(errCode, desc);
+ initCause(e);
+ }
+
+
+}
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/conf/RequestQueryGlobalConfig.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/conf/RequestQueryGlobalConfig.scala
new file mode 100644
index 0000000000..f525297c02
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/conf/RequestQueryGlobalConfig.scala
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.protocol.conf
+
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel}
+import com.webank.wedatasphere.linkis.protocol.{CacheableProtocol, RetryableProtocol}
+
+trait ConfigProtocol
+
+case class RequestQueryGlobalConfig(username: String) extends CacheableProtocol with RetryableProtocol with ConfigProtocol
+
+case class RequestQueryEngineConfig(userCreatorLabel: UserCreatorLabel, engineTypeLabel: EngineTypeLabel, filter: String = null) extends CacheableProtocol with RetryableProtocol with ConfigProtocol
+
+case class RequestQueryEngineTypeDefault(engineTypeLabel: EngineTypeLabel) extends CacheableProtocol with RetryableProtocol with ConfigProtocol
+
+case class RequestConfigByLabel(labels: java.util.List[Label[_]], isMerge: Boolean = true) extends CacheableProtocol with RetryableProtocol with ConfigProtocol
+
+
diff --git a/core/cloudProtocol/src/main/scala/com/webank/wedatasphere/linkis/protocol/config/ResponseQueryConfig.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/conf/ResponseQueryConfig.scala
similarity index 89%
rename from core/cloudProtocol/src/main/scala/com/webank/wedatasphere/linkis/protocol/config/ResponseQueryConfig.scala
rename to linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/conf/ResponseQueryConfig.scala
index 2b9129eb0d..50c6c3c8c5 100644
--- a/core/cloudProtocol/src/main/scala/com/webank/wedatasphere/linkis/protocol/config/ResponseQueryConfig.scala
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/conf/ResponseQueryConfig.scala
@@ -14,13 +14,10 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.protocol.config
+package com.webank.wedatasphere.linkis.governance.common.protocol.conf
import java.util
-/**
- * Created by enjoyyin on 2018/10/17.
- */
class ResponseQueryConfig extends ConfigProtocol {
private var keyAndValue: util.Map[String, String] = _
def getKeyAndValue :util.Map[String, String] = keyAndValue
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/engineconn/RequestEngineStatus.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/engineconn/RequestEngineStatus.scala
new file mode 100644
index 0000000000..8cdbfbc5bd
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/engineconn/RequestEngineStatus.scala
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.protocol.engineconn
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.governance.common.entity.NodeExistStatus
+import com.webank.wedatasphere.linkis.protocol.message.RequestProtocol
+
+import java.util
+
+case class RequestEngineStatusBatch(engineList: util.List[ServiceInstance]) extends RequestProtocol
+
+case class ResponseEngineStatusBatch(engineStatus: util.Map[ServiceInstance, NodeExistStatus], msg: String)
+
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/RequestTask.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/RequestTask.scala
new file mode 100644
index 0000000000..6666d56474
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/RequestTask.scala
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.protocol.task
+
+import java.util
+
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.protocol.message.RequestProtocol
+
+trait RequestTask {
+ def getCode: String
+ def setCode(code: String): Unit
+ def getLock: String
+ def setLock(lock: String): Unit
+ def getProperties: util.Map[String, Object]
+ def setProperties(properties: util.Map[String, Object])
+ def data(key: String, value: Object): Unit
+ def getLabels: util.List[Label[_]]
+ def setLabels(labels: util.List[Label[_]])
+}
+object RequestTask {
+ private val header = "#rt_"
+ val RESULT_SET_STORE_PATH = header + "rs_store_path"
+}
+class RequestTaskExecute extends RequestTask with RequestProtocol {
+ private var code: String = _
+ private var lock: String = _
+ private var properties: util.Map[String, Object] = new util.HashMap[String, Object]
+
+ private var labels: util.List[Label[_]] = new util.ArrayList[Label[_]](4)
+
+ override def getCode: String = code
+
+ override def setCode(code: String): Unit = this.code = code
+
+ override def getLock: String = lock
+
+ override def setLock(lock: String): Unit = this.lock = lock
+
+ override def getProperties: util.Map[String, Object] = properties
+
+ override def setProperties(properties: util.Map[String, Object]): Unit = this.properties = properties
+
+ override def data(key: String, value: Object): Unit = {
+ if(properties == null) properties = new util.HashMap[String, Object]
+ properties.put(key, value)
+ }
+
+
+
+ override def getLabels: util.List[Label[_]] = labels
+
+ override def setLabels(labels: util.List[Label[_]]): Unit = this.labels = labels
+}
+
+trait TaskState extends RequestProtocol {}
+
+case class RequestTaskPause(execId: String) extends TaskState
+case class RequestTaskResume(execId: String) extends TaskState
+case class RequestTaskKill(execId: String) extends TaskState
+
+
+/**
+ * The status of requesting job execution, mainly used for:
+ * 1. If the engine has not sent information to the entity for a long time, the enumeration periodically requests status information
+ * 2. If the entance of the request engine is hanged, the newly taken entity first requests the status of the job from the engine, the engine maintains the new sender information,
+ * and then sends log information to the sender.
+ * 请求Job执行的状态，主要用于：
+ * 1. 如果engine很久没有给entrance发送信息，entrance定时请求一次状态信息
+ * 2. 如果请求engine的entrance挂掉了，新接手的entrance第一次向engine请求Job的状态，engine维系新的sender信息，后续向该sender发送日志信息
+ */
+case class RequestTaskStatus(execId: String) extends TaskState
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/ResponseEngineConnPid.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/ResponseEngineConnPid.scala
new file mode 100644
index 0000000000..1fa8400949
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/ResponseEngineConnPid.scala
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.protocol.task
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.protocol.RetryableProtocol
+import com.webank.wedatasphere.linkis.protocol.message.RequestProtocol
+
+
+/**
+ *
+ * @param pid
+ */
+case class ResponseEngineConnPid(serviceInstance: ServiceInstance, pid: String, ticketId: String) extends RetryableProtocol with RequestProtocol
+
+
+
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/ResponseTaskExecute.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/ResponseTaskExecute.scala
new file mode 100644
index 0000000000..248fb82d93
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/protocol/task/ResponseTaskExecute.scala
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.protocol.task
+
+import java.util
+
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus
+import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
+import com.webank.wedatasphere.linkis.protocol.message.RequestProtocol
+import com.webank.wedatasphere.linkis.protocol.{BroadcastProtocol, RetryableProtocol}
+
+case class ResponseTaskExecute(execId: String)
+
+case class ResponseTaskProgress(execId: String, progress: Float, progressInfo: Array[JobProgressInfo]) extends RetryableProtocol with RequestProtocol
+
+case class ResponseEngineLock(lock: String)
+
+case class EngineConcurrentInfo(runningTasks: Int, pendingTasks: Int, succeedTasks: Int, failedTasks: Int)
+case class EngineOverloadInfo(maxMemory: Long, usedMemory: Long, systemCPUUsed: Float)
+
+case class ResponseEngineStatusChanged(instance: String, fromStatus: ExecutionNodeStatus, toStatus: ExecutionNodeStatus,
+ overload: EngineOverloadInfo, concurrent: EngineConcurrentInfo)
+ extends BroadcastProtocol
+
+case class ResponseEngineInfo(createEntranceInstance: String, creator: String, user: String, properties: util.Map[String, String])
+
+case class ResponseEngineStatus(instance: String, status: ExecutionNodeStatus, overload: EngineOverloadInfo, concurrent: EngineConcurrentInfo,
+ engineInfo: ResponseEngineInfo)
+
+case class ResponseTaskLog(execId: String, log: String) extends RetryableProtocol with RequestProtocol
+
+case class ResponseTaskError(execId: String, errorMsg: String) extends RetryableProtocol with RequestProtocol
+
+case class ResponseTaskStatus(execId: String, status: ExecutionNodeStatus) extends RetryableProtocol with RequestProtocol
+
+case class ResponseTaskResultSet(execId: String, output: String, alias: String) extends RetryableProtocol with RequestProtocol
+
+case class ResponseTaskResultSize(execId: String, resultSize: Int) extends RetryableProtocol with RequestProtocol
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/utils/EngineConnArguments.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/utils/EngineConnArguments.scala
new file mode 100644
index 0000000000..e54827aa1b
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/utils/EngineConnArguments.scala
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.utils
+
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
+
+trait EngineConnArguments {
+
+ def getSpringConfMap: Map[String, String]
+
+ def getEngineConnConfMap: Map[String, String]
+
+}
+
+trait EngineConnArgumentsBuilder {
+
+ def addSpringConf(confMap: Map[String, String]): EngineConnArgumentsBuilder
+
+ def addSpringConf(key: String, value: String): EngineConnArgumentsBuilder
+
+ def addEngineConnConf(confMap: Map[String, String]): EngineConnArgumentsBuilder
+
+ def addEngineConnConf(key: String, value: String): EngineConnArgumentsBuilder
+
+ def build(): EngineConnArguments
+
+}
+
+object EngineConnArgumentsBuilder {
+ def newBuilder(): EngineConnArgumentsBuilder = new DefaultEngineConnArgumentsBuilder
+}
+
+class DefaultEngineConnArgumentsBuilder extends EngineConnArgumentsBuilder {
+
+ private val engineConnOptionMap = new mutable.HashMap[String, String]()
+ private val springOptionMap = new mutable.HashMap[String, String]()
+
+ override def addSpringConf(confMap: Map[String, String]): EngineConnArgumentsBuilder = {
+ springOptionMap ++= confMap
+ this
+ }
+
+ override def addSpringConf(key: String, value: String): EngineConnArgumentsBuilder = {
+ springOptionMap += key -> value
+ this
+ }
+
+ override def addEngineConnConf(confMap: Map[String, String]): EngineConnArgumentsBuilder = {
+ engineConnOptionMap ++= confMap
+ this
+ }
+
+ override def addEngineConnConf(key: String, value: String): EngineConnArgumentsBuilder = {
+ engineConnOptionMap += key -> value
+ this
+ }
+
+ override def build(): EngineConnArguments = new EngineConnArguments {
+ override def getSpringConfMap: Map[String, String] = springOptionMap.toMap
+ override def getEngineConnConfMap: Map[String, String] = engineConnOptionMap.toMap
+ }
+}
+
+trait EngineConnArgumentsParser {
+
+ def parseToObj(args: Array[String]): EngineConnArguments
+
+ def parseToArgs(engineConnArguments: EngineConnArguments): Array[String]
+
+}
+object EngineConnArgumentsParser {
+
+ val LABEL_PREFIX = "label."
+
+ private val parser = new DefaultEngineConnArgumentsParser
+
+ def getEngineConnArgumentsParser: EngineConnArgumentsParser = parser
+}
+
+class DefaultEngineConnArgumentsParser extends EngineConnArgumentsParser {
+
+ protected val ENGINE_CONN_CONF = "--engineconn-conf"
+ protected val SPRING_CONF = "--spring-conf"
+
+ protected val keyValueRegex = "([^=]+)=(.+)".r
+
+ override def parseToObj(args: Array[String]): EngineConnArguments = {
+ var i = 0
+ val argumentsBuilder = new DefaultEngineConnArgumentsBuilder
+ while(i < args.length) {
+ args(i) match {
+ case ENGINE_CONN_CONF =>
+ addKeyValue(args(i + 1), (key, value) => {
+ argumentsBuilder.addEngineConnConf(key, value)
+ i += 1
+ })
+ case SPRING_CONF =>
+ addKeyValue(args(i + 1), (key, value) => {
+ argumentsBuilder.addSpringConf(key, value)
+ i += 1
+ })
+ case _ => throw new IllegalArgumentException(s"illegal command line, ${args(i)} cannot recognize.")
+ }
+ i += 1
+ }
+ argumentsBuilder.build()
+ }
+
+ private def addKeyValue(keyValue: String, argumentsBuilder: (String, String) => Unit): Unit = keyValue match {
+ case keyValueRegex(key, value) =>
+ argumentsBuilder(key, value)
+ case _ => throw new IllegalArgumentException("illegal command line, format: --conf key=value.")
+ }
+
+ override def parseToArgs(engineConnArguments: EngineConnArguments): Array[String] = {
+ val options = ArrayBuffer[String]()
+ def write(confMap: Map[String, String], optionType: String): Unit = confMap.foreach { case (key, value) =>
+ if (StringUtils.isNotEmpty(key) && StringUtils.isNotEmpty(value)) {
+ options += optionType
+ options += (key + "=" + value)
+ }
+ }
+ write(engineConnArguments.getEngineConnConfMap, ENGINE_CONN_CONF)
+ write(engineConnArguments.getSpringConfMap, SPRING_CONF)
+ options.toArray
+ }
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/utils/GovernanceConstant.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/utils/GovernanceConstant.scala
new file mode 100644
index 0000000000..a64e6db06d
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/com/webank/wedatasphere/linkis/governance/common/utils/GovernanceConstant.scala
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.governance.common.utils
+
+object GovernanceConstant {
+
+ val TASK_SOURCE_MAP_KEY = "source"
+
+ val TASK_RESOURCES_STR = "resources"
+
+ val TASK_RESOURCE_ID_STR = "resourceId"
+
+ val TASK_RESOURCE_VERSION_STR = "version"
+
+ val TASK_RESOURCE_FILE_NAME_STR = "fileName"
+
+ val REQUEST_ENGINE_STATUS_BATCH_LIMIT = 500
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/pom.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/pom.xml
new file mode 100644
index 0000000000..4a1411e937
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/pom.xml
@@ -0,0 +1,70 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../../pom.xml
+
+ 4.0.0
+
+ linkis-engineconn-linux-launch
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-manager-core
+ ${linkis.version}
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/src/main/scala/com/webank/wedatasphere/linkis/ecm/linux/launch/LinuxProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/src/main/scala/com/webank/wedatasphere/linkis/ecm/linux/launch/LinuxProcessEngineConnLaunch.scala
new file mode 100644
index 0000000000..1782faa255
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/src/main/scala/com/webank/wedatasphere/linkis/ecm/linux/launch/LinuxProcessEngineConnLaunch.scala
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.linux.launch
+
+import com.webank.wedatasphere.linkis.ecm.core.launch.ProcessEngineConnLaunch
+
+
+
+class LinuxProcessEngineConnLaunch extends ProcessEngineConnLaunch {
+
+ override protected def execFile: Array[String] = Array("sh", getPreparedExecFile)
+
+ override protected def sudoCommand(user: String, command: String): Array[String] =
+ Array("sudo", "su", "-", user, "-c", command)
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/pom.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/pom.xml
new file mode 100644
index 0000000000..6a2fdead19
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/pom.xml
@@ -0,0 +1,85 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../pom.xml
+
+ 4.0.0
+
+ linkis-engineconn-manager-core
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-manager-common
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-computation-governance-common
+ ${linkis.version}
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-plugin-core
+ ${linkis.version}
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ ${basedir}/src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/EngineConn.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/EngineConn.scala
new file mode 100644
index 0000000000..df8b71f7d8
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/EngineConn.scala
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.engineconn
+
+import java.io.Closeable
+import java.util
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.ecm.core.launch.{EngineConnLaunchRunner, EngineConnManagerEnv}
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.NodeResource
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.EngineConnCreationDesc
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+
+trait EngineConn extends Closeable {
+
+ def getTickedId: String
+
+ def setTickedId(tickedId: String)
+
+ def getServiceInstance: ServiceInstance
+
+ def setServiceInstance(serviceInstance: ServiceInstance): Unit
+
+ def getResource: NodeResource
+
+ def setResource(resource: NodeResource): Unit
+
+ def getLabels: util.List[Label[_]]
+
+ def setLabels(labels: util.List[Label[_]]): Unit
+
+ def getStatus: NodeStatus
+
+ def setStatus(status: NodeStatus): Unit
+
+ def getCreationDesc: EngineConnCreationDesc
+
+ def setCreationDesc(desc: EngineConnCreationDesc): Unit
+
+ def getEngineConnInfo: EngineConnInfo
+
+ def setEngineConnInfo(engineConnInfo: EngineConnInfo): Unit
+
+ def getEngineConnManagerEnv: EngineConnManagerEnv
+
+ def setEngineConnManagerEnv(env: EngineConnManagerEnv): Unit
+
+ def getEngineConnLaunchRunner: EngineConnLaunchRunner
+
+ def setEngineConnLaunchRunner(runner: EngineConnLaunchRunner): Unit
+
+ def setPid(pid: String): Unit
+
+ def getPid: String
+
+ override def close(): Unit = {}
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/EngineConnInfo.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/EngineConnInfo.scala
new file mode 100644
index 0000000000..e2fcd4a81c
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/EngineConnInfo.scala
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.engineconn
+
+
+class EngineConnInfo {
+
+ private var errorCode: Int = 0
+ private var errorMsg: String = _
+
+ def setErrorCode(errorCode: Int): Unit = this.errorCode = errorCode
+
+ def getErrorCode: Int = errorCode
+
+ def setErrorMsg(errorMsg: String): Unit = this.errorMsg = errorMsg
+
+ def getErrorMsg: String = errorMsg
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/YarnEngineConn.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/YarnEngineConn.scala
new file mode 100644
index 0000000000..2a31060f39
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/engineconn/YarnEngineConn.scala
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.engineconn
+
+
+trait YarnEngineConn extends EngineConn {
+
+ def getApplicationId: String
+
+ def setApplicationId(applicationId: String): Unit
+
+ def getApplicationURL: String
+
+ def setApplicationURL(applicationURL: String): Unit
+
+ def getYarnMode: String
+
+ def setYarnMode(yarnMode: String): Unit
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/DiscoveryMsgGenerator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/DiscoveryMsgGenerator.scala
new file mode 100644
index 0000000000..dafe19cc94
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/DiscoveryMsgGenerator.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.launch
+
+import java.util
+
+import scala.collection.JavaConversions._
+
+
+trait DiscoveryMsgGenerator {
+
+ def generate(engineConnManagerEnv: EngineConnManagerEnv): java.util.Map[String, String]
+
+}
+
+class EurekaDiscoveryMsgGenerator extends DiscoveryMsgGenerator {
+ override def generate(engineConnManagerEnv: EngineConnManagerEnv): util.Map[String, String] =
+ engineConnManagerEnv.properties.get("eureka.client.serviceUrl.defaultZone")
+ .map(v => Map("eureka.client.serviceUrl.defaultZone" -> v)).getOrElse[Map[String,String]](Map.empty)
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnLaunch.scala
new file mode 100644
index 0000000000..e2973b61df
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnLaunch.scala
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.launch
+
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest
+
+
+
+trait EngineConnLaunch {
+
+ def setEngineConnLaunchRequest(request: EngineConnLaunchRequest): Unit
+
+ def setEngineConnManagerEnv(engineConnManagerEnv: EngineConnManagerEnv): Unit
+
+ def launch(): Unit
+
+ def kill(): Unit
+
+ def isAlive: Boolean
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnLaunchRunner.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnLaunchRunner.scala
new file mode 100644
index 0000000000..bf81c3ac39
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnLaunchRunner.scala
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.launch
+
+
+trait EngineConnLaunchRunner extends Runnable {
+
+ /**
+ *
+ * @param launch
+ */
+ def setEngineConnLaunch(launch: EngineConnLaunch): Unit
+
+ def getEngineConnLaunch: EngineConnLaunch
+
+ def stop(): Unit
+
+}
+class EngineConnLaunchRunnerImpl extends EngineConnLaunchRunner {
+
+ private var launch: EngineConnLaunch = _
+ /**
+ *
+ * @param launch
+ */
+ override def setEngineConnLaunch(launch: EngineConnLaunch): Unit = this.launch = launch
+
+ override def getEngineConnLaunch: EngineConnLaunch = launch
+
+ override def stop(): Unit = launch.kill()
+
+ override def run(): Unit = {
+ launch.launch()
+ }
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnManagerEnv.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnManagerEnv.scala
new file mode 100644
index 0000000000..71ef80222a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/EngineConnManagerEnv.scala
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.launch
+
+
+trait EngineConnManagerEnv {
+
+ val engineConnManagerHomeDir: String
+ val engineConnWorkDir: String
+ val engineConnLogDirs: String
+ val engineConnTempDirs: String
+ val engineConnManagerHost: String
+ val engineConnManagerPort: String
+ val linkDirs: Map[String, String]
+ val properties: Map[String, String]
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineCommandBuilder.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineCommandBuilder.scala
new file mode 100644
index 0000000000..1f851a6e92
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineCommandBuilder.scala
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.launch
+
+import java.io.OutputStream
+
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.process.LaunchConstants
+import org.apache.commons.io.IOUtils
+
+
+trait ProcessEngineCommandBuilder {
+
+ def setCommand(command: Array[String]): Unit
+
+ def setEnv(key: String, value: String): Unit
+
+ def newLine(command: Array[String])
+
+ def link(fromPath: String, toPath: String)
+
+ def mkdir(dir: String): Unit
+
+ def writeTo(output: OutputStream): Unit
+
+ def replaceExpansionMarker(value: String): String
+
+}
+
+abstract class ShellProcessEngineCommandBuilder extends ProcessEngineCommandBuilder {
+
+ private val LINE_SEPARATOR = System.getProperty("line.separator")
+ private val sb = new StringBuilder
+
+ def newLine(command: String): Unit = newLine(Array(command))
+
+ override def newLine(command: Array[String]): Unit = {
+ command.foreach(sb ++= _)
+ sb ++= LINE_SEPARATOR
+ }
+
+ def writeTo(output: OutputStream): Unit = {
+ IOUtils.write(sb, output)
+ }
+
+ override def replaceExpansionMarker(value: String): String = value.replaceAll(LaunchConstants.EXPANSION_MARKER_LEFT, "\\${")
+ .replaceAll(LaunchConstants.EXPANSION_MARKER_RIGHT, "}")
+}
+
+class UnixProcessEngineCommandBuilder extends ShellProcessEngineCommandBuilder {
+
+ newLine("#!/bin/bash")
+
+ private def addErrorCheck(): Unit = {
+ newLine("linkis_engineconn_errorcode=$?")
+ newLine("if [ $linkis_engineconn_errorcode -ne 0 ]")
+ newLine("then")
+ newLine(" exit $linkis_engineconn_errorcode")
+ newLine("fi")
+ }
+
+ override def setCommand(command: Array[String]): Unit = {
+ newLine(Array(s"exec /bin/bash -c ${"\""}${command.mkString(" ")}${"\""}"))
+ addErrorCheck()
+ }
+
+ override def setEnv(key: String, value: String): Unit = newLine(Array("export ", key, "=\"", value, "\""))
+
+ override def link(fromPath: String, toPath: String): Unit = {
+ newLine(Array("ln -sf \"", fromPath, "\" \"", toPath, "\""))
+ addErrorCheck()
+ }
+
+ override def mkdir(dir: String): Unit = {
+ newLine(Array("mkdir -p ", dir))
+ addErrorCheck()
+ }
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineCommandExec.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineCommandExec.scala
new file mode 100644
index 0000000000..fbc1724603
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineCommandExec.scala
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.launch
+
+import java.io.File
+import java.util
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+
+
+trait ProcessEngineCommandExec {
+
+ def execute(): Unit
+
+ def getProcess: Process
+
+}
+
+class ShellProcessEngineCommandExec(command: Array[String], baseDir: String,
+ environment: util.Map[String, String], timeout: Long) extends
+ ProcessEngineCommandExec with Logging {
+
+ private var process: Process = _
+
+ def this(command: Array[String], baseDir: String) = this(command, baseDir, null, 0)
+
+ def this(command: Array[String]) = this(command, null)
+
+ override def execute(): Unit = {
+
+ info(s"Invoke subProcess, base dir ${this.baseDir} cmd is: ${command.mkString}")
+ val builder = new ProcessBuilder(command: _*)
+ if (environment != null) builder.environment.putAll(this.environment)
+ if (baseDir != null) builder.directory(new File(this.baseDir))
+ builder.redirectErrorStream(true)
+ process = builder.start
+ }
+
+ override def getProcess: Process = process
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala
new file mode 100644
index 0000000000..9146b56618
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.launch
+
+import java.io.{File, InputStream, OutputStream}
+import java.net.ServerSocket
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.governance.common.conf.GovernanceCommonConf
+import com.webank.wedatasphere.linkis.governance.common.utils.{EngineConnArgumentsBuilder, EngineConnArgumentsParser}
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.conf.EnvConfiguration
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.process.Environment._
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.process.LaunchConstants._
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.process.{Environment, ProcessEngineConnLaunchRequest}
+import org.apache.commons.io.{FileUtils, IOUtils}
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions._
+
+
+trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging {
+
+ private var request: ProcessEngineConnLaunchRequest = _
+ private var engineConnManagerEnv: EngineConnManagerEnv = _
+ private var discoveryMsgGenerator: DiscoveryMsgGenerator = _
+ private var processBuilder: ProcessEngineCommandBuilder = _
+ private var preparedExecFile: String = _
+ private var process: Process = _
+
+ private var engineConnPort: String = _
+
+ protected def newProcessEngineConnCommandBuilder(): ProcessEngineCommandBuilder = new UnixProcessEngineCommandBuilder
+
+ protected def newProcessEngineConnCommandExec(command: Array[String], workDir: String): ProcessEngineCommandExec =
+ new ShellProcessEngineCommandExec(command, workDir)
+
+ override def setEngineConnLaunchRequest(request: EngineConnLaunchRequest): Unit = request match {
+ case processEngineConnLaunchRequest: ProcessEngineConnLaunchRequest =>
+ this.request = processEngineConnLaunchRequest
+ case _ => //TODO exception
+ }
+
+ override def setEngineConnManagerEnv(engineConnManagerEnv: EngineConnManagerEnv): Unit = this.engineConnManagerEnv = engineConnManagerEnv
+
+ def setDiscoveryMsgGenerator(discoveryMsgGenerator: DiscoveryMsgGenerator): Unit = this.discoveryMsgGenerator = discoveryMsgGenerator
+
+ def getEngineConnLaunchRequest: EngineConnLaunchRequest = request
+
+ private def initializeEnv(): Unit = {
+ val environment = request.environment
+ def putIfExists(enum: Environment): Unit = {
+ val key = enum.toString
+ val conf = CommonVars.apply(key, "")
+ if(StringUtils.isNotBlank(conf.getValue)) environment.put(key, conf.getValue)
+ }
+ Environment.values foreach {
+ case USER => environment.put(USER.toString, request.user)
+ case ECM_HOME => environment.put(ECM_HOME.toString, engineConnManagerEnv.engineConnManagerHomeDir)
+ case PWD => environment.put(PWD.toString, engineConnManagerEnv.engineConnWorkDir)
+ case LOG_DIRS => environment.put(LOG_DIRS.toString, engineConnManagerEnv.engineConnLogDirs)
+ case TEMP_DIRS => environment.put(TEMP_DIRS.toString, engineConnManagerEnv.engineConnTempDirs)
+ case ECM_HOST => environment.put(ECM_HOST.toString, engineConnManagerEnv.engineConnManagerHost)
+ case ECM_PORT => environment.put(ECM_PORT.toString, engineConnManagerEnv.engineConnManagerPort)
+ case HADOOP_HOME => putIfExists(HADOOP_HOME)
+ case HADOOP_CONF_DIR => putIfExists(HADOOP_CONF_DIR)
+ case HIVE_CONF_DIR => putIfExists(HIVE_CONF_DIR)
+ case RANDOM_PORT => environment.put(RANDOM_PORT.toString, findAvailPort().toString)
+ case _ =>
+ }
+ }
+
+ private def findAvailPort(): Int = {
+ val socket = new ServerSocket(0)
+ Utils.tryFinally(socket.getLocalPort)(IOUtils.closeQuietly(socket))
+ }
+
+ private def addAvailPort(value: String): String = {
+ var existsRandomPort = value.contains(CLASS_PATH_SEPARATOR)
+ var newValue = value
+ while(existsRandomPort) {
+ newValue = value.replaceFirst(CLASS_PATH_SEPARATOR, findAvailPort().toString)
+ existsRandomPort = value.contains(CLASS_PATH_SEPARATOR)
+ }
+ newValue
+ }
+
+ override def launch(): Unit = {
+ request.necessaryEnvironments.foreach{e =>
+ val env = CommonVars(e, "")
+ if(StringUtils.isEmpty(env.getValue))
+ throw new ErrorException(30000, s"Necessary environment $e is not exists!(必须的环境变量 $e 不存在！)") //TODO exception
+ else request.environment.put(e, env.getValue)
+ }
+ prepareCommand()
+ val exec = newProcessEngineConnCommandExec(sudoCommand(request.user, execFile.mkString(" ")), engineConnManagerEnv.engineConnWorkDir)
+ exec.execute()
+ process = exec.getProcess
+ }
+
+ protected def execFile: Array[String]
+
+ def getEngineConnPort: String = engineConnPort
+
+ protected def getCommandArgs: Array[String] = {
+ if (request.creationDesc.properties.exists { case (k, v) => k.contains(" ") || (v != null && v.contains(" ")) })
+ throw new ErrorException(30000, "Startup parameters contain spaces!(启动参数中包含空格！)") //TODO exception
+ val arguments = EngineConnArgumentsBuilder.newBuilder()
+ engineConnPort = findAvailPort().toString
+ var springConf = Map("spring.application.name" -> GovernanceCommonConf.ENGINE_CONN_SPRING_NAME.getValue,
+ "server.port" -> engineConnPort, "spring.profiles.active" -> "engineconn",
+ "logging.config" -> s"classpath:${EnvConfiguration.LOG4J2_XML_FILE.getValue}") ++: discoveryMsgGenerator.generate(engineConnManagerEnv)
+ request.creationDesc.properties.filter(_._1.startsWith("spring.")).foreach { case (k, v) =>
+ springConf = springConf += (k -> v)
+ }
+ arguments.addSpringConf(springConf.toMap)
+ var engineConnConf = Map("ticketId" -> request.ticketId, "user" -> request.user)
+ engineConnConf = engineConnConf ++: request.labels.map(l => EngineConnArgumentsParser.LABEL_PREFIX + l.getLabelKey -> l.getStringValue).toMap
+ engineConnConf = engineConnConf ++: request.creationDesc.properties.filterNot(_._1.startsWith("spring.")).toMap
+ arguments.addEngineConnConf(engineConnConf)
+ EngineConnArgumentsParser.getEngineConnArgumentsParser.parseToArgs(arguments.build())
+ }
+
+ override def kill(): Unit = process.destroy()
+
+ override def isAlive: Boolean = process.isAlive
+
+ protected def prepareCommand(): Unit = {
+ processBuilder = newProcessEngineConnCommandBuilder()
+ engineConnManagerEnv.linkDirs.foreach{case (k, v) => processBuilder.link(k, v)}
+ initializeEnv()
+ request.environment.foreach{ case (k, v) =>
+ var value = v.replaceAll(CLASS_PATH_SEPARATOR, File.pathSeparator)
+ value = addAvailPort(value)
+ processBuilder.setEnv(k, processBuilder.replaceExpansionMarker(value))
+ }
+ val execCommand = request.commands.map(processBuilder.replaceExpansionMarker(_)).map(addAvailPort) ++ getCommandArgs
+ //execCommand = sudoCommand(request.user, execCommand.mkString(" "))
+ processBuilder.setCommand(execCommand)
+ preparedExecFile = new File(engineConnManagerEnv.engineConnWorkDir, "engineConnExec.sh").getPath
+ val output = getFileOutputStream
+ Utils.tryFinally(processBuilder.writeTo(output))(output.close())
+ }
+
+ protected def sudoCommand(user: String, command: String): Array[String]
+
+ protected def getFileOutputStream: OutputStream = FileUtils.openOutputStream(new File(preparedExecFile))
+
+ protected def getPreparedExecFile: String = preparedExecFile
+
+ def getProcessInputStream:InputStream = process.getInputStream
+
+ def processWaitFor:Int =process.waitFor
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMAsyncListenerBus.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMAsyncListenerBus.scala
new file mode 100644
index 0000000000..fc13ec07de
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMAsyncListenerBus.scala
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.listener
+
+import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus
+
+
+class ECMAsyncListenerBus(capacity: Int, name: String)
+ (consumerThreadSize: Int, threadMaxFreeTime: Long)
+ extends ListenerEventBus[ECMEventListener, ECMEvent](capacity, name)(consumerThreadSize, threadMaxFreeTime) {
+ /**
+ * Post an event to the specified listener. `onPostEvent` is guaranteed to be called in the same
+ * thread for all listeners.
+ */
+ override protected def doPostEvent(listener: ECMEventListener, event: ECMEvent): Unit = listener.onEvent(event)
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMEvent.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMEvent.java
new file mode 100644
index 0000000000..011137d69a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMEvent.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+
+
+public interface ECMEvent extends Event {
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMEventListener.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMEventListener.scala
new file mode 100644
index 0000000000..242972feb1
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMEventListener.scala
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.listener
+
+import com.webank.wedatasphere.linkis.common.listener.{Event, EventListener}
+import com.webank.wedatasphere.linkis.common.utils.Logging
+
+
+trait ECMEventListener extends EventListener with Logging {
+
+ def onEvent(event: ECMEvent)
+
+ override def onEventError(event: Event, t: Throwable): Unit = {
+ error(s"async message invoke failed ${event.toString}", t)
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMSyncListenerBus.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMSyncListenerBus.scala
new file mode 100644
index 0000000000..208fe6fd68
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/listener/ECMSyncListenerBus.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.listener
+
+import com.webank.wedatasphere.linkis.common.listener.ListenerBus
+
+class ECMSyncListenerBus extends ListenerBus[ECMEventListener, ECMEvent] {
+ /**
+ * Post an event to the specified listener. `onPostEvent` is guaranteed to be called in the same
+ * thread for all listeners.
+ */
+ override protected def doPostEvent(listener: ECMEventListener, event: ECMEvent): Unit = listener.onEvent(event)
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/metrics/ECMMetrics.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/metrics/ECMMetrics.scala
new file mode 100644
index 0000000000..6245c57b5e
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/metrics/ECMMetrics.scala
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.metrics
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+
+
+
+trait ECMMetrics {
+
+ def getStartingEngineConns: Array[_ <: EngineConn]
+
+ def getRunningEngineConns: Array[EngineConn]
+
+ def getSucceedEngineConns: Array[EngineConn]
+
+ def getFailedEngineConns: Array[EngineConn]
+
+ def increaseStartingEngineConn(engineConn: EngineConn): Unit
+
+ def increaseRunningEngineConn(engineConn: EngineConn): Unit
+
+ def increaseSuccessEngineConn(engineConn: EngineConn): Unit
+
+ def increaseFailedEngineConn(engineConn: EngineConn): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/report/ECMHealthReport.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/report/ECMHealthReport.scala
new file mode 100644
index 0000000000..44fef7ce3a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/report/ECMHealthReport.scala
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.report
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.manager.common.entity.metrics.NodeOverLoadInfo
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.Resource
+
+
+trait ECMHealthReport extends NodeHealthReport {
+
+ def setProtectedResource(protectedResource: Resource): Unit
+
+ def getProtectResource: Resource
+
+ def setOverload(nodeOverLoadInfo: NodeOverLoadInfo): Unit
+
+ def getOverload: NodeOverLoadInfo
+
+ def setRunningEngineConns(runningEngines: Array[EngineConn])
+
+ def getRunningEngineConns: Array[EngineConn]
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/report/NodeHealthReport.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/report/NodeHealthReport.scala
new file mode 100644
index 0000000000..ebd7c87a22
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/com/webank/wedatasphere/linkis/ecm/core/report/NodeHealthReport.scala
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.core.report
+
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.Resource
+
+
+trait NodeHealthReport {
+
+ def getReportTime: Long
+
+ def setReportTime(reportTime: Long): Unit
+
+ def getNodeStatus: NodeStatus
+
+ def setNodeStatus(nodeStatus: NodeStatus): Unit
+
+ def setNodeMsg(msg: String): Unit
+
+ def getNodeMsg: String
+
+ def getUsedResource: Resource
+
+ def setUsedResource(resource: Resource): Unit
+
+ def getTotalResource: Resource
+
+ def setTotalResource(resource: Resource): Unit
+
+ def setNodeId(nodeId: String): Unit
+
+ def getNodeId: String
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml
new file mode 100644
index 0000000000..06b7609354
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml
@@ -0,0 +1,141 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../pom.xml
+
+ 4.0.0
+
+ linkis-engineconn-manager-server
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-manager-core
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-linux-launch
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-message-scheduler
+ ${linkis.version}
+ provided
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-bml-client
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-common
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-storage
+ ${linkis.version}
+ provided
+
+
+
+ org.json4s
+ json4s-jackson_${scala.binary.version}
+ ${json4s.version}
+ provided
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ 2.3
+ false
+
+
+ make-assembly
+ package
+
+ single
+
+
+
+ src/main/assembly/distribution.xml
+
+
+
+
+
+ false
+ out
+ false
+ false
+
+ src/main/assembly/distribution.xml
+
+
+
+
+
+
+ ${basedir}/src/main/resources
+
+ **/*.properties
+ **/*.xml
+ **/*.yml
+
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/assembly/distribution.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/assembly/distribution.xml
new file mode 100644
index 0000000000..ef1578ec9c
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/assembly/distribution.xml
@@ -0,0 +1,73 @@
+
+
+
+ linkis-engineconn-manager
+
+ dir
+
+ false
+ linkis-engineconn-manager
+
+
+
+
+
+ lib
+ true
+ true
+ false
+ true
+ true
+
+
+
+
+
+
+
+ ${basedir}/src/main/resources
+
+ *
+
+ 0777
+ conf
+ unix
+
+
+
+
+
+
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/com/webank/wedatasphere/linkis/ecm/server/ECMApplication.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/com/webank/wedatasphere/linkis/ecm/server/ECMApplication.java
new file mode 100644
index 0000000000..9e38ce8793
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/com/webank/wedatasphere/linkis/ecm/server/ECMApplication.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server;
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication;
+import com.webank.wedatasphere.linkis.common.ServiceInstance;
+import com.webank.wedatasphere.linkis.ecm.core.listener.ECMAsyncListenerBus;
+import com.webank.wedatasphere.linkis.ecm.core.listener.ECMSyncListenerBus;
+import com.webank.wedatasphere.linkis.ecm.server.context.ECMContext;
+import com.webank.wedatasphere.linkis.ecm.server.listener.ECMClosedEvent;
+import com.webank.wedatasphere.linkis.ecm.server.listener.ECMReadyEvent;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.event.ContextClosedEvent;
+import org.springframework.context.event.EventListener;
+
+import static com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration.ECM_ASYNC_BUS_WAITTOEMPTY_TIME;
+
+
+public class ECMApplication extends DataWorkCloudApplication {
+
+ private static ECMContext ecmContext;
+
+ private volatile static boolean ready;
+
+ private static ServiceInstance ecmServiceInstance;
+
+ private static String[] parmas;
+
+ public static void main(String[] args) throws ReflectiveOperationException {
+ parmas = args;
+ DataWorkCloudApplication.main(args);
+ }
+
+ public static ECMContext getContext() {
+ return ecmContext;
+ }
+
+ public static void setContext(ECMContext context) {
+ ecmContext = context;
+ }
+
+ public static ServiceInstance getECMServiceInstance() {
+ return ecmServiceInstance;
+ }
+
+ public static void setECMServiceInstance(ServiceInstance serviceInstance) {
+ ecmServiceInstance = serviceInstance;
+ }
+
+ public static boolean isReady() {
+ return ready;
+ }
+
+ public static void setReady(boolean applicationReady) {
+ ready = applicationReady;
+ }
+
+ public static String[] getParmas() {
+ return parmas;
+ }
+}
+
+
+@Configuration
+class ECMApplicationListener {
+
+ private final Log logger = LogFactory.getLog(this.getClass());
+
+ @EventListener
+ public void onApplicationReady(ApplicationReadyEvent event) {
+ ServiceInstance serviceInstance = DataWorkCloudApplication.getServiceInstance();
+ ECMApplication.setECMServiceInstance(serviceInstance);
+ ECMContext context = event.getApplicationContext().getBean(ECMContext.class);
+ ECMApplication.setContext(context);
+ ECMAsyncListenerBus emAsyncListenerBus = context.getECMAsyncListenerBus();
+ ECMSyncListenerBus emSyncListenerBus = context.getECMSyncListenerBus();
+ emAsyncListenerBus.start();
+ ECMReadyEvent ecmReadyEvent = new ECMReadyEvent(ECMApplication.getParmas());
+ emAsyncListenerBus.postToAll(ecmReadyEvent);
+ emSyncListenerBus.postToAll(ecmReadyEvent);
+ ECMApplication.setReady(true);
+ logger.info(String.format("ECM:%s is ready", serviceInstance));
+ }
+
+ @EventListener
+ public void onApplicationClosed(ContextClosedEvent contextClosedEvent) {
+ ServiceInstance serviceInstance = DataWorkCloudApplication.getServiceInstance();
+ ECMApplication.setReady(false);
+ ECMClosedEvent ecmClosedEvent = new ECMClosedEvent();
+ ECMApplication.getContext().getECMSyncListenerBus().postToAll(ecmClosedEvent);
+ ECMAsyncListenerBus ecmAsyncListenerBus = ECMApplication.getContext().getECMAsyncListenerBus();
+ ecmAsyncListenerBus.postToAll(ecmClosedEvent);
+ logger.info(String.format("wait ECM:%s asyncBus empty", serviceInstance));
+ try {
+ ecmAsyncListenerBus.waitUntilEmpty(ECM_ASYNC_BUS_WAITTOEMPTY_TIME());
+ } catch (Throwable e) {
+ logger.error("wait ECM asyncBus empty failed", e);
+ }
+ logger.info("ECM asyncBus is empty");
+ ecmAsyncListenerBus.stop();
+ logger.info("ECM is closed");
+ }
+
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/application.yml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/application.yml
new file mode 100644
index 0000000000..b644f75f86
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/application.yml
@@ -0,0 +1,29 @@
+server:
+ port: 9102
+spring:
+ application:
+ name: linkis-cg-engineconnmanager
+
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://127.0.0.1:20303/eureka/
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+ health:
+ db:
+ enabled: false
+logging:
+ config: classpath:log4j2.xml
+
+ribbon:
+ ReadTimeout: 60000
+ ConnectTimeout: 60000
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/linkis-server.properties b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/linkis-server.properties
new file mode 100644
index 0000000000..6c12e6aa27
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/linkis-server.properties
@@ -0,0 +1,18 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+##restful
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.em.restful
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/log4j2.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/log4j2.xml
new file mode 100644
index 0000000000..0f19fc3c17
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/resources/log4j2.xml
@@ -0,0 +1,33 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/conf/ECMConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/conf/ECMConfiguration.scala
new file mode 100644
index 0000000000..3ad3d73796
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/conf/ECMConfiguration.scala
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.conf
+
+import java.io.File
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType}
+import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils
+import com.webank.wedatasphere.linkis.governance.common.conf.GovernanceCommonConf
+
+import scala.concurrent.duration.Duration
+
+
+object ECMConfiguration {
+
+ //listenerbus
+ val ECM_ASYNC_BUS_CAPACITY: Int = CommonVars("wds.linkis.ecm.async.bus.capacity", 500).getValue
+ val ECM_ASYNC_BUS_NAME: String = CommonVars("wds.linkis.ecm.async.bus.name", "em_async_bus").getValue
+ val ECM_ASYNC_BUS_CONSUMER_SIZE: Int = CommonVars("wds.linkis.ecm.async.bus.consumer.size", 30).getValue
+ val ECM_ASYNC_BUS_THREAD_MAX_FREE_TIME: Long = CommonVars("wds.linkis.ecm.async.bus.max.free.time", ByteTimeUtils.timeStringAsMs("2m")).getValue
+ val ECM_ASYNC_BUS_WAITTOEMPTY_TIME: Long = CommonVars("wds.linkis.ecm.async.bus.waittoempty.time", 5000L).getValue
+
+ //resource
+ val ECM_MAX_MEMORY_AVAILABLE: Long = CommonVars[Long]("wds.linkis.ecm.memory.max", ByteTimeUtils.byteStringAsBytes("80g")).getValue
+ val ECM_MAX_CORES_AVAILABLE: Int = CommonVars[Integer]("wds.linkis.ecm.cores.max", 50).getValue
+ val ECM_MAX_CREATE_INSTANCES: Int = CommonVars[Integer]("wds.linkis.ecm.engineconn.instances.max", 50).getValue
+
+ val ECM_PROTECTED_MEMORY: Long = CommonVars[Long]("wds.linkis.ecm.protected.memory", ByteTimeUtils.byteStringAsBytes("4g")).getValue
+ val ECM_PROTECTED_CORES: Int = CommonVars[Integer]("wds.linkis.ecm.protected.cores.max", 2).getValue
+ val ECM_PROTECTED_INSTANCES: Int = CommonVars[Integer]("wds.linkis.ecm.protected.engine.instances", 2).getValue
+
+ val MANAGER_SPRING_NAME: String = GovernanceCommonConf.MANAGER_SPRING_NAME.getValue
+
+ val ENGINE_CONN_MANAGER_SPRING_NAME: String = GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue
+
+ val ECM_HEALTH_REPORT_PERIOD: Long = CommonVars("wds.linkis.ecm.health.report.period", 5).getValue
+
+ val ECM_HEALTH_REPORT_DELAY: Long = CommonVars("wds.linkis.ecm.health.report.delay", 10).getValue
+
+ val ENGINECONN_SPRING_NAME: String = CommonVars("wds.linkis.engineconn.spring.name", "linkis-cg-engineplugin").getValue
+
+ val ECM_HOME_DIR: String = CommonVars("wds.linkis.ecm.home.dir", this.getClass.getResource("/").getPath).getValue
+
+ val ENGINECONN_ROOT_DIR: String = CommonVars("wds.linkis.engineconn.root.dir", s"${ECM_HOME_DIR}engineConnRootDir").getValue
+
+ val ENGINECONN_PUBLIC_DIR: String = CommonVars("wds.linkis.engineconn.public.dir", s"$ENGINECONN_ROOT_DIR${File.separator}engineConnPublickDir").getValue
+
+ val ECM_LAUNCH_MAX_THREAD_SIZE: Int = CommonVars("wds.linkis.ecm.launch.max.thread.size", 100).getValue
+
+ val ECM_EUREKA_DEFAULTZONE: String = CommonVars("wds.linkis.eureka.defaultZone", "http://127.0.0.1:20303/eureka/").getValue
+
+ /**
+ * engineconn创建时间，如果为0，则使用ms中默认的
+ */
+ val ENGINECONN_CREATE_DURATION: Duration = Duration(CommonVars("wds.linkis.ecm.engineconn.create.duration", 0).getValue, TimeUnit.MILLISECONDS)
+
+ val WAIT_ENGINECONN_PID = CommonVars("wds.linkis.engineconn.wait.callback.pid", new TimeType("3s"))
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/context/DefaultECMContext.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/context/DefaultECMContext.scala
new file mode 100644
index 0000000000..226c0a4048
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/context/DefaultECMContext.scala
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.context
+
+import com.webank.wedatasphere.linkis.ecm.core.listener.{ECMAsyncListenerBus, ECMSyncListenerBus}
+import com.webank.wedatasphere.linkis.ecm.core.metrics.ECMMetrics
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration._
+import com.webank.wedatasphere.linkis.ecm.server.metrics.DefaultECMMetrics
+import org.springframework.stereotype.Component
+
+
+@Component
+class DefaultECMContext extends ECMContext {
+
+ private val emSyncListenerBus = new ECMSyncListenerBus
+
+ private val emAsyncListenerBus = new ECMAsyncListenerBus(ECM_ASYNC_BUS_CAPACITY, ECM_ASYNC_BUS_NAME)(ECM_ASYNC_BUS_CONSUMER_SIZE, ECM_ASYNC_BUS_THREAD_MAX_FREE_TIME)
+
+ private val metrics = new DefaultECMMetrics
+
+
+ override def getECMAsyncListenerBus: ECMAsyncListenerBus = emAsyncListenerBus
+
+ override def getECMSyncListenerBus: ECMSyncListenerBus = emSyncListenerBus
+
+ override def getECMMetrics: ECMMetrics = metrics
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/context/ECMContext.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/context/ECMContext.scala
new file mode 100644
index 0000000000..42312ff750
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/context/ECMContext.scala
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.context
+
+import com.webank.wedatasphere.linkis.ecm.core.listener.{ECMAsyncListenerBus, ECMSyncListenerBus}
+import com.webank.wedatasphere.linkis.ecm.core.metrics.ECMMetrics
+
+
+trait ECMContext {
+
+ def getECMAsyncListenerBus: ECMAsyncListenerBus
+
+ def getECMSyncListenerBus: ECMSyncListenerBus
+
+ def getECMMetrics:ECMMetrics
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/converter/ECMEngineConverter.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/converter/ECMEngineConverter.scala
new file mode 100644
index 0000000000..ebdec3affb
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/converter/ECMEngineConverter.scala
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.converter
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.{EngineConn, YarnEngineConn}
+import com.webank.wedatasphere.linkis.ecm.server.engineConn.DefaultYarnEngineConn
+import org.springframework.beans.BeanUtils
+
+
+object ECMEngineConverter {
+
+ def engineConn2YarnEngineConn(engineConn: EngineConn)(implicit updateFunction: YarnEngineConn => Unit): YarnEngineConn = {
+ val conn = new DefaultYarnEngineConn
+ BeanUtils.copyProperties(engineConn, conn)
+ updateFunction(conn)
+ conn
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/engineConn/DefaultEngineConn.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/engineConn/DefaultEngineConn.scala
new file mode 100644
index 0000000000..620eed4b8a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/engineConn/DefaultEngineConn.scala
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.engineConn
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.{EngineConn, EngineConnInfo}
+import com.webank.wedatasphere.linkis.ecm.core.launch.{EngineConnLaunchRunner, EngineConnManagerEnv}
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.NodeResource
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.EngineConnCreationDesc
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+
+class DefaultEngineConn extends EngineConn {
+
+ /**
+ * starting :初始状态
+ * success:成功启动并且安全退出
+ * Failed：启动失败
+ * Running：容器运行中
+ */
+ @volatile private var status: NodeStatus = _
+
+ private var tickedId: String = _
+
+ private var resource: NodeResource = _
+
+ private var labels: util.List[Label[_]] = _
+
+ private var engineConnCreationDesc: EngineConnCreationDesc = _
+
+ private var engineConnInfo: EngineConnInfo = _
+
+ private var ecmEnv: EngineConnManagerEnv = _
+
+ private var engineConnLaunchRunner: EngineConnLaunchRunner = _
+
+ private var instance: ServiceInstance = _
+
+ private var pid: String = _
+
+ override def getResource: NodeResource = resource
+
+ override def setResource(resource: NodeResource): Unit = this.resource = resource
+
+ override def getLabels: util.List[Label[_]] = labels
+
+ override def setLabels(labels: util.List[Label[_]]): Unit = this.labels = labels
+
+ override def getStatus: NodeStatus = status
+
+ override def getCreationDesc: EngineConnCreationDesc = engineConnCreationDesc
+
+ override def getEngineConnInfo: EngineConnInfo = engineConnInfo
+
+ override def getEngineConnLaunchRunner: EngineConnLaunchRunner = engineConnLaunchRunner
+
+ override def setEngineConnLaunchRunner(runner: EngineConnLaunchRunner): Unit = this.engineConnLaunchRunner = runner
+
+ override def setEngineConnInfo(engineConnInfo: EngineConnInfo): Unit = this.engineConnInfo = engineConnInfo
+
+ override def getTickedId: String = tickedId
+
+ override def setTickedId(tickedId: String): Unit = this.tickedId = tickedId
+
+ override def setStatus(status: NodeStatus): Unit = this.status = status
+
+ override def setCreationDesc(engineConnCreationDesc: EngineConnCreationDesc): Unit = this.engineConnCreationDesc = engineConnCreationDesc
+
+ override def getServiceInstance: ServiceInstance = instance
+
+ override def setServiceInstance(instance: ServiceInstance): Unit = this.instance = instance
+
+ override def getPid: String = pid
+
+ override def setPid(pid: String): Unit = this.pid = pid
+
+ override def getEngineConnManagerEnv: EngineConnManagerEnv = ecmEnv
+
+ override def setEngineConnManagerEnv(env: EngineConnManagerEnv): Unit = this.ecmEnv = env
+
+ override def toString = s"DefaultEngineConn($status, $tickedId, $resource, $labels, $engineConnCreationDesc, $engineConnInfo, $ecmEnv, $engineConnLaunchRunner, $instance, $pid)"
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/engineConn/DefaultYarnEngineConn.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/engineConn/DefaultYarnEngineConn.scala
new file mode 100644
index 0000000000..001361d4e2
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/engineConn/DefaultYarnEngineConn.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.engineConn
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.YarnEngineConn
+
+
+class DefaultYarnEngineConn extends DefaultEngineConn with YarnEngineConn {
+
+ var applicationId: String = _
+
+ var applicationURL: String = _
+
+ var yarnMode: String = _
+
+ override def getApplicationId: String = applicationId
+
+ override def setApplicationId(applicationId: String): Unit = this.applicationId = applicationId
+
+ override def getApplicationURL: String = applicationURL
+
+ override def setApplicationURL(applicationURL: String): Unit = this.applicationURL = applicationURL
+
+ override def getYarnMode: String = yarnMode
+
+ override def setYarnMode(yarnMode: String): Unit = this.yarnMode = yarnMode
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/exception/ECMErrorException.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/exception/ECMErrorException.scala
new file mode 100644
index 0000000000..03f6b9a4ec
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/exception/ECMErrorException.scala
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.exception
+
+import com.webank.wedatasphere.linkis.common.exception.{ErrorException, WarnException}
+
+
+class ECMErrorException(errorCode: Int, desc: String) extends ErrorException(errorCode, desc) {
+ def this(errorCode: Int, desc: String, t: Throwable) {
+ this(errorCode, desc)
+ this.initCause(t)
+ }
+}
+
+class ECMWarnException(errCode: Int, desc: String) extends WarnException(errCode, desc)
+
+
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/listener/ECMReadyEvent.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/listener/ECMReadyEvent.scala
new file mode 100644
index 0000000000..ba6fa5c628
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/listener/ECMReadyEvent.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.listener
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.ecm.core.listener.ECMEvent
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.ResponseEngineConnPid
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol}
+
+
+case class ECMReadyEvent(params: Array[String]) extends ECMEvent
+
+case class ECMClosedEvent() extends ECMEvent
+
+case class EngineConnStatusChageEvent(from: NodeStatus, to: NodeStatus) extends ECMEvent
+
+case class YarnAppIdCallbackEvent(protocol: YarnAPPIdCallbackProtocol) extends ECMEvent
+
+case class YarnInfoCallbackEvent(protocol: YarnInfoCallbackProtocol) extends ECMEvent
+
+case class EngineConnPidCallbackEvent(protocol: ResponseEngineConnPid) extends ECMEvent
+
+case class EngineConnAddEvent(conn: EngineConn) extends ECMEvent
+
+case class EngineConnStatusChangeEvent(tickedId: String, updateStatus: NodeStatus) extends ECMEvent
+
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/metrics/DefaultECMMetrics.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/metrics/DefaultECMMetrics.scala
new file mode 100644
index 0000000000..5b24a7d8b2
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/metrics/DefaultECMMetrics.scala
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.metrics
+
+
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicInteger
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.ecm.core.metrics.ECMMetrics
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus._
+
+import scala.collection.JavaConversions._
+
+
+class DefaultECMMetrics extends ECMMetrics {
+
+
+ private val startingEngineConnCount = new AtomicInteger(0)
+
+ private val runningEngineConnCount = new AtomicInteger(0)
+
+ private val successEngineConnCount = new AtomicInteger(0)
+
+ private val failedEngineConnCount = new AtomicInteger(0)
+
+ private val startingEngineConnMap = new ConcurrentHashMap[String, EngineConn](64)
+
+ private val runningEngineConnMap = new ConcurrentHashMap[String, EngineConn](64)
+
+ private val successEngineConnMap = new ConcurrentHashMap[String, EngineConn](64)
+
+ private val failedEngineConnMap = new ConcurrentHashMap[String, EngineConn](64)
+
+ override def getStartingEngineConns: Array[EngineConn] = getEngineConns(startingEngineConnMap)
+
+ override def getRunningEngineConns: Array[EngineConn] = getEngineConns(runningEngineConnMap)
+
+ override def getSucceedEngineConns: Array[EngineConn] = getEngineConns(successEngineConnMap)
+
+ override def getFailedEngineConns: Array[EngineConn] = getEngineConns(failedEngineConnMap)
+
+ private val getEngineConns = (map: ConcurrentHashMap[String, EngineConn]) => map.values().toSeq.toArray
+
+ private val decreaseEngineConnMetric = (engineConn: EngineConn, map: ConcurrentHashMap[String, EngineConn], count: AtomicInteger) => {
+ val conn = map.remove(engineConn.getTickedId)
+ if (conn != null) {
+ count.decrementAndGet()
+ }
+ }
+
+ private val increaseEngineConnMetric = (engineConn: EngineConn, map: ConcurrentHashMap[String, EngineConn], count: AtomicInteger, status: NodeStatus) => {
+ if (engineConn.getStatus.equals(status)) {
+ count.incrementAndGet()
+ map.put(engineConn.getTickedId, engineConn)
+ }
+ }
+
+ override def increaseStartingEngineConn(engineConn: EngineConn): Unit = {
+ increaseEngineConnMetric(engineConn, startingEngineConnMap, startingEngineConnCount, Starting)
+ }
+
+ def decreseStartingEngineConn(engineConn: EngineConn): Unit = {
+ decreaseEngineConnMetric(engineConn, startingEngineConnMap, startingEngineConnCount)
+ }
+
+ override def increaseRunningEngineConn(engineConn: EngineConn): Unit = {
+ increaseEngineConnMetric(engineConn, runningEngineConnMap, runningEngineConnCount, Running)
+ }
+
+ def decreseRunningEngineConn(engineConn: EngineConn): Unit = {
+ decreaseEngineConnMetric(engineConn, runningEngineConnMap, runningEngineConnCount)
+ }
+
+ override def increaseSuccessEngineConn(engineConn: EngineConn): Unit = {
+ increaseEngineConnMetric(engineConn, successEngineConnMap, successEngineConnCount, Success)
+ }
+
+ def decreaseSuccessEngineConn(engineConn: EngineConn): Unit = {
+ decreaseEngineConnMetric(engineConn, successEngineConnMap, successEngineConnCount)
+ }
+
+ override def increaseFailedEngineConn(engineConn: EngineConn): Unit = {
+ increaseEngineConnMetric(engineConn, failedEngineConnMap, failedEngineConnCount, Failed)
+ }
+
+ def decreaseFailedEngineConn(engineConn: EngineConn): Unit = {
+ decreaseEngineConnMetric(engineConn, failedEngineConnMap, failedEngineConnCount)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/report/DefaultECMHealthReport.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/report/DefaultECMHealthReport.scala
new file mode 100644
index 0000000000..de3e6e75d4
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/report/DefaultECMHealthReport.scala
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.report
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.ecm.core.report.ECMHealthReport
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.metrics.NodeOverLoadInfo
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.Resource
+
+
+class DefaultECMHealthReport extends ECMHealthReport {
+
+ private var protectedResource: Resource = _
+
+ private var totalResource: Resource = _
+
+ private var usedResource: Resource = _
+
+ private var overload: NodeOverLoadInfo = _
+
+ private var runningEngineConns: Array[EngineConn] = _
+
+ private var reportTime: Long = _
+
+ private var nodeStatus: NodeStatus = _
+
+ private var nodeMsg: String = _
+
+ private var nodeId: String = _
+
+ override def setProtectedResource(protectedResource: Resource): Unit = this.protectedResource = protectedResource
+
+ override def getProtectResource: Resource = protectedResource
+
+ override def setOverload(overload: NodeOverLoadInfo): Unit = this.overload = overload
+
+ override def getOverload: NodeOverLoadInfo = overload
+
+ override def setRunningEngineConns(runningEngines: Array[EngineConn]): Unit = this.runningEngineConns = runningEngineConns
+
+ override def getRunningEngineConns: Array[EngineConn] = runningEngineConns
+
+ override def getReportTime: Long = reportTime
+
+ override def setReportTime(reportTime: Long): Unit = this.reportTime = reportTime
+
+ override def getNodeStatus: NodeStatus = nodeStatus
+
+ override def setNodeStatus(nodeStatus: NodeStatus): Unit = this.nodeStatus = nodeStatus
+
+ override def setNodeMsg(nodeMsg: String): Unit = this.nodeMsg = nodeMsg
+
+ override def getNodeMsg: String = nodeMsg
+
+ override def getUsedResource: Resource = usedResource
+
+ override def setUsedResource(usedResource: Resource): Unit = this.usedResource = usedResource
+
+ override def getTotalResource: Resource = totalResource
+
+ override def setTotalResource(totalResource: Resource): Unit = this.totalResource = totalResource
+
+ override def setNodeId(nodeId: String): Unit = this.nodeId = nodeId
+
+ override def getNodeId: String = nodeId
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMHealthService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMHealthService.scala
new file mode 100644
index 0000000000..2df4e9d45a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMHealthService.scala
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.ecm.core.report.ECMHealthReport
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHeartbeatRequest}
+
+
+trait ECMHealthService{
+
+ def getLastEMHealthReport: ECMHealthReport
+
+ def reportHealth(report: ECMHealthReport): Unit
+
+ def generateHealthReport(reportTime:Long): ECMHealthReport
+
+ def dealNodeHeartbeatRequest(nodeHeartbeatRequest: NodeHeartbeatRequest): NodeHeartbeatMsg
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMMetricsService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMMetricsService.scala
new file mode 100644
index 0000000000..408bf25c62
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMMetricsService.scala
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+
+trait ECMMetricsService {
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMRegisterService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMRegisterService.scala
new file mode 100644
index 0000000000..2ade991a79
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ECMRegisterService.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.manager.common.protocol.em.{RegisterEMRequest, StopEMRequest}
+
+
+trait ECMRegisterService {
+
+ def registerECM(event: RegisterEMRequest): Unit
+
+ def unRegisterECM(event: StopEMRequest): Unit
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnKillService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnKillService.scala
new file mode 100644
index 0000000000..de7c5b8528
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnKillService.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.{EngineStopRequest, EngineStopResponse}
+
+
+trait EngineConnKillService {
+
+ def dealEngineConnStop(engineStopRequest: EngineStopRequest): EngineStopResponse
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnLaunchService.scala
new file mode 100644
index 0000000000..c98801d374
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnLaunchService.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.manager.common.entity.node.EngineNode
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.{EngineConnBuildRequest, EngineConnLaunchRequest}
+
+
+
+trait EngineConnLaunchService {
+ /**
+ * launchEngine启动一个引擎的方法
+ *
+ * @param engineConnBuildRequest 封装了引擎启动的参数
+ */
+ def launchEngineConn(engineConnBuildRequest: EngineConnBuildRequest): EngineNode
+
+ def launchEngineConn(engineConnLaunchRequest: EngineConnLaunchRequest, duration: Long): EngineNode
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnListService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnListService.scala
new file mode 100644
index 0000000000..bfb7ef8942
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnListService.scala
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import java.util
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.ecm.core.launch.EngineConnLaunchRunner
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.Resource
+
+
+/**
+ * The enginelistservice interface manages the interface started by the engine
+ * The most important submit method is to put the thread that starts the engine into the thread pool to start
+ * EngineListService接口管理引擎启动的接口
+ * 最重要的submit方法是将启动引擎的线程放入到线程池中进行启动
+ */
+trait EngineConnListService {
+
+ def init(): Unit
+
+ def getEngineConn(engineConnId: String): Option[EngineConn]
+
+ def getEngineConns: util.List[EngineConn]
+
+ def addEngineConn(engineConn: EngineConn): Unit
+
+ def killEngineConn(engineConnId: String): Unit
+
+ def getUsedResources: Resource
+
+ def submit(runner: EngineConnLaunchRunner): Option[EngineConn]
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnPidCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnPidCallbackService.scala
new file mode 100644
index 0000000000..4d751de00b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnPidCallbackService.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.ResponseEngineConnPid
+
+
+trait EngineConnPidCallbackService {
+
+
+ def dealPid(protocol: ResponseEngineConnPid): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnStatusCallbackService.scala
new file mode 100644
index 0000000000..8e234ded25
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/EngineConnStatusCallbackService.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineConnStatusCallback
+
+
+trait EngineConnStatusCallbackService {
+
+ def dealEngineConnStatusCallback(protocol: EngineConnStatusCallback): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/LocalDirsHandleService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/LocalDirsHandleService.scala
new file mode 100644
index 0000000000..6ff1b35092
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/LocalDirsHandleService.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+
+trait LocalDirsHandleService {
+
+ def getEngineConnManagerHomeDir: String
+
+ def getEngineConnWorkDir(user: String, ticketId: String): String
+
+ def getEngineConnPublicDir: String
+
+ def getEngineConnLogDir(user: String, ticketId: String): String
+
+ def getEngineConnTmpDir(user: String, ticketId: String): String
+
+ def cleanup(): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/LogCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/LogCallbackService.scala
new file mode 100644
index 0000000000..74f829d20d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/LogCallbackService.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.protocol.callback.LogCallbackProtocol
+
+
+
+trait LogCallbackService {
+
+ def dealLog(protocol: LogCallbackProtocol): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ResourceLocalizationService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ResourceLocalizationService.scala
new file mode 100644
index 0000000000..0a74ea8dcf
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/ResourceLocalizationService.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest
+
+
+trait ResourceLocalizationService {
+
+ def handleInitEngineConnResources(request: EngineConnLaunchRequest, engineConn: EngineConn): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/YarnCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/YarnCallbackService.scala
new file mode 100644
index 0000000000..4ed8ecf75f
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/YarnCallbackService.scala
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service
+
+import com.webank.wedatasphere.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol}
+
+
+trait YarnCallbackService {
+
+ def dealApplicationId(protocol: YarnAPPIdCallbackProtocol): Unit
+
+ def dealApplicationURI(protocol: YarnInfoCallbackProtocol): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala
new file mode 100644
index 0000000000..7a0e7d4335
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.{EngineConn, EngineConnInfo}
+import com.webank.wedatasphere.linkis.ecm.core.launch._
+import com.webank.wedatasphere.linkis.ecm.server.ECMApplication
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration._
+import com.webank.wedatasphere.linkis.ecm.server.engineConn.DefaultEngineConn
+import com.webank.wedatasphere.linkis.ecm.server.listener.{EngineConnAddEvent, EngineConnStatusChangeEvent}
+import com.webank.wedatasphere.linkis.ecm.server.service.{EngineConnLaunchService, ResourceLocalizationService}
+import com.webank.wedatasphere.linkis.ecm.server.util.ECMUtils
+import com.webank.wedatasphere.linkis.governance.common.conf.GovernanceCommonConf
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus.Failed
+import com.webank.wedatasphere.linkis.manager.common.entity.node.{AMEngineNode, EngineNode}
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest
+import com.webank.wedatasphere.linkis.rpc.Sender
+import org.apache.commons.lang.exception.ExceptionUtils
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContextExecutorService, Future}
+import scala.util.{Failure, Success}
+
+
+abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService with Logging {
+
+
+ protected implicit val executor: ExecutionContextExecutorService = Utils.newCachedExecutionContext(ECM_LAUNCH_MAX_THREAD_SIZE, "EngineConn-Manager-Thread-")
+
+ protected var resourceLocalizationService: ResourceLocalizationService = _
+
+ def setResourceLocalizationService(service: ResourceLocalizationService): Unit = this.resourceLocalizationService = service
+
+
+ def beforeLaunch(conn: EngineConn, duration: Long): Unit = {}
+
+ def afterLaunch(conn: EngineConn, duration: Long): Unit = {}
+
+ override def launchEngineConn(request: EngineConnLaunchRequest, duration: Long): EngineNode = {
+ //1.创建engineConn和runner,launch 并设置基础属性
+ val conn = createEngineConn
+ val runner = createEngineConnLaunchRunner
+ val launch = createEngineConnLaunch
+ launch.setEngineConnLaunchRequest(request)
+ runner.setEngineConnLaunch(launch)
+ conn.setEngineConnLaunchRunner(runner)
+ conn.setLabels(request.labels)
+ conn.setCreationDesc(request.creationDesc)
+ conn.setResource(request.nodeResource)
+ conn.setTickedId(request.ticketId)
+ conn.setStatus(NodeStatus.Starting)
+ conn.setEngineConnInfo(new EngineConnInfo)
+ //2.资源本地化，并且设置ecm的env环境信息
+ getResourceLocalizationServie.handleInitEngineConnResources(request, conn)
+ //3.添加到list
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnAddEvent(conn))
+ //4.run
+ try {
+ beforeLaunch(conn, duration)
+ runner.run()
+ launch match {
+ case pro: ProcessEngineConnLaunch =>
+ val serviceInstance = ServiceInstance(GovernanceCommonConf.ENGINE_CONN_SPRING_NAME.getValue, ECMUtils.getInstanceByPort(pro.getEngineConnPort))
+ conn.setServiceInstance(serviceInstance)
+ case _ =>
+ }
+
+ val future = Future {
+ afterLaunch(conn, duration)
+ }
+
+ future onComplete {
+ case Failure(t) =>
+ throw t
+ case Success(_) =>
+ info(s"init ${conn.getServiceInstance} succeed.")
+ }
+ //超时忽略，如果状态翻转了则直接返回
+ Utils.tryQuietly(Await.result(future, Duration(WAIT_ENGINECONN_PID.getValue.toLong, TimeUnit.MILLISECONDS)))
+ } catch {
+ //failed，1.被ms打断，2.超时，3.普通错误，比如process
+ case t: Throwable =>
+ error(s"init ${conn.getServiceInstance} failed, now stop and delete it. message: ${t.getMessage}")
+ conn.getEngineConnLaunchRunner.stop()
+ Sender.getSender(MANAGER_SPRING_NAME).send(EngineConnStatusCallbackToAM(conn.getServiceInstance,
+ NodeStatus.ShuttingDown, " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t)))
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(conn.getTickedId, Failed))
+ throw t
+ }
+ val engineNode = new AMEngineNode()
+ engineNode.setLabels(conn.getLabels)
+
+ engineNode.setServiceInstance(conn.getServiceInstance)
+ engineNode.setOwner(request.user)
+ engineNode.setMark("process")
+ engineNode
+ }
+
+ def createEngineConn: EngineConn = new DefaultEngineConn
+
+
+ def createEngineConnLaunchRunner: EngineConnLaunchRunner = new EngineConnLaunchRunnerImpl
+
+
+ def createEngineConnLaunch: EngineConnLaunch
+
+
+ def getResourceLocalizationServie: ResourceLocalizationService = {
+ // TODO: null 抛出异常
+ this.resourceLocalizationService
+ }
+
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
new file mode 100644
index 0000000000..d8c292c02a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import java.io.File
+import java.nio.file.Paths
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.common.utils.{Utils, ZipUtils}
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.ecm.core.launch.EngineConnManagerEnv
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration._
+import com.webank.wedatasphere.linkis.ecm.server.service.{LocalDirsHandleService, ResourceLocalizationService}
+import com.webank.wedatasphere.linkis.ecm.server.util.ECMUtils
+import com.webank.wedatasphere.linkis.manager.common.protocol.bml.BmlResource
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.process.ProcessEngineConnLaunchRequest
+import com.webank.wedatasphere.linkis.storage.FSFactory
+import com.webank.wedatasphere.linkis.storage.fs.FileSystem
+import com.webank.wedatasphere.linkis.storage.utils.{FileSystemUtils, StorageUtils}
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable
+
+
+class BmlResourceLocalizationService extends ResourceLocalizationService {
+
+ private implicit val fs: FileSystem = FSFactory.getFs(StorageUtils.FILE).asInstanceOf[FileSystem]
+
+ fs.init(null)
+
+ private val seperator = File.separator
+
+ private val schema = StorageUtils.FILE_SCHEMA
+
+ private var localDirsHandleService: LocalDirsHandleService = _
+
+ def setLocalDirsHandleService(localDirsHandleService: LocalDirsHandleService): Unit = this.localDirsHandleService = localDirsHandleService
+
+ override def handleInitEngineConnResources(request: EngineConnLaunchRequest, engineConn: EngineConn): Unit = {
+ // TODO: engineType判断是否下载到本地 unzip
+ //engine_type resourceId version判断是否更新，或者重新下载，将path给到properties
+ request match {
+ case request: ProcessEngineConnLaunchRequest =>
+ val files = request.bmlResources
+ val linkDirsP = new mutable.HashMap[String, String]
+ val user = request.user
+ val ticketId = request.ticketId
+ val workDir = createDirIfNotExit(localDirsHandleService.getEngineConnWorkDir(user, ticketId))
+ val emHomeDir = createDirIfNotExit(localDirsHandleService.getEngineConnManagerHomeDir)
+ val logDirs = createDirIfNotExit(localDirsHandleService.getEngineConnLogDir(user, ticketId))
+ val tmpDirs = createDirIfNotExit(localDirsHandleService.getEngineConnTmpDir(user, ticketId))
+ files.foreach(downloadBmlResource(request, linkDirsP, _, workDir))
+ engineConn.getEngineConnLaunchRunner.getEngineConnLaunch.setEngineConnManagerEnv(new EngineConnManagerEnv {
+ override val engineConnManagerHomeDir: String = emHomeDir
+ override val engineConnWorkDir: String = workDir
+ override val engineConnLogDirs: String = logDirs
+ override val engineConnTempDirs: String = tmpDirs
+ override val engineConnManagerHost: String = Utils.getComputerName
+ override val engineConnManagerPort: String = DataWorkCloudApplication.getApplicationContext.getEnvironment.getProperty("server.port")
+ override val linkDirs: Map[String, String] = linkDirsP.toMap
+ // TODO: 注册发现信息的配置化
+ override val properties: Map[String, String] = Map("eureka.client.serviceUrl.defaultZone" -> ECM_EUREKA_DEFAULTZONE)
+ })
+ case _ =>
+ }
+ }
+
+
+ private val bmlResourceSuffix = ".zip"
+
+ private def createDirIfNotExit(noSchemaPath: String): String = {
+ val fsPath = new FsPath(schema + noSchemaPath)
+ if (!fs.exists(fsPath)) {
+ FileSystemUtils.mkdirs(fs, fsPath, Utils.getJvmUser)
+ fs.setPermission(fsPath, "rwxrwx---")
+ }
+ noSchemaPath
+ }
+
+ def downloadBmlResource(request: ProcessEngineConnLaunchRequest, linkDirs: mutable.HashMap[String, String], resource: BmlResource, workDir: String): Unit = {
+ val resourceId = resource.getResourceId
+ val version = resource.getVersion
+ val user = request.user
+ resource.getVisibility match {
+ case BmlResource.BmlResourceVisibility.Public =>
+ val publicDir = localDirsHandleService.getEngineConnPublicDir
+ val bmlResourceDir = schema + Paths.get(publicDir, resourceId, version).toFile.getPath
+ val fsPath = new FsPath(bmlResourceDir)
+ if (!fs.exists(fsPath)) {
+ ECMUtils.downLoadBmlResourceToLocal(resource, user, fsPath.getPath)
+ val unzipDir = fsPath.getSchemaPath + File.separator + resource.getFileName.substring(0, resource.getFileName.lastIndexOf("."))
+ FileSystemUtils.mkdirs(fs, new FsPath(unzipDir), Utils.getJvmUser)
+ ZipUtils.unzip(bmlResourceDir + File.separator + resource.getFileName, unzipDir)
+ fs.delete(new FsPath(bmlResourceDir + File.separator + resource.getFileName))
+ }
+ //2.软连，并且添加到map
+ val dirAndFileList = fs.listPathWithError(fsPath)
+ dirAndFileList.getFsPaths.foreach {
+ case path: FsPath =>
+ val name = new File(path.getPath).getName
+ linkDirs.put(path.getPath, workDir + seperator + name)
+ }
+ case BmlResource.BmlResourceVisibility.Private =>
+ val fsPath = new FsPath(schema + workDir)
+ if (!fs.exists(fsPath)) {
+ FileSystemUtils.mkdirs(fs, fsPath, Utils.getJvmUser)
+ ECMUtils.downLoadBmlResourceToLocal(resource, user, fsPath.getPath)
+ ZipUtils.unzip(schema + workDir + File.separator + resource.getFileName, fsPath.getSchemaPath)
+ fs.delete(new FsPath(schema + workDir + File.separator + resource.getFileName))
+ }
+ case BmlResource.BmlResourceVisibility.Label =>
+ }
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMHealthService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMHealthService.scala
new file mode 100644
index 0000000000..feb5cb1074
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMHealthService.scala
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import java.util.Date
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.linkis.common.utils.Utils
+import com.webank.wedatasphere.linkis.ecm.core.listener.{ECMEvent, ECMEventListener}
+import com.webank.wedatasphere.linkis.ecm.core.report.ECMHealthReport
+import com.webank.wedatasphere.linkis.ecm.server.ECMApplication
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration._
+import com.webank.wedatasphere.linkis.ecm.server.listener.{ECMClosedEvent, ECMReadyEvent}
+import com.webank.wedatasphere.linkis.ecm.server.report.DefaultECMHealthReport
+import com.webank.wedatasphere.linkis.ecm.server.service.{ECMHealthService, EngineConnListService}
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus}
+import com.webank.wedatasphere.linkis.manager.common.entity.metrics.{NodeHealthyInfo, NodeOverLoadInfo}
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.{CommonNodeResource, LoadInstanceResource}
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHeartbeatRequest}
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.rpc.Sender
+import org.springframework.beans.factory.annotation.Autowired
+
+
+class DefaultECMHealthService extends ECMHealthService with ECMEventListener {
+
+ val maxResource = new LoadInstanceResource(ECM_MAX_MEMORY_AVAILABLE, ECM_MAX_CORES_AVAILABLE, ECM_MAX_CREATE_INSTANCES)
+ val minResource = new LoadInstanceResource(ECM_PROTECTED_MEMORY, ECM_PROTECTED_CORES, ECM_PROTECTED_INSTANCES)
+ private val runtime: Runtime = Runtime.getRuntime
+
+ private val future = Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
+ override def run(): Unit = {
+ if (ECMApplication.isReady) {
+ reportHealth(getLastEMHealthReport)
+ }
+ }
+ }, ECM_HEALTH_REPORT_DELAY, ECM_HEALTH_REPORT_PERIOD, TimeUnit.SECONDS)
+
+ @Autowired
+ private var engineConnListService: EngineConnListService = _
+
+ override def getLastEMHealthReport: ECMHealthReport = {
+ val report = new DefaultECMHealthReport
+ report.setNodeId(ECMApplication.getECMServiceInstance.toString)
+ report.setNodeStatus(NodeStatus.Running)
+ report.setTotalResource(maxResource)
+ report.setProtectedResource(minResource)
+ report.setUsedResource(engineConnListService.getUsedResources)
+ report.setReportTime(new Date().getTime)
+ report.setRunningEngineConns(ECMApplication.getContext.getECMMetrics.getRunningEngineConns)
+ val info = new NodeOverLoadInfo
+ info.setMaxMemory(runtime.maxMemory())
+ info.setUsedMemory(runtime.totalMemory() - runtime.freeMemory())
+ // TODO: 根据系统获取当前操作系统负载
+ report.setOverload(info)
+ report
+ }
+
+ // TODO: 可能还需要个判断health状态的方法
+
+ override def reportHealth(report: ECMHealthReport): Unit = {
+ val heartbeat: NodeHeartbeatMsg = transferECMHealthReportToNodeHeartbeatMsg(report)
+ Sender.getSender(MANAGER_SPRING_NAME).send(heartbeat)
+ }
+
+ private def transferECMHealthReportToNodeHeartbeatMsg(report: ECMHealthReport) = {
+ val heartbeat = new NodeHeartbeatMsg
+ heartbeat.setOverLoadInfo(report.getOverload)
+ heartbeat.setStatus(report.getNodeStatus)
+ heartbeat.setServiceInstance(ECMApplication.getECMServiceInstance)
+ val resource = new CommonNodeResource
+ resource.setMaxResource(maxResource)
+ resource.setMinResource(minResource)
+ resource.setUsedResource(resource.getUsedResource)
+ heartbeat.setNodeResource(resource)
+ heartbeat.setHeartBeatMsg("")
+ val nodeHealthyInfo = new NodeHealthyInfo
+ nodeHealthyInfo.setMsg("")
+ nodeHealthyInfo.setNodeHealthy(NodeHealthy.Healthy)
+ heartbeat.setHealthyInfo(nodeHealthyInfo)
+ heartbeat
+ }
+
+ override def generateHealthReport(reportTime: Long): ECMHealthReport = {
+ // TODO: 历史查询
+ null
+ }
+
+ private def emShutdownHealthReport(event: ECMClosedEvent): Unit = {
+ val report = getLastEMHealthReport
+ report.setNodeStatus(NodeStatus.ShuttingDown)
+ reportHealth(report)
+ }
+
+ private def emReadyHealthReport(event: ECMReadyEvent): Unit = reportHealth(getLastEMHealthReport)
+
+
+ override def onEvent(event: ECMEvent): Unit = event match {
+ case event: ECMReadyEvent =>
+ emReadyHealthReport(event)
+ case event: ECMClosedEvent =>
+ cancelHealthReportThread(event)
+ emShutdownHealthReport(event)
+ presistenceLeftReports(event)
+ case _ =>
+ }
+
+ private def cancelHealthReportThread(event: ECMClosedEvent): Unit = {
+
+ }
+
+ private def presistenceLeftReports(event: ECMClosedEvent): Unit = {
+ // TODO: 持久化 剩余的reports
+ }
+
+ @Receiver
+ override def dealNodeHeartbeatRequest(nodeHeartbeatRequest: NodeHeartbeatRequest): NodeHeartbeatMsg = {
+ val hearlthReport = getLastEMHealthReport
+ transferECMHealthReportToNodeHeartbeatMsg(hearlthReport)
+ }
+
+
+ }
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMMetricsService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMMetricsService.scala
new file mode 100644
index 0000000000..9185b63055
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMMetricsService.scala
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import com.webank.wedatasphere.linkis.ecm.core.listener.{ECMEvent, ECMEventListener}
+import com.webank.wedatasphere.linkis.ecm.server.service.ECMMetricsService
+
+
+class DefaultECMMetricsService extends ECMMetricsService with ECMEventListener{
+ override def onEvent(event: ECMEvent): Unit = ???
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala
new file mode 100644
index 0000000000..7607230a65
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import java.util
+import java.util.Collections
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.ecm.core.listener.{ECMEvent, ECMEventListener}
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration._
+import com.webank.wedatasphere.linkis.ecm.server.listener.{ECMClosedEvent, ECMReadyEvent}
+import com.webank.wedatasphere.linkis.ecm.server.service.ECMRegisterService
+import com.webank.wedatasphere.linkis.manager.common.entity.resource._
+import com.webank.wedatasphere.linkis.manager.common.protocol.em.{RegisterEMRequest, StopEMRequest}
+import com.webank.wedatasphere.linkis.manager.label.constant.LabelKeyConstant
+import com.webank.wedatasphere.linkis.rpc.Sender
+
+
+class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener with Logging {
+
+ private implicit def readyEvent2RegisterECMRequest(event: ECMReadyEvent): RegisterEMRequest = {
+ val request = new RegisterEMRequest
+ val instance = Sender.getThisServiceInstance
+ request.setUser(Utils.getJvmUser)
+ request.setServiceInstance(instance)
+ request.setAlias(instance.getApplicationName)
+ request.setLabels(getLabelsFromArgs(event.params))
+ request.setNodeResource(getEMRegiterResourceFromConfiguration)
+ request
+ }
+
+ private def getLabelsFromArgs(params: Array[String]): util.Map[String, AnyRef] = {
+ import scala.collection.JavaConversions._
+ val labelRegex = """label\.(.+)\.(.+)=(.+)""".r
+ val labels = new util.HashMap[String, AnyRef]()
+ // TODO: magic
+ labels += LabelKeyConstant.SERVER_ALIAS_KEY -> Collections.singletonMap("alias", ENGINE_CONN_MANAGER_SPRING_NAME)
+ // TODO: group by key
+ /* params.foreach {
+ case labelRegex(key, valueKey, valueContent) => labels += key -> (valueKey, valueContent)
+ case _ =>
+ }*/
+ labels
+ }
+
+ private def getEMRegiterResourceFromConfiguration: NodeResource = {
+ val maxResource = new LoadInstanceResource(ECM_MAX_MEMORY_AVAILABLE, ECM_MAX_CORES_AVAILABLE, ECM_MAX_CREATE_INSTANCES)
+ val minResource = new LoadInstanceResource(ECM_PROTECTED_MEMORY, ECM_PROTECTED_CORES, ECM_PROTECTED_INSTANCES)
+ val nodeResource = new CommonNodeResource
+ nodeResource.setResourceType(ResourceType.LoadInstance)
+ nodeResource.setExpectedResource(Resource.getZeroResource(maxResource))
+ nodeResource.setLeftResource(maxResource)
+ nodeResource.setLockedResource(Resource.getZeroResource(maxResource))
+ nodeResource.setMaxResource(maxResource)
+ nodeResource.setMinResource(minResource)
+ nodeResource.setUsedResource(Resource.getZeroResource(maxResource))
+ nodeResource
+ }
+
+ override def onEvent(event: ECMEvent): Unit = event match {
+ case event: ECMReadyEvent => registerECM(event)
+ case event: ECMClosedEvent => unRegisterECM(event)
+ case _ =>
+ }
+
+ private implicit def closeEvent2StopECMRequest(event: ECMClosedEvent): StopEMRequest = {
+ val request = new StopEMRequest
+ val instance = Sender.getThisServiceInstance
+ request.setUser(Utils.getJvmUser)
+ request.setEm(instance)
+ request
+ }
+
+ override def registerECM(request: RegisterEMRequest): Unit = {
+ info("start register ecm")
+ Sender.getSender(MANAGER_SPRING_NAME).send(request)
+ }
+
+ override def unRegisterECM(request: StopEMRequest): Unit = {
+ info("start unRegister ecm")
+ Sender.getSender(MANAGER_SPRING_NAME).send(request)
+ }
+
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java
new file mode 100644
index 0000000000..bb139669a7
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl;
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance;
+import com.webank.wedatasphere.linkis.common.utils.Utils;
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn;
+import com.webank.wedatasphere.linkis.ecm.server.service.EngineConnKillService;
+import com.webank.wedatasphere.linkis.ecm.server.service.EngineConnListService;
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineStopRequest;
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineStopResponse;
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineSuicideRequest;
+import com.webank.wedatasphere.linkis.message.annotation.Receiver;
+import com.webank.wedatasphere.linkis.rpc.Sender;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class DefaultEngineConnKillService implements EngineConnKillService {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultEngineConnKillService.class);
+
+ private EngineConnListService engineConnListService;
+
+ public void setEngineConnListService(EngineConnListService engineConnListService) {
+ this.engineConnListService = engineConnListService;
+ }
+
+ @Override
+ @Receiver
+ public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest) {
+ EngineConn engineConn = getEngineConnByServiceInstance(engineStopRequest.getServiceInstance());
+ EngineStopResponse response = new EngineStopResponse();
+ if (null != engineConn) {
+ if(!killEngineConnByPid(engineConn)) {
+ response.setStopStatus(false);
+ response.setMsg("Kill engine " + engineConn.getServiceInstance().toString() + " failed.");
+ } else {
+ response.setStopStatus(true);
+ response.setMsg("Kill engine " + engineConn.getServiceInstance().toString() + " succeed.");
+ }
+ } else {
+ logger.warn("Cannot find engineconn : " + engineStopRequest.getServiceInstance().toString() + " in this engineConnManager engineConn list, cannot kill.");
+ response.setStopStatus(false);
+ response.setMsg("EngineConn " + engineStopRequest.getServiceInstance().toString() + " was not found in this engineConnManager.");
+ }
+ if (!response.getStopStatus()) {
+ EngineSuicideRequest request = new EngineSuicideRequest(engineStopRequest.getServiceInstance(), engineStopRequest.getUser());
+ try {
+ Sender.getSender(engineStopRequest.getServiceInstance()).send(request);
+ response.setStopStatus(true);
+ response.setMsg(response.getMsg() + " Now send suicide request to engine.");
+ } catch (Exception e) {
+ response.setMsg(response.getMsg() + " Sended suicide request to engine error, " + e.getMessage());
+ }
+ }
+ return response;
+ }
+
+ private EngineConn getEngineConnByServiceInstance(ServiceInstance serviceInstance) {
+ if (null == serviceInstance) {
+ return null;
+ }
+ List engineConnList = engineConnListService.getEngineConns();
+ for (EngineConn engineConn : engineConnList) {
+ if (engineConn.getServiceInstance().equals(serviceInstance)) {
+ return engineConn;
+ }
+ }
+ return null;
+ }
+
+ private boolean killEngineConnByPid(EngineConn engineConn) {
+ logger.info("try to kill {} toString with pid({}).", engineConn.getServiceInstance().toString(), engineConn.getPid());
+ if (StringUtils.isNotBlank(engineConn.getPid())) {
+ String k15cmd = "sudo kill " + engineConn.getPid();
+ String k9cmd = "sudo kill -9 " + engineConn.getPid();
+ int tryNum = 1;
+ try {
+ while (isProcessAlive(engineConn.getPid()) && tryNum <= 3) {
+ logger.info("{} still alive with pid({}), use shell command to kill it. try {}++", engineConn.getServiceInstance().toString(), engineConn.getPid(), tryNum++);
+ if (tryNum < 3) {
+ Utils.exec(k15cmd.split(" "), 3000L);
+ } else {
+ Utils.exec(k9cmd.split(" "), 3000L);
+ }
+ Thread.sleep(3000);
+ }
+ } catch (InterruptedException e) {
+ logger.error("Interrupted while killing engine {} with pid({})." + engineConn.getServiceInstance().toString(), engineConn.getPid());
+ }
+ if (isProcessAlive(engineConn.getPid())) {
+ return false;
+ } else {
+ return true;
+ }
+ } else {
+ logger.warn("cannot kill {} with empty pid.", engineConn.getServiceInstance().toString());
+ return false;
+ }
+ }
+
+ private boolean isProcessAlive(String pid) {
+ String findCmd = "\"ps -ef | grep " + pid + " | grep EngineConnServer | awk '{print \\\"exists_\\\"$2}' | grep " + pid + " \"";
+ List cmdList = new ArrayList<>();
+ cmdList.add("sudo");
+ cmdList.add("-S");
+ cmdList.add("bash");
+ cmdList.add("-c");
+ cmdList.add(findCmd);
+ try {
+ // todo
+ String rs = Utils.exec(cmdList.toArray(new String[0]), 5000L);
+ return null != rs && rs.contains("exists_" + pid);
+ } catch (Exception e) {
+ logger.error("Method isProcessAlive failed, " + e.getMessage());
+ }
+ return true;
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala
new file mode 100644
index 0000000000..04d20229c1
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+
+import com.google.common.collect.Interners
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.{EngineConn, YarnEngineConn}
+import com.webank.wedatasphere.linkis.ecm.core.launch.EngineConnLaunchRunner
+import com.webank.wedatasphere.linkis.ecm.core.listener.{ECMEvent, ECMEventListener}
+import com.webank.wedatasphere.linkis.ecm.server.ECMApplication
+import com.webank.wedatasphere.linkis.ecm.server.converter.ECMEngineConverter
+import com.webank.wedatasphere.linkis.ecm.server.listener._
+import com.webank.wedatasphere.linkis.ecm.server.service.EngineConnListService
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.{Resource, ResourceType}
+
+import scala.collection.JavaConversions._
+
+
+class DefaultEngineConnListService extends EngineConnListService with ECMEventListener with Logging {
+ /**
+ * key:tickedId,value :engineConn
+ */
+ private val engineConnMap = new ConcurrentHashMap[String, EngineConn]
+
+ val lock = Interners.newWeakInterner[String]
+
+ override def init(): Unit = {}
+
+ override def getEngineConn(engineConnId: String): Option[EngineConn] = Option(engineConnMap.get(engineConnId))
+
+ override def getEngineConns: util.List[EngineConn] = engineConnMap.values().toList
+
+ override def addEngineConn(engineConn: EngineConn): Unit = {
+ if (ECMApplication.isReady)
+ engineConnMap.put(engineConn.getTickedId, engineConn)
+ }
+
+ override def killEngineConn(engineConnId: String): Unit = {
+ val conn = engineConnMap.remove(engineConnId)
+ if (conn != null) {
+ Utils.tryAndWarn{
+ conn.close()
+ info(s"engineconn ${conn.getPid} was closed.")
+ }
+ }
+ }
+
+ override def getUsedResources: Resource = engineConnMap.values().map(_.getResource.getMinResource).fold(Resource.initResource(ResourceType.Default))(_ + _)
+
+ override def submit(runner: EngineConnLaunchRunner): Option[EngineConn] = ???
+
+ def updateYarnAppId(event: YarnAppIdCallbackEvent): Unit = {
+ updateYarnEngineConn(x => x.setApplicationId(event.protocol.applicationId), event.protocol.nodeId)
+ }
+
+ def updateYarnEngineConn(implicit updateFunction: YarnEngineConn => Unit, nodeId: String): Unit = {
+ lock.intern(nodeId) synchronized {
+ engineConnMap.get(nodeId) match {
+ case e: YarnEngineConn => updateFunction(e)
+ case e: EngineConn =>
+ engineConnMap.put(nodeId, ECMEngineConverter.engineConn2YarnEngineConn(e))
+ }
+ }
+ }
+
+ def updateEngineConn(updateFunction: EngineConn => Unit, nodeId: String): Unit = {
+ lock.intern(nodeId) synchronized {
+ engineConnMap.get(nodeId) match {
+ case e: EngineConn => updateFunction(e)
+ }
+ }
+ }
+
+ def updateYarnInfo(event: YarnInfoCallbackEvent): Unit = {
+ updateYarnEngineConn(x => x.setApplicationURL(event.protocol.uri), event.protocol.nodeId)
+ }
+
+ def updatePid(event: EngineConnPidCallbackEvent): Unit = {
+ updateEngineConn(x => {
+ x.setPid(event.protocol.pid)
+ x.setServiceInstance(event.protocol.serviceInstance)
+ }, event.protocol.ticketId)
+ }
+
+ def updateEngineConnStatus(tickedId: String, updateStatus: NodeStatus): Unit = {
+ updateEngineConn(x => x.setStatus(updateStatus), tickedId)
+ }
+
+ override def onEvent(event: ECMEvent): Unit = {
+ info(s"Deal event $event")
+ event match {
+ case event: ECMClosedEvent => shutdownEngineConns(event)
+ case event: YarnAppIdCallbackEvent => updateYarnAppId(event)
+ case event: YarnInfoCallbackEvent => updateYarnInfo(event)
+ case event: EngineConnPidCallbackEvent => updatePid(event)
+ case EngineConnAddEvent(engineConn) => addEngineConn(engineConn)
+ case EngineConnStatusChangeEvent(tickedId, updateStatus) => updateEngineConnStatus(tickedId, updateStatus)
+ case _ =>
+ }
+ }
+
+ private def shutdownEngineConns(event: ECMClosedEvent): Unit = {
+ engineConnMap.keys().foreach(killEngineConn)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala
new file mode 100644
index 0000000000..1f0a6b3002
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.ecm.server.ECMApplication
+import com.webank.wedatasphere.linkis.ecm.server.listener.EngineConnPidCallbackEvent
+import com.webank.wedatasphere.linkis.ecm.server.service.EngineConnPidCallbackService
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.ResponseEngineConnPid
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+
+
+class DefaultEngineConnPidCallbackService extends EngineConnPidCallbackService with Logging {
+
+ @Receiver
+ override def dealPid(protocol: ResponseEngineConnPid): Unit = {
+ //1.设置pid
+ //2.设置serviceInstance
+ //3.状态为running
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnPidCallbackEvent(protocol))
+ }
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala
new file mode 100644
index 0000000000..bea04016bd
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.ecm.server.ECMApplication
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SPRING_NAME
+import com.webank.wedatasphere.linkis.ecm.server.listener.EngineConnStatusChangeEvent
+import com.webank.wedatasphere.linkis.ecm.server.service.EngineConnStatusCallbackService
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus.{Failed, Running}
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.{EngineConnStatusCallback, EngineConnStatusCallbackToAM}
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.rpc.Sender
+import org.springframework.stereotype.Service
+
+
+@Service
+class DefaultEngineConnStatusCallbackService extends EngineConnStatusCallbackService with Logging {
+
+ @Receiver
+ override def dealEngineConnStatusCallback(protocol: EngineConnStatusCallback): Unit = {
+ info(s"Start to deal EngineConnStatusCallback $protocol")
+
+ if (NodeStatus.isAvailable(protocol.status)) {
+
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(protocol.ticketId, Running))
+ } else {
+
+ Sender.getSender(MANAGER_SPRING_NAME).send(EngineConnStatusCallbackToAM(protocol.serviceInstance,
+ protocol.status, protocol.initErrorMsg))
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(protocol.ticketId, Failed))
+ }
+
+ info(s"Finished to deal EngineConnStatusCallback $protocol")
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultLocalDirsHandleService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultLocalDirsHandleService.scala
new file mode 100644
index 0000000000..039a58f84e
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultLocalDirsHandleService.scala
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import java.io.File
+import java.nio.file.Paths
+
+import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration._
+import com.webank.wedatasphere.linkis.ecm.server.service.LocalDirsHandleService
+
+
+class DefaultLocalDirsHandleService extends LocalDirsHandleService {
+
+ // TODO: 检测当前磁盘的健康状态，如果目录满了，需要上报am
+
+ override def cleanup(): Unit = ???
+
+
+ override def getEngineConnManagerHomeDir: String = ECM_HOME_DIR
+
+ override def getEngineConnWorkDir(user: String, ticketId: String): String = new FsPath(Paths.get(ENGINECONN_ROOT_DIR, user, "workDir", ticketId).toFile.getPath).getPath
+
+ override def getEngineConnPublicDir: String = ENGINECONN_PUBLIC_DIR
+
+ override def getEngineConnLogDir(user: String, ticketId: String): String = s"${getEngineConnWorkDir(user, ticketId)}${File.separator}logs"
+
+ override def getEngineConnTmpDir(user: String, ticketId: String): String = s"${getEngineConnWorkDir(user, ticketId)}${File.separator}tmp"
+
+
+}
+
+
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala
new file mode 100644
index 0000000000..381a5276f6
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import com.webank.wedatasphere.linkis.ecm.server.ECMApplication
+import com.webank.wedatasphere.linkis.ecm.server.listener.{YarnAppIdCallbackEvent, YarnInfoCallbackEvent}
+import com.webank.wedatasphere.linkis.ecm.server.service.YarnCallbackService
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol}
+
+
+class DefaultYarnCallbackService extends YarnCallbackService {
+
+ @Receiver
+ override def dealApplicationId(protocol: YarnAPPIdCallbackProtocol): Unit = {
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(YarnAppIdCallbackEvent(protocol))
+ }
+
+ @Receiver
+ override def dealApplicationURI(protocol: YarnInfoCallbackProtocol): Unit = {
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(YarnInfoCallbackEvent(protocol))
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/LinuxProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/LinuxProcessEngineConnLaunchService.scala
new file mode 100644
index 0000000000..8dfcbedc9d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/LinuxProcessEngineConnLaunchService.scala
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import com.webank.wedatasphere.linkis.ecm.core.launch.{DiscoveryMsgGenerator, EngineConnLaunch, EurekaDiscoveryMsgGenerator}
+import com.webank.wedatasphere.linkis.ecm.linux.launch.LinuxProcessEngineConnLaunch
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration.{ENGINECONN_SPRING_NAME, _}
+import com.webank.wedatasphere.linkis.manager.common.entity.node.EngineNode
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.entity.{EngineConnBuildRequest, EngineConnLaunchRequest}
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.message.builder.ServiceMethodContext
+import com.webank.wedatasphere.linkis.message.conf.MessageSchedulerConf._
+import com.webank.wedatasphere.linkis.rpc.Sender
+
+import scala.concurrent.duration.Duration
+
+
+class LinuxProcessEngineConnLaunchService extends ProcessEngineConnLaunchService {
+
+ /**
+ * launchEngine启动一个引擎的方法
+ *
+ * @param engineConnBuildRequest 封装了引擎启动的参数
+ */
+ @Receiver
+ def launchEngineConn(engineConnBuildRequest: EngineConnBuildRequest, smc: ServiceMethodContext): EngineNode = {
+ Sender.getSender(ENGINECONN_SPRING_NAME).ask(engineConnBuildRequest) match {
+ case request: EngineConnLaunchRequest if ENGINECONN_CREATE_DURATION._1 != 0L =>
+ launchEngineConn(request, ENGINECONN_CREATE_DURATION._1)
+ case request: EngineConnLaunchRequest =>
+ launchEngineConn(request, smc.getAttribute(DURATION_KEY).asInstanceOf[Duration]._1)
+ }
+ }
+
+ override def launchEngineConn(engineConnBuildRequest: EngineConnBuildRequest): EngineNode = {
+ Sender.getSender(ENGINECONN_SPRING_NAME).ask(engineConnBuildRequest) match {
+ case request: EngineConnLaunchRequest =>
+ launchEngineConn(request, ENGINECONN_CREATE_DURATION._1)
+ }
+ }
+
+ def createDiscoveryMsgGenerator: DiscoveryMsgGenerator = new EurekaDiscoveryMsgGenerator
+
+ override def createEngineConnLaunch: EngineConnLaunch = {
+ val launch = new LinuxProcessEngineConnLaunch
+ launch.setDiscoveryMsgGenerator(createDiscoveryMsgGenerator)
+ launch
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
new file mode 100644
index 0000000000..852499e754
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.service.impl
+
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.linkis.common.utils.Utils
+import com.webank.wedatasphere.linkis.ecm.core.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.ecm.core.launch.ProcessEngineConnLaunch
+import com.webank.wedatasphere.linkis.ecm.server.ECMApplication
+import com.webank.wedatasphere.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SPRING_NAME
+import com.webank.wedatasphere.linkis.ecm.server.exception.ECMErrorException
+import com.webank.wedatasphere.linkis.ecm.server.listener.EngineConnStatusChangeEvent
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus._
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM
+import com.webank.wedatasphere.linkis.rpc.Sender
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang.exception.ExceptionUtils
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Future, TimeoutException}
+
+
+abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchService {
+
+ override def afterLaunch(conn: EngineConn, duration: Long): Unit = {
+ conn.getEngineConnLaunchRunner.getEngineConnLaunch match {
+ case launch: ProcessEngineConnLaunch => try {
+ processMonitorThread(conn, launch, duration)
+ } catch {
+ case e: ECMErrorException =>
+ warn("EngineConn init failed", e)
+ val logPath = conn.getEngineConnManagerEnv.engineConnWorkDir + "/logs"
+ Sender.getSender(MANAGER_SPRING_NAME).send(EngineConnStatusCallbackToAM(conn.getServiceInstance,
+ NodeStatus.ShuttingDown, "Failed to start EngineConn, reason: " + ExceptionUtils.getRootCauseMessage(e) + s"You can go to this path($logPath) to find the reason or ask the administrator for help"))
+ }
+ case _ =>
+ }
+ }
+
+ private def processMonitorThread(engineConn: EngineConn, launch: ProcessEngineConnLaunch, timeout: Long): Unit = {
+ val isCompleted: EngineConn => Boolean = engineConn => engineConn.getStatus == Success || engineConn.getStatus == Failed
+ val tickedId = engineConn.getTickedId
+ Future {
+ val iterator = IOUtils.lineIterator(launch.getProcessInputStream, "utf-8")
+ while (!isCompleted(engineConn) && iterator.hasNext) {
+ println(s"${engineConn.getTickedId}:${iterator.next()}")
+ }
+ val exitCode = Option(launch.processWaitFor)
+ if (exitCode.exists(_ != 0)) {
+ // TODO: 错误日志获取
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(tickedId, ShuttingDown))
+ } else {
+ ECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(tickedId, Success))
+ }
+ }
+ Utils.tryThrow(Utils.waitUntil(() => engineConn.getStatus != Starting, Duration(timeout, TimeUnit.MILLISECONDS))) {
+ case e: TimeoutException =>
+ throw new ECMErrorException(10000, s"wait for $engineConn initial timeout.")
+ case e: InterruptedException => //比如被ms cancel
+ throw new ECMErrorException(10000, s"wait for $engineConn initial interrupted.")
+ case t: Throwable =>
+ error(s"unexpected error, now shutdown it.")
+ throw t
+ }
+ if (engineConn.getStatus == ShuttingDown) {
+ throw new ECMErrorException(10000, s"Failed to init $engineConn, status shutting down")
+ }
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/spring/ECMSpringConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/spring/ECMSpringConfiguration.scala
new file mode 100644
index 0000000000..21e0b74ff5
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/spring/ECMSpringConfiguration.scala
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.spring
+
+import com.webank.wedatasphere.linkis.ecm.core.listener.ECMEventListener
+import com.webank.wedatasphere.linkis.ecm.server.context.{DefaultECMContext, ECMContext}
+import com.webank.wedatasphere.linkis.ecm.server.service._
+import com.webank.wedatasphere.linkis.ecm.server.service.impl._
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
+import org.springframework.context.annotation.{Bean, Configuration}
+
+
+@Configuration
+class ECMSpringConfiguration {
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultEngineConnManagerContext: ECMContext = {
+ new DefaultECMContext
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultYarnCallbackService: YarnCallbackService = {
+ new DefaultYarnCallbackService
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getBmlResourceLocalizationService(context: ECMContext, localDirsHandleService: LocalDirsHandleService): ResourceLocalizationService = {
+ val service: BmlResourceLocalizationService = new BmlResourceLocalizationService
+ service.setLocalDirsHandleService(localDirsHandleService)
+ service
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultLogCallbackService: LogCallbackService = {
+ null
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultlocalDirsHandleService: LocalDirsHandleService = {
+ new DefaultLocalDirsHandleService
+ }
+
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultEngineConnPidCallbackService: EngineConnPidCallbackService = {
+ new DefaultEngineConnPidCallbackService
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultEngineConnListService(context: ECMContext): EngineConnListService = {
+ implicit val service: DefaultEngineConnListService = new DefaultEngineConnListService
+ registerSyncListener(context)
+ service
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getLinuxProcessEngineConnLaunchService(resourceLocalizationService: ResourceLocalizationService): EngineConnLaunchService = {
+ val service = new LinuxProcessEngineConnLaunchService
+ service.setResourceLocalizationService(resourceLocalizationService)
+ service
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultECMRegisterService(context: ECMContext): ECMRegisterService = {
+ implicit val service: DefaultECMRegisterService = new DefaultECMRegisterService
+ registerSyncListener(context)
+ service
+ }
+
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultECMHealthService(context: ECMContext): ECMHealthService = {
+ implicit val service: DefaultECMHealthService = new DefaultECMHealthService
+ registerSyncListener(context)
+ service
+ }
+
+ @Bean
+ @ConditionalOnMissingBean
+ def getDefaultEngineConnKillService(engineConnListService: EngineConnListService): EngineConnKillService = {
+ val service = new DefaultEngineConnKillService
+ service.setEngineConnListService(engineConnListService)
+ service
+ }
+
+ private def registerSyncListener(context: ECMContext)(implicit listener: ECMEventListener): Unit = {
+ context.getECMSyncListenerBus.addListener(listener)
+ }
+
+ private def registerASyncListener(context: ECMContext)(implicit listener: ECMEventListener): Unit = {
+ context.getECMAsyncListenerBus.addListener(listener)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/util/ECMUtils.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/util/ECMUtils.scala
new file mode 100644
index 0000000000..0c41ac14bf
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/com/webank/wedatasphere/linkis/ecm/server/util/ECMUtils.scala
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ecm.server.util
+
+import java.io.{File, InputStream}
+import java.util
+
+import com.webank.wedatasphere.linkis.bml.client.{BmlClient, BmlClientFactory}
+import com.webank.wedatasphere.linkis.bml.protocol.BmlDownloadResponse
+import com.webank.wedatasphere.linkis.ecm.server.exception.ECMErrorException
+import com.webank.wedatasphere.linkis.manager.common.protocol.bml.BmlResource
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.storage.fs.FileSystem
+import org.apache.commons.io.{FileUtils, IOUtils}
+
+import scala.collection.JavaConversions._
+
+
+object ECMUtils {
+
+ private def download(resource: BmlResource, userName: String): util.Map[String, Object] = {
+ val client: BmlClient = createBMLClient(userName)
+ var response: BmlDownloadResponse = null
+ if (resource.getVersion == null) {
+ response = client.downloadShareResource(userName, resource.getResourceId)
+ } else {
+ response = client.downloadShareResource(userName, resource.getResourceId, resource.getVersion)
+ }
+ if (!response.isSuccess) throw new ECMErrorException(911115, "下载失败")
+ val map = new util.HashMap[String, Object]
+ map += "path" -> response.fullFilePath
+ map += "is" -> response.inputStream
+ }
+
+ def downLoadBmlResourceToLocal(resource: BmlResource, userName: String, path: String)(implicit fs: FileSystem): Unit = {
+ val is = download(resource, userName).get("is").asInstanceOf[InputStream]
+ val os = FileUtils.openOutputStream(new File(path + File.separator + resource.getFileName))
+ IOUtils.copy(is, os)
+ IOUtils.closeQuietly(os)
+ IOUtils.closeQuietly(is)
+ }
+
+ private def createBMLClient(userName: String): BmlClient = {
+ if (userName == null)
+ BmlClientFactory.createBmlClient()
+ else
+ BmlClientFactory.createBmlClient(userName)
+ }
+
+
+ private val address = Sender.getThisInstance.substring(0, Sender.getThisInstance.lastIndexOf(":"))
+
+ def getInstanceByPort(port: String): String = address + ":" + port
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/pom.xml b/linkis-computation-governance/linkis-engineconn-manager/pom.xml
new file mode 100644
index 0000000000..91420a63c0
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn-manager/pom.xml
@@ -0,0 +1,36 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+ 4.0.0
+
+ linkis-engineconn-manager
+ pom
+
+ linkis-engineconn-manager-core
+ linkis-engineconn-manager-server
+ linkis-engineconn-linux-launch
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/pom.xml
new file mode 100644
index 0000000000..444957637e
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/pom.xml
@@ -0,0 +1,128 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../pom.xml
+
+ 4.0.0
+
+ linkis-computation-engineconn
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-executor-core
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-accessible-executor
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-resource-executor
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-callback-service
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-storage
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-message-scheduler
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-cs-engine-support
+ ${linkis.version}
+
+
+ reflections
+ org.reflections
+
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-computation-governance-common
+ ${linkis.version}
+
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ ${basedir}/src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala
new file mode 100644
index 0000000000..cc4661aaf7
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.conf
+
+import com.webank.wedatasphere.linkis.common.conf.{ByteType, CommonVars}
+
+
+object ComputationExecutorConf {
+
+ val ENGINECONN_RESULT_SET_STORE_PATH = CommonVars("wds.linkis.engineconn.resultSet.default.store.path", "hdfs:///tmp")
+
+ val ENGINE_RESULT_SET_MAX_CACHE = CommonVars("wds.linkis.engine.resultSet.cache.max", new ByteType("0k"))
+
+ val ENGINE_DEFAULT_LIMIT = CommonVars("wds.linkis.engine.default.limit", 5000)
+
+ val ENGINE_LOCK_DEFAULT_EXPIRE_TIME = CommonVars("wds.linkis.engine.lock.expire.time", 2 * 60 * 1000)
+
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSEnginePreExecuteHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSEnginePreExecuteHook.scala
new file mode 100644
index 0000000000..dc07740049
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSEnginePreExecuteHook.scala
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.cs
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.EngineExecutionContext
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.hook.ComputationExecutorHook
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest
+
+
+class CSEnginePreExecuteHook extends ComputationExecutorHook with Logging {
+
+ private val csResourceParser: CSResourceParser = new CSResourceParser
+
+ override def getHookName: String = "ContextServicePreHook"
+
+ override def beforeExecutorExecute(engineExecutionContext: EngineExecutionContext, engineCreationContext: EngineCreationContext, code: String): String = {
+ val props = engineExecutionContext.getProperties
+ if (null != props && props.containsKey(CSCommonUtils.CONTEXT_ID_STR)) {
+ var parsedCode = code
+ val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap (props)
+ val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap (props)
+ engineExecutionContext.addProperty (CSCommonUtils.CONTEXT_ID_STR, contextIDValueStr)
+ engineExecutionContext.addProperty (CSCommonUtils.NODE_NAME_STR, nodeNameStr)
+ info (s"Start to call cs engine pre hook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr")
+ parsedCode = csResourceParser.parse (props, parsedCode, contextIDValueStr, nodeNameStr)
+ info (s"Finished to call cs engine pre hook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr")
+ parsedCode
+ } else {
+ code
+ }
+ }
+
+
+}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSResourceParser.scala
similarity index 86%
rename from ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala
rename to linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSResourceParser.scala
index df09dfc170..fb70c01b93 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSResourceParser.scala
@@ -11,23 +11,18 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.engine.cs
-
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.cs
import java.util
import java.util.regex.Pattern
import com.webank.wedatasphere.linkis.cs.client.service.CSResourceService
-import com.webank.wedatasphere.linkis.engine.PropertiesExecuteRequest
import org.apache.commons.lang.StringUtils
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
-/**
- * @author peacewong
- * @date 2020/3/5 19:42
- */
+
class CSResourceParser {
private val pb = Pattern.compile("cs://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE)
@@ -41,7 +36,7 @@ class CSResourceParser {
bmlResourceNames.toArray
}
- def parse(executeRequest: PropertiesExecuteRequest, code: String, contextIDValueStr: String, nodeNameStr: String): String = {
+ def parse(props: util.Map[String, Object], code: String, contextIDValueStr: String, nodeNameStr: String): String = {
//TODO getBMLResource peaceWong
val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr)
@@ -66,7 +61,7 @@ class CSResourceParser {
}
}
- executeRequest.properties.put("resources", parsedResources)
+ props.put("resources", parsedResources)
StringUtils.replaceEach(code, preFixNames.toArray, parsedNames.toArray)
}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSTableRegister.scala
similarity index 92%
rename from ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala
rename to linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSTableRegister.scala
index e6c4dadc87..e2ff4593c6 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/cs/CSTableRegister.scala
@@ -11,10 +11,9 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.engine.cs
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.cs
import java.util.Date
-
import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetWriter
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.Logging
@@ -24,22 +23,19 @@ import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.{ContextScope
import com.webank.wedatasphere.linkis.cs.common.entity.metadata.{CSColumn, CSTable}
import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey
import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
-import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.EngineExecutionContext
import com.webank.wedatasphere.linkis.storage.domain.Column
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
import org.apache.commons.lang.StringUtils
-/**
- * @author peacewong
- * @date 2020/3/13 16:29
- */
+
object CSTableRegister extends Logging{
- def registerTempTable(engineExecutorContext: EngineExecutorContext,
+ def registerTempTable(engineExecutionContext: EngineExecutionContext,
writer: ResultSetWriter[_ <: MetaData, _ <: Record], alias: String, columns: Array[Column]): Unit = {
- val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties)
- val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties)
+ val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutionContext.getProperties)
+ val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutionContext.getProperties)
if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) {
info(s"Start to register TempTable nodeName:$nodeNameStr")
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/CommonEngineConnTask.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/CommonEngineConnTask.scala
new file mode 100644
index 0000000000..ef1e0cf3dc
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/CommonEngineConnTask.scala
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.entity
+import java.util
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+
+class CommonEngineConnTask(val taskId: String, var taskSupportRetry: Boolean = false) extends EngineConnTask {
+
+ private var code: String = _
+ private var prop: util.Map[String, Object] = _
+ private var status: ExecutionNodeStatus = ExecutionNodeStatus.Inited
+ private var labels: Array[Label[_]] = new Array[Label[_]](0)
+ private var serviceInstance: ServiceInstance = _
+
+ override def getTaskId: String = taskId
+
+ override def isTaskSupportRetry: Boolean = taskSupportRetry
+
+ override def getCode: String = code
+
+ override def setCode(code: String): Unit = this.code = code
+
+ override def getProperties: util.Map[String, Object] = prop
+
+ override def setProperties(properties: util.Map[String, Object]): Unit = this.prop = properties
+
+ override def data(key: String, value: Object): Unit = this.prop.put(key, value)
+
+ override def getStatus: ExecutionNodeStatus = status
+
+ override def setStatus(status: ExecutionNodeStatus): Unit = this.status = status
+
+ override def getLables: Array[Label[_]] = labels
+
+ override def setLabels(labels: Array[Label[_]]): Unit = this.labels = labels
+
+ override def getCallbackServiceInstance(): ServiceInstance = serviceInstance
+
+ override def setCallbackServiceInstance(serviceInstance: ServiceInstance): Unit = this.serviceInstance = serviceInstance
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/EngineConnTask.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/EngineConnTask.scala
new file mode 100644
index 0000000000..3f5e957789
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/EngineConnTask.scala
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.entity
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+
+trait EngineConnTask {
+
+ def getTaskId: String
+
+ def isTaskSupportRetry: Boolean
+
+ def getCode: String
+
+ def setCode(code: String): Unit
+
+ def getProperties: util.Map[String, Object]
+
+ def setProperties(properties: util.Map[String, Object])
+
+ def data(key: String, value: Object): Unit
+
+ def getStatus: ExecutionNodeStatus
+
+ def setStatus(taskStatus: ExecutionNodeStatus): Unit
+
+
+ def getLables: Array[Label[_]]
+
+ def setLabels(labels: Array[Label[_]]): Unit
+
+ def getCallbackServiceInstance(): ServiceInstance
+
+ def setCallbackServiceInstance(serviceInstance: ServiceInstance): Unit
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/TaskStatus.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/TaskStatus.scala
new file mode 100644
index 0000000000..dd150e5fe0
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/entity/TaskStatus.scala
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.entity
+
+
+object TaskStatus extends Enumeration {
+
+ type TaskStatus = Value
+ val PENDING = Value(0)
+ val RUNNING = Value(1)
+ val SUCCESS = Value(2)
+ val ERROR = Value(3)
+ val CANCEL = Value(4)
+
+ def isCompleted(status: TaskStatus): Boolean = {
+ if (status == PENDING || status == RUNNING) false
+ else true
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ComputationEngineConnExecution.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ComputationEngineConnExecution.scala
new file mode 100644
index 0000000000..32b23d104c
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ComputationEngineConnExecution.scala
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.execute
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.common.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.engineconn.common.execution.EngineExecution
+import com.webank.wedatasphere.linkis.engineconn.core.executor.ExecutorManager
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.{Executor, ResourceExecutor}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.engineconn.executor.service.ManagerService
+import com.webank.wedatasphere.linkis.governance.common.exception.engineconn.{EngineConnExecutorErrorCode, EngineConnExecutorErrorException}
+import com.webank.wedatasphere.linkis.manager.common.protocol.resource.ResourceUsedProtocol
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
+
+
+class ComputationEngineConnExecution extends EngineExecution with Logging {
+
+ private val executorManager: ExecutorManager = ExecutorManager.getInstance()
+
+ /**
+ * 创建出默认的Executor
+ *
+ * @param engineCreationContext
+ * @param engineConn
+ */
+ override def execute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {
+ Utils.tryCatch{
+ init(engineCreationContext)
+ val defaultExecutor = executorManager.getDefaultExecutor.asInstanceOf[ComputationExecutor]
+ reportUsedResource(defaultExecutor, engineCreationContext)
+ reportLabel(defaultExecutor.getExecutorLabels())
+ info(s"Engine Executor ${defaultExecutor.getId()} started.")
+ info("Debug: engineCreationContext: " + BDPJettyServerHelper.gson.toJson(engineCreationContext))
+ }{
+ t =>
+ error(s"Init executor error. ", t)
+ throw new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INIT_EXECUTOR_FAILED, "Init executor failed. ", t)
+ }
+ }
+
+ private def init(engineCreationContext: EngineCreationContext): Unit = {
+ // msContext.getPublisher.publish(RequestTaskExecute)
+ val listenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext
+ listenerBusContext.getEngineConnAsyncListenerBus.start()
+ }
+
+ private def reportUsedResource(executor: Executor, engineCreationContext: EngineCreationContext): Unit = {
+ Utils.tryCatch(
+ executor match {
+ case resourceExecutor: ResourceExecutor =>
+ ManagerService.getManagerService.reportUsedResource(ResourceUsedProtocol(Sender.getThisServiceInstance, resourceExecutor.getCurrentNodeResource(), engineCreationContext.getTicketId))
+ case _ =>
+ info("do not to report usedResource")
+ }
+ ) {
+ t => error("ReportUsedResource error. cause: " + t.getCause)
+ }
+ }
+
+ private def reportLabel(labels: util.List[Label[_]]): Unit = {
+ ManagerService.getManagerService.labelReport(labels)
+ info("Reported label to am.")
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala
new file mode 100644
index 0000000000..c10fcf5f7b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.execute
+
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.{Cache, CacheBuilder}
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.entity.AccessibleExecutor
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.TaskStatusChangedEvent
+import com.webank.wedatasphere.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant}
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.entity.EngineConnTask
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.hook.ComputationExecutorHook
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.parser.CodeParser
+import com.webank.wedatasphere.linkis.engineconn.core.engineconn.EngineConnManager
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.{LabelExecutor, ResourceExecutor}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.{EngineConcurrentInfo, RequestTask}
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.creation.ExecutorFactory
+import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
+import com.webank.wedatasphere.linkis.scheduler.executer._
+import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang.exception.ExceptionUtils
+
+
+abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) extends AccessibleExecutor with ResourceExecutor with LabelExecutor with Logging {
+
+ private val listenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext
+
+ // private val taskMap: util.Map[String, EngineConnTask] = new ConcurrentHashMap[String, EngineConnTask](8)
+ private val taskCache: Cache[String, EngineConnTask] = CacheBuilder.newBuilder().expireAfterAccess(EngineConnConf.ENGINE_TASK_EXPIRE_TIME.getValue, TimeUnit.MILLISECONDS)
+ .maximumSize(EngineConnConstant.MAX_TASK_NUM).build()
+
+ private var engineInitialized: Boolean = false
+
+ private var codeParser: Option[CodeParser] = None
+
+ private var runningTasks: Int = 0
+
+ private var pendingTasks: Int = 0
+
+ private var succeedTasks: Int = 0
+
+ private var failedTasks: Int = 0
+
+ private var lastTask: EngineConnTask = null
+
+ final override def tryReady: Boolean = {
+ transition(NodeStatus.Unlock)
+ if (!engineInitialized) {
+ engineInitialized = true
+ }
+ true
+ }
+
+
+ override def init: Unit = {
+ info(s"executor($getId()) is ready ")
+ tryReady
+ }
+
+ def tryShutdown(): Boolean = {
+ this.ensureAvailable(transition(NodeStatus.ShuttingDown))
+ close()
+ true
+ }
+
+ def tryFailed(): Boolean = {
+ this.whenStatus(NodeStatus.ShuttingDown, transition(NodeStatus.Failed))
+ true
+ }
+
+
+ def getSucceedNum = succeedTasks
+
+ def getFailedNum = failedTasks
+
+ def getRunningTask: Int = runningTasks
+
+ private def getExecutorConcurrentInfo: EngineConcurrentInfo = EngineConcurrentInfo(runningTasks, pendingTasks, succeedTasks, failedTasks)
+
+ def isEngineInitialized = engineInitialized
+
+ protected def callback(): Unit = {}
+
+ override def close(): Unit = {
+ if (null != lastTask) synchronized {
+ killTask(lastTask.getTaskId)
+ } else {
+ killTask("By close")
+ }
+ super.close()
+ }
+
+ // override def getName: String = ComputationExecutorConf.DEFAULT_COMPUTATION_NAME
+
+ protected def ensureOp[A](f: => A): A = if (!this.engineInitialized)
+ f
+ else ensureIdle(f)
+
+ def execute(engineConnTask: EngineConnTask): ExecuteResponse = {
+
+ runningTasks += 1
+
+ taskCache.put(engineConnTask.getTaskId, engineConnTask)
+ lastTask = engineConnTask
+
+ transformTaskStatus(engineConnTask, ExecutionNodeStatus.Running)
+
+ ensureOp {
+ val engineExecutionContext = createEngineExecutionContext(engineConnTask)
+ var hookedCode = engineConnTask.getCode
+ Utils.tryCatch {
+ val engineCreationContext = EngineConnManager.getEngineConnManager.getEngineConn().getEngineCreationContext
+ ComputationExecutorHook.getComputationExecutorHooks.foreach(hook => {
+ hookedCode = hook.beforeExecutorExecute(engineExecutionContext, engineCreationContext, hookedCode)
+ })
+ } {
+ case e: Throwable => logger.info("failed to do with hook", e)
+ }
+ if (hookedCode.length > 100) {
+ info(s"hooked after code: ${hookedCode.substring(0, 100)} ....")
+ } else {
+ info(s"hooked after code: $hookedCode ")
+ }
+ val localPath = System.getenv(EngineConnConf.ENGINE_CONN_LOCAL_LOG_DIRS_KEY.getValue)
+ engineExecutionContext.appendStdout(s"EngineConn local log path : ${DataWorkCloudApplication.getServiceInstance.toString} ${localPath}")
+ var response: ExecuteResponse = null
+ val incomplete = new StringBuilder
+ val codes = Utils.tryCatch(getCodeParser().map(_.parse(hookedCode, engineExecutionContext)).getOrElse(Array(hookedCode))) { e =>
+ info("Your code will be submitted in overall mode")
+ Array(hookedCode)
+ }
+ engineExecutionContext.setTotalParagraph(codes.length)
+ codes.indices.foreach({
+ index =>
+ if (ExecutionNodeStatus.Cancelled == engineConnTask.getStatus) return ErrorExecuteResponse("Job is killed by user!", null)
+ val code = codes(index)
+ engineExecutionContext.setCurrentParagraph(index + 1)
+ response = Utils.tryCatch(if (incomplete.nonEmpty) executeCompletely(engineExecutionContext, code, incomplete.toString())
+ else executeLine(engineExecutionContext, code)
+ ) {
+ t => ErrorExecuteResponse(ExceptionUtils.getRootCauseMessage(t), t)
+ }
+ info(s"Finished to execute task ${engineConnTask.getTaskId}")
+ incomplete ++= code
+ response match {
+ case e: ErrorExecuteResponse =>
+ failedTasks += 1
+ error("execute code failed!", e.t)
+ return response
+ case SuccessExecuteResponse() =>
+ engineExecutionContext.appendStdout("\n")
+ incomplete.setLength(0)
+ case e: OutputExecuteResponse =>
+ incomplete.setLength(0)
+ val output = if (StringUtils.isNotEmpty(e.getOutput) && e.getOutput.length > outputPrintLimit)
+ e.getOutput.substring(0, outputPrintLimit) else e.getOutput
+ engineExecutionContext.appendStdout(output)
+ if (StringUtils.isNotBlank(e.getOutput)) engineExecutionContext.sendResultSet(e)
+ case _: IncompleteExecuteResponse =>
+ incomplete ++= incompleteSplitter
+ }
+ })
+ Utils.tryCatch(engineExecutionContext.close()) {
+ t =>
+ response = ErrorExecuteResponse("send resultSet to entrance failed!", t)
+ failedTasks += 1
+ }
+ runningTasks -= 1
+ lastTask = null
+ response match {
+ case _: OutputExecuteResponse =>
+ succeedTasks += 1
+ transformTaskStatus(engineConnTask, ExecutionNodeStatus.Succeed)
+ SuccessExecuteResponse()
+ case s: SuccessExecuteResponse =>
+ succeedTasks += 1
+ transformTaskStatus(engineConnTask, ExecutionNodeStatus.Succeed)
+ s
+ case _ => response
+ }
+ }
+
+
+ }
+
+ def setCodeParser(codeParser: CodeParser) = this.codeParser = Some(codeParser)
+
+ def getCodeParser(): Option[CodeParser] = this.codeParser
+
+ def executeLine(engineExecutorContext: EngineExecutionContext, code: String): ExecuteResponse
+
+ protected def incompleteSplitter = "\n"
+
+ def executeCompletely(engineExecutorContext: EngineExecutionContext, code: String, completedLine: String): ExecuteResponse
+
+ def progress(): Float
+
+ def getProgressInfo: Array[JobProgressInfo]
+
+ protected def createEngineExecutionContext(engineConnTask: EngineConnTask): EngineExecutionContext = {
+ val userCreator = ExecutorFactory.parseUserWithCreator(engineConnTask.getLables)
+
+ val engineExecutionContext = if (null != userCreator && StringUtils.isNotBlank(userCreator.user)) {
+ new EngineExecutionContext(this, userCreator.user)
+ } else {
+ new EngineExecutionContext(this)
+ }
+ if (engineConnTask.getProperties.containsKey(RequestTask.RESULT_SET_STORE_PATH)) {
+ engineExecutionContext.setStorePath(engineConnTask.getProperties.get(RequestTask.RESULT_SET_STORE_PATH).toString)
+ }
+ engineExecutionContext.setJobId(engineConnTask.getTaskId)
+ engineExecutionContext.getProperties.putAll(engineConnTask.getProperties)
+ engineExecutionContext.setLabels(engineConnTask.getLables)
+ engineExecutionContext
+ }
+
+ def killTask(taskId: String): Unit = {
+ Utils.tryAndWarn {
+ val task = taskCache.getIfPresent(taskId)
+ task.setStatus(ExecutionNodeStatus.Cancelled)
+ if (null != task) {
+ transformTaskStatus(task, ExecutionNodeStatus.Cancelled)
+ }
+ }
+ }
+
+ def transformTaskStatus(task: EngineConnTask, newStatus: ExecutionNodeStatus) = {
+ val oriStatus = task.getStatus
+ oriStatus match {
+ case ExecutionNodeStatus.Scheduled =>
+ if (task.getStatus != newStatus) {
+ task.setStatus(newStatus)
+ }
+ case ExecutionNodeStatus.Running =>
+ if (newStatus == ExecutionNodeStatus.Succeed || newStatus == ExecutionNodeStatus.Failed || newStatus == ExecutionNodeStatus.Cancelled) {
+ task.setStatus(newStatus)
+ } else {
+ error(s"Task status change error. task: ${task}, newStatus : ${newStatus}")
+ }
+ case _ =>
+ error(s"Task status change error. task: ${task}, newStatus : ${newStatus}")
+ }
+ if (oriStatus != newStatus) {
+ listenerBusContext.getEngineConnSyncListenerBus.postToAll(TaskStatusChangedEvent(task.getTaskId, oriStatus, newStatus))
+ }
+ }
+
+ def getTaskById(taskId: String): EngineConnTask = {
+ taskCache.getIfPresent(taskId)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala
new file mode 100644
index 0000000000..c8bfe2c597
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.execute
+
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.entity.EngineConnTask
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.ConcurrentExecutor
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
+
+
+abstract class ConcurrentComputationExecutor(override val outputPrintLimit: Int = 1000) extends ComputationExecutor(outputPrintLimit) with ConcurrentExecutor {
+
+ override def execute(engineConnTask: EngineConnTask): ExecuteResponse = {
+ if (getRunningTask >= getConcurrentLimit) synchronized {
+ if (getRunningTask >= getConcurrentLimit) {
+ info(s"running task($getRunningTask) > concurrent limit $getConcurrentLimit, now to mark engine to busy ")
+ transition(NodeStatus.Busy)
+ }
+ }
+ val response = super.execute(engineConnTask)
+ if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) synchronized {
+ if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) {
+ info(s"running task($getRunningTask) < concurrent limit $getConcurrentLimit, now to mark engine to Unlock ")
+ transition(NodeStatus.Unlock)
+ }
+ }
+ response
+ }
+
+ protected override def ensureOp[A](f: => A): A = if (!isEngineInitialized)
+ f
+ else ensureIdle(f, false)
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala
new file mode 100644
index 0000000000..5e268f682b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.execute
+
+import java.io.File
+import java.util
+import java.util.concurrent.atomic.AtomicInteger
+
+import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSet, ResultSetWriter}
+import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils
+import com.webank.wedatasphere.linkis.cs.storage.CSTableResultSetWriter
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.{TaskLogUpdateEvent, TaskProgressUpdateEvent, TaskResultCreateEvent, TaskResultSizeCreatedEvent}
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.conf.ComputationExecutorConf
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.{EngineConnAsyncListenerBus, EngineConnSyncListenerBus, ExecutorListenerBusContext}
+import com.webank.wedatasphere.linkis.governance.common.exception.engineconn.EngineConnExecutorErrorException
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
+import com.webank.wedatasphere.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse}
+import com.webank.wedatasphere.linkis.storage.resultset.table.TableResultSet
+import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter}
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang.time.DateFormatUtils
+
+import scala.collection.mutable.ArrayBuffer
+
+
+class EngineExecutionContext(executor: ComputationExecutor, executorUser: String = Utils.getJvmUser) extends Logging {
+
+ private val resultSetFactory = ResultSetFactory.getInstance
+
+ private val resultSetWriters = ArrayBuffer[ResultSetWriter[_ <: MetaData, _ <: Record]]()
+
+ private var defaultResultSetWriter: ResultSetWriter[_ <: MetaData, _ <: Record] = _
+
+ private var resultSize = 0
+
+ private var interrupted = false
+
+ private var jobId: Option[String] = None
+ private val aliasNum = new AtomicInteger(0)
+ protected var storePath: Option[String] = None
+
+ private val properties: java.util.Map[String, Object] = new util.HashMap[String, Object]()
+
+ private var labels: Array[Label[_]] = new Array[Label[_]](0)
+
+ private var totalParagraph = 0
+ private var currentParagraph = 0
+
+ def kill(): Unit = interrupted = true
+
+ def isKilled: Boolean = interrupted
+
+ def getTotalParagraph: Int = totalParagraph
+
+ def setTotalParagraph(totalParagraph: Int): Unit = this.totalParagraph = totalParagraph
+
+ def getCurrentParagraph: Int = currentParagraph
+
+ def setCurrentParagraph(currentParagraph: Int): Unit = this.currentParagraph = currentParagraph
+
+ def pushProgress(progress: Float, progressInfo: Array[JobProgressInfo]): Unit = {
+ val listenerBus = getEngineSyncListenerBus
+ jobId.foreach(jId => {
+ listenerBus.postToAll(TaskProgressUpdateEvent(jId, progress, progressInfo))
+ })
+ }
+
+ def sendResultSet(resultSetWriter: ResultSetWriter[_ <: MetaData, _ <: Record]): Unit = {
+ info("Start to send res to entrance")
+ val fileName = new File(resultSetWriter.toFSPath.getPath).getName
+ val index = if (fileName.indexOf(".") < 0) fileName.length else fileName.indexOf(".")
+ val alias = if (fileName.startsWith("_")) fileName.substring(1, index) else fileName.substring(0, fileName.indexOf("_"))
+ Utils.tryFinally(sendResultSet(resultSetWriter.toString(), alias)) {
+ IOUtils.closeQuietly(resultSetWriter)
+ resultSetWriters synchronized resultSetWriters -= resultSetWriter
+ }
+ }
+
+ def sendResultSet(output: String): Unit = sendResultSet(output, "_" + aliasNum.getAndIncrement())
+
+ def appendTextResultSet(output: String): Unit = {
+ if (defaultResultSetWriter == null) aliasNum synchronized {
+ if (defaultResultSetWriter == null) {
+ defaultResultSetWriter = createDefaultResultSetWriter(ResultSetFactory.TEXT_TYPE)
+ defaultResultSetWriter.addMetaData(new LineMetaData())
+ resultSetWriters += defaultResultSetWriter
+ }
+ }
+ defaultResultSetWriter.addRecord(new LineRecord(output))
+ }
+
+ private def sendResultSet(output: String, alias: String): Unit = {
+ if (StringUtils.isEmpty(output)) return
+ if (resultSetFactory.isResultSetPath(output) || resultSetFactory.isResultSet(output)) {
+ val listenerBus = getEngineSyncListenerBus
+ jobId.foreach(jId => {
+ //TODO peacewong executor.getEngineServerContext().getEngineAsyncListenerBus().post(ResultSetCreatedEvent(jId, output, alias))
+ listenerBus.postToAll(TaskResultCreateEvent(jId, output, alias))
+ resultSize += 1
+ })
+ } else throw new EngineConnExecutorErrorException(50050, "unknown resultSet: " + output)
+ }
+
+ def setJobId(jobId: String) = this.jobId = Option(jobId)
+
+ def getJobId = jobId
+
+ def setStorePath(storePath: String) = this.storePath = Option(storePath)
+
+ def getLables: Array[Label[_]] = labels
+
+ def setLabels(labels: Array[Label[_]]): Unit = this.labels = labels
+
+ def sendResultSet(outputExecuteResponse: OutputExecuteResponse): Unit = outputExecuteResponse match {
+ case AliasOutputExecuteResponse(alias, output) => sendResultSet(output, alias)
+ case output: OutputExecuteResponse => sendResultSet(output.getOutput, "_" + aliasNum.getAndIncrement())
+ }
+
+ def getProperties: java.util.Map[String, Object] = properties
+
+ def addProperty(key: String, value: String): Unit = properties.put(key, value)
+
+ protected def getDefaultStorePath: String = {
+ val path = ComputationExecutorConf.ENGINECONN_RESULT_SET_STORE_PATH.getValue
+ (if (path.endsWith("/")) path else path + "/") + Utils.getJvmUser + "/" +
+ DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd") + "/" + getJobId.get + "/" + System.nanoTime
+ }
+
+ def createDefaultResultSetWriter(): ResultSetWriter[_ <: MetaData, _ <: Record] = {
+ // createResultSetWriter(resultSetFactory.getResultSetByType(engine.getDefaultResultSetType))
+ createResultSetWriter(resultSetFactory.getResultSetByType(resultSetFactory.getResultSetType(0))) // todo check
+ }
+
+ def createDefaultResultSetWriter(alias: String): ResultSetWriter[_ <: MetaData, _ <: Record] =
+ createResultSetWriter(resultSetFactory.getResultSetByType(resultSetFactory.getResultSetType(0)), alias) // todo check
+
+ def createResultSetWriter(resultSetType: String): ResultSetWriter[_ <: MetaData, _ <: Record] =
+ createResultSetWriter(resultSetFactory.getResultSetByType(resultSetType), null)
+
+ def createResultSetWriter(resultSet: ResultSet[_ <: MetaData, _ <: Record]): ResultSetWriter[_ <: MetaData, _ <: Record] =
+ createResultSetWriter(resultSet, null)
+
+ def createResultSetWriter(resultSetType: String, alias: String): ResultSetWriter[_ <: MetaData, _ <: Record] =
+ createResultSetWriter(resultSetFactory.getResultSetByType(resultSetType), alias)
+
+ def createResultSetWriter(resultSet: ResultSet[_ <: MetaData, _ <: Record], alias: String): ResultSetWriter[_ <: MetaData, _ <: Record] = {
+ val filePath = storePath.getOrElse(getDefaultStorePath)
+ val fileName = if (StringUtils.isEmpty(alias)) "_" + aliasNum.getAndIncrement() else alias + "_" + aliasNum.getAndIncrement()
+ val resultSetPath = resultSet.getResultSetPath(new FsPath(filePath), fileName)
+ //update by peaceWong 20200402
+ val resultSetWriter = resultSet match {
+ case result: TableResultSet =>
+ val contextIDStr = ContextServiceUtils.getContextIDStrByMap(getProperties)
+ val nodeName = ContextServiceUtils.getNodeNameStrByMap(getProperties)
+ if (StringUtils.isNotBlank(contextIDStr) && StringUtils.isNotBlank(nodeName)) {
+ val csWriter = new CSTableResultSetWriter(result, ComputationExecutorConf.ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, contextIDStr, nodeName, alias)
+ csWriter.setProxyUser(executorUser)
+ csWriter
+ } else {
+ ResultSetWriter.getResultSetWriter(resultSet, ComputationExecutorConf.ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, executorUser)
+ }
+ case _ => ResultSetWriter.getResultSetWriter(resultSet, ComputationExecutorConf.ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, executorUser)
+ }
+ //update by peaceWong 20200402 end
+ resultSetWriters synchronized resultSetWriters += resultSetWriter
+ resultSetWriter
+ }
+
+ def appendStdout(log: String): Unit = if (!executor.isEngineInitialized) {
+ executor.info(log)
+ } else {
+ val listenerBus = getEngineSyncListenerBus
+ // jobId.foreach(jId => listenerBus.post(TaskLogUpdateEvent(jId, log)))
+ jobId.foreach(jId => listenerBus.postToAll(TaskLogUpdateEvent(jId, log)))
+ }
+
+ def sendProgress(progress: Float, progressInfos: Array[JobProgressInfo]): Unit = {
+ if (executor.isEngineInitialized) {
+ val listenerBus = getEngineSyncListenerBus
+ // jobId.foreach(jId => listenerBus.post(TaskProgressUpdateEvent(jId, progress, progressInfos)))
+ jobId.foreach(jId => listenerBus.postToAll(TaskProgressUpdateEvent(jId, progress, progressInfos)))
+ }
+ }
+
+ def close(): Unit = {
+ resultSetWriters.toArray.foreach(sendResultSet)
+ val listenerBus = getEngineSyncListenerBus
+ jobId.foreach(jId => {
+ listenerBus.postToAll(TaskResultSizeCreatedEvent(jId, resultSize))
+ })
+ resultSetWriters.clear()
+ }
+
+ private def getEngineAsyncListenerBus: EngineConnAsyncListenerBus = {
+ ExecutorListenerBusContext.getExecutorListenerBusContext.getEngineConnAsyncListenerBus
+ }
+
+ private def getEngineSyncListenerBus: EngineConnSyncListenerBus = {
+ ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnSyncListenerBus
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala
new file mode 100644
index 0000000000..6c594899e6
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.hook
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.callback.service.{EngineConnAfterStartCallback, EngineConnPidCallback}
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.common.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.engineconn.common.hook.EngineConnHook
+import com.webank.wedatasphere.linkis.engineconn.core.hook.ShutdownHook
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineConnStatusCallback
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
+import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang.exception.ExceptionUtils
+
+
+class ComputationEngineConnHook extends EngineConnHook with Logging {
+
+ override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = {
+ info("<----------Start Spring app---------->")
+ val parser = DWCArgumentsParser.parse(engineCreationContext.getArgs)
+ DWCArgumentsParser.setDWCOptionMap(parser.getDWCConfMap)
+ val existsExcludePackages = ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.getValue
+ if (!StringUtils.isEmpty(existsExcludePackages))
+ DataWorkCloudApplication.setProperty(ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.key, existsExcludePackages)
+ // 加载spring类
+ DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(parser.getSpringConfMap))
+
+ val engineConnPidCallBack = new EngineConnPidCallback(engineCreationContext.getEMInstance)
+ Utils.tryAndError(engineConnPidCallBack.callback())
+ info("<----------Spring app init success---------->")
+ }
+
+
+ override def beforeExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {}
+
+ override def afterExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {}
+
+ override def afterEngineServerStartFailed(engineCreationContext: EngineCreationContext, throwable: Throwable): Unit = {
+
+ val engineConnAfterStartCallback = new EngineConnAfterStartCallback(engineCreationContext.getEMInstance)
+ engineConnAfterStartCallback.callback(EngineConnStatusCallback(Sender.getThisServiceInstance,
+ engineCreationContext.getTicketId, NodeStatus.ShuttingDown, ExceptionUtils.getRootCauseMessage(throwable)))
+ error("init engineConn failed! now exit", throwable)
+ ShutdownHook.getShutdownHook.notifyError(throwable)
+ }
+
+ override def afterEngineServerStartSuccess(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {
+ val engineConnAfterStartCallback = new EngineConnAfterStartCallback(engineCreationContext.getEMInstance)
+ engineConnAfterStartCallback.callback(EngineConnStatusCallback(Sender.getThisServiceInstance,
+ engineCreationContext.getTicketId, NodeStatus.Unlock, "success"))
+ warn("init engineConn success!")
+ }
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala
new file mode 100644
index 0000000000..90a47ceffd
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.hook
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.EngineExecutionContext
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.creation.ExecutorFactory
+import org.reflections.Reflections
+
+import scala.collection.JavaConverters.asScalaSetConverter
+import scala.collection.mutable.ArrayBuffer
+
+
+trait ComputationExecutorHook {
+
+ def getHookName(): String
+
+ def beforeExecutorExecute(engineExecutionContext: EngineExecutionContext, engineCreationContext: EngineCreationContext, codeBeforeHook: String): String = codeBeforeHook
+
+// def afterExecutorExecute(engineExecutionContext: EngineExecutionContext, code: String): Unit
+}
+
+object ComputationExecutorHook extends Logging {
+
+ private lazy val computationExecutorHooks: Array[ComputationExecutorHook] = initComputationExecutorHook
+
+ private def initComputationExecutorHook: Array[ComputationExecutorHook] = {
+ val hooks = new ArrayBuffer[ComputationExecutorHook]
+ Utils.tryCatch {
+ val reflections = new Reflections("com.webank.wedatasphere.linkis.engineconn.computation", classOf[ComputationExecutorHook])
+ val allSubClass = reflections.getSubTypesOf(classOf[ComputationExecutorHook])
+ allSubClass.asScala.foreach(l => {
+ hooks += l.newInstance
+ })
+ } {
+ t: Throwable =>
+ error(t.getMessage)
+ }
+ hooks.toArray
+ }
+
+ def getComputationExecutorHooks = computationExecutorHooks
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/ResultSetListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/ResultSetListener.scala
new file mode 100644
index 0000000000..8bce49bf30
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/ResultSetListener.scala
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.listener
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.{TaskResultCreateEvent, TaskResultSizeCreatedEvent}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnSyncListener
+
+
+trait ResultSetListener extends EngineConnSyncListener {
+
+ def onResultSetCreated(taskResultCreateEvent: TaskResultCreateEvent): Unit
+
+ def onResultSizeCreated(taskResultSizeCreatedEvent: TaskResultSizeCreatedEvent): Unit
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/TaskProgressListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/TaskProgressListener.scala
new file mode 100644
index 0000000000..4c34719fb6
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/TaskProgressListener.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.listener
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.TaskProgressUpdateEvent
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnSyncListener
+
+
+trait TaskProgressListener extends EngineConnSyncListener {
+
+ def onProgressUpdate(taskProgressUpdateEvent: TaskProgressUpdateEvent)
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/TaskStatusListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/TaskStatusListener.scala
new file mode 100644
index 0000000000..a2db53e4df
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/listener/TaskStatusListener.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.listener
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.TaskStatusChangedEvent
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnSyncListener
+
+
+trait TaskStatusListener extends EngineConnSyncListener {
+
+ def onTaskStatusChanged(taskStatusChangedEvent: TaskStatusChangedEvent): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/parser/CodeParser.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/parser/CodeParser.scala
new file mode 100644
index 0000000000..2f9af2483f
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/parser/CodeParser.scala
@@ -0,0 +1,235 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.parser
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.conf.ComputationExecutorConf
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.EngineExecutionContext
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.parser.CodeType.CodeType
+import org.apache.commons.lang.StringUtils
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
+
+
+trait CodeParser {
+
+ def parse(code: String, engineExecutorContext: EngineExecutionContext): Array[String]
+
+}
+
+abstract class SingleCodeParser extends CodeParser {
+ val codeType: CodeType
+
+ def canParse(codeType: String): Boolean = {
+ CodeType.getType(codeType) == this.codeType
+ }
+}
+
+abstract class CombinedEngineCodeParser extends CodeParser {
+ val parsers: Array[SingleCodeParser]
+
+ def getCodeType(code: String, engineExecutorContext: EngineExecutionContext): String
+
+ override def parse(code: String, engineExecutorContext: EngineExecutionContext): Array[String] = {
+ val codeType = getCodeType(code, engineExecutorContext)
+ parsers.find(_.canParse(codeType)) match {
+ case Some(parser) => parser.parse(code, engineExecutorContext)
+ case None => Array(code)
+ }
+ }
+}
+
+class ScalaCodeParser extends SingleCodeParser with Logging {
+
+ override val codeType: CodeType = CodeType.Scala
+
+ override def parse(code: String, engineExecutorContext: EngineExecutionContext): Array[String] = {
+ //val realCode = StringUtils.substringAfter(code, "\n")
+ val codeBuffer = new ArrayBuffer[String]()
+ val statementBuffer = new ArrayBuffer[String]()
+ code.split("\n").foreach {
+ case "" =>
+ case l if l.startsWith(" ") || l.startsWith("\t") => if (!l.trim.startsWith("//")) statementBuffer.append(l)
+ case l if l.startsWith("@") => statementBuffer.append(l)
+ case l if StringUtils.isNotBlank(l) =>
+ if (l.trim.startsWith("}")) {
+ statementBuffer.append(l)
+ } else {
+ if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n"))
+ statementBuffer.clear()
+ //statementBuffer.append("%scala")
+ statementBuffer.append(l)
+ }
+ case _ =>
+ }
+ if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n"))
+ codeBuffer.toArray
+ }
+}
+
+class PythonCodeParser extends SingleCodeParser {
+
+ override val codeType: CodeType = CodeType.Python
+ val openBrackets = Array("{", "(", "[")
+ val closeBrackets = Array("}", ")", "]")
+ val LOG: Logger = LoggerFactory.getLogger(getClass)
+
+ override def parse(code: String, engineExecutorContext: EngineExecutionContext): Array[String] = {
+ //val realCode = StringUtils.substringAfter(code, "\n")
+ val bracketStack = new mutable.Stack[String]
+ val codeBuffer = new ArrayBuffer[String]()
+ val statementBuffer = new ArrayBuffer[String]()
+ var notDoc = true
+ //quotationMarks is used to optimize the three quotes problem（quotationMarks用来优化三引号问题）
+ var quotationMarks: Boolean = false
+ code.split("\n").foreach {
+ case "" =>
+ case l if l.trim.contains("\"\"\"") || l.trim.contains("""'''""") => quotationMarks = !quotationMarks
+ statementBuffer.append(l)
+ recordBrackets(bracketStack, l)
+ case l if quotationMarks => statementBuffer.append(l)
+ //shanhuang 用于修复python的引号问题
+ //recordBrackets(bracketStack, l)
+ case l if notDoc && l.startsWith("#") =>
+ case l if StringUtils.isNotBlank(statementBuffer.last) && statementBuffer.last.endsWith("""\""") =>
+ statementBuffer.append(l)
+ case l if notDoc && l.startsWith(" ") =>
+ statementBuffer.append(l)
+ recordBrackets(bracketStack, l.trim)
+ case l if notDoc && l.startsWith("\t") =>
+ statementBuffer.append(l)
+ recordBrackets(bracketStack, l.trim)
+ case l if notDoc && l.startsWith("@") =>
+ statementBuffer.append(l)
+ recordBrackets(bracketStack, l.trim)
+ case l if notDoc && l.startsWith("else") => //LOG.info("I am else")
+ statementBuffer.append(l)
+ recordBrackets(bracketStack, l.trim)
+ case l if notDoc && l.startsWith("elif") => //LOG.info("I am elif")
+ statementBuffer.append(l)
+ recordBrackets(bracketStack, l.trim)
+ case l if notDoc && StringUtils.isNotBlank(l) =>
+ if (statementBuffer.nonEmpty && bracketStack.isEmpty) {
+ codeBuffer.append(statementBuffer.mkString("\n"))
+ statementBuffer.clear()
+ }
+ // statementBuffer.append("%python")
+ statementBuffer.append(l)
+ recordBrackets(bracketStack, l.trim)
+ case _ =>
+ }
+ if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n"))
+ codeBuffer.toArray
+ }
+
+ def recordBrackets(bracketStack: mutable.Stack[String], l: String): Unit = {
+ val real = l.replace("\"\"\"", "").replace("'''", "").trim
+ if (StringUtils.endsWithAny(real, openBrackets)) {
+ for (i <- (0 to real.length - 1).reverse) {
+ val token = real.substring(i, i + 1)
+ if (openBrackets.contains(token)) {
+ bracketStack.push(token)
+ }
+ }
+ }
+ if (StringUtils.startsWithAny(real, closeBrackets)) {
+ for (i <- 0 to real.length - 1) {
+ val token = real.substring(i, i + 1)
+ if (closeBrackets.contains(token)) {
+ bracketStack.pop()
+ }
+ }
+ }
+ }
+
+}
+
+
+object Main {
+ def main(args: Array[String]): Unit = {
+ val codeParser = new PythonCodeParser
+ val code = "if True: \n print 1 \nelif N=123: \n print 456 \nelse: \n print 789"
+ println(code)
+ val arrCodes = codeParser.parse(code, null)
+ print(arrCodes.mkString("||\n"))
+ }
+}
+
+class SQLCodeParser extends SingleCodeParser {
+
+ override val codeType: CodeType = CodeType.SQL
+
+ val separator = ";"
+
+ val defaultLimit: Int = ComputationExecutorConf.ENGINE_DEFAULT_LIMIT.getValue
+
+ override def parse(code: String, engineExecutorContext: EngineExecutionContext): Array[String] = {
+ //val realCode = StringUtils.substringAfter(code, "\n")
+ val codeBuffer = new ArrayBuffer[String]()
+
+ def appendStatement(sqlStatement: String): Unit = {
+ codeBuffer.append(sqlStatement)
+ }
+
+ if (StringUtils.contains(code, separator)) {
+ StringUtils.split(code, ";").foreach {
+ case s if StringUtils.isBlank(s) =>
+ case s if isSelectCmdNoLimit(s) => appendStatement(s);
+ case s => appendStatement(s);
+ }
+ } else {
+ code match {
+ case s if StringUtils.isBlank(s) =>
+ case s if isSelectCmdNoLimit(s) => appendStatement(s);
+ case s => appendStatement(s);
+ }
+ }
+ codeBuffer.toArray
+ }
+
+ def isSelectCmdNoLimit(cmd: String): Boolean = {
+ var code = cmd.trim
+ if (!cmd.split("\\s+")(0).equalsIgnoreCase("select")) return false
+ if (code.contains("limit")) code = code.substring(code.lastIndexOf("limit")).trim
+ else if (code.contains("LIMIT")) code = code.substring(code.lastIndexOf("LIMIT")).trim.toLowerCase
+ else return true
+ val hasLimit = code.matches("limit\\s+\\d+\\s*;?")
+ if (hasLimit) {
+ if (code.indexOf(";") > 0) code = code.substring(5, code.length - 1).trim
+ else code = code.substring(5).trim
+ val limitNum = code.toInt
+ if (limitNum > defaultLimit) throw new IllegalArgumentException("We at most allowed to limit " + defaultLimit + ", but your SQL has been over the max rows.")
+ }
+ !hasLimit
+ }
+}
+
+object CodeType extends Enumeration {
+ type CodeType = Value
+ val Python, SQL, Scala, Shell, Other = Value
+
+ def getType(codeType: String): CodeType = codeType.toLowerCase() match {
+ case "python" | "pyspark" | "py" => Python
+ case "sql" | "hql" => SQL
+ case "scala" => Scala
+ case "shell" => Shell
+ case _ => Other
+ }
+}
+
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/rs/RsOutputStream.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/rs/RsOutputStream.scala
similarity index 86%
rename from ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/rs/RsOutputStream.scala
rename to linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/rs/RsOutputStream.scala
index ab1897422b..42d229ee9d 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/rs/RsOutputStream.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/rs/RsOutputStream.scala
@@ -14,22 +14,19 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.engine.rs
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.rs
import java.io.OutputStream
import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetWriter
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.EngineExecutionContext
import com.webank.wedatasphere.linkis.storage.LineRecord
import scala.collection.mutable.ArrayBuffer
-/**
- * Created by enjoyyin on 2018/11/28.
- */
class RsOutputStream extends OutputStream with Logging{
private val line = ArrayBuffer[Byte]()
private var isReady = false
@@ -47,8 +44,8 @@ class RsOutputStream extends OutputStream with Logging{
}
}
- def reset(engineExecutorContext: EngineExecutorContext) = {
- writer = engineExecutorContext.createDefaultResultSetWriter()
+ def reset(engineExecutionContext: EngineExecutionContext) = {
+ writer = engineExecutionContext.createDefaultResultSetWriter()
writer.addMetaData(null)
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/service/TaskExecutionService.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/service/TaskExecutionService.scala
new file mode 100644
index 0000000000..aa45ed2a4e
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/service/TaskExecutionService.scala
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.service
+
+import com.webank.wedatasphere.linkis.governance.common.protocol.task._
+import com.webank.wedatasphere.linkis.message.builder.ServiceMethodContext
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
+
+
+trait TaskExecutionService {
+
+ def execute(requestTask: RequestTask, smc: ServiceMethodContext): ExecuteResponse
+
+// def taskStatus(taskID: String): ResponseTaskStatus
+
+ def taskProgress(taskID: String): ResponseTaskProgress
+
+ def taskLog(taskID: String): ResponseTaskLog
+
+ def killTask(taskID: String): Unit
+
+ /*def pauseTask(taskID: String): Unit
+
+ def resumeTask(taskID: String): Unit*/
+
+
+ def dealRequestTaskStatus(requestTaskStatus: RequestTaskStatus): ResponseTaskStatus
+
+ def dealRequestTaskPause(requestTaskPause: RequestTaskPause): Unit
+
+ def dealRequestTaskKill(requestTaskKill: RequestTaskKill): Unit
+
+ def dealRequestTaskResume(requestTaskResume: RequestTaskResume): Unit
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala
new file mode 100644
index 0000000000..1de9a72089
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala
@@ -0,0 +1,349 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.service
+
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.concurrent.{ExecutorService, Executors, Future, TimeUnit}
+
+import com.google.common.cache.{Cache, CacheBuilder}
+import com.webank.wedatasphere.linkis.common.listener.Event
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.LogListener
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event._
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.log.LogHelper
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.service.LockService
+import com.webank.wedatasphere.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant}
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.entity.{CommonEngineConnTask, EngineConnTask}
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.ComputationExecutor
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.listener.{ResultSetListener, TaskProgressListener, TaskStatusListener}
+import com.webank.wedatasphere.linkis.engineconn.computation.executor.utlis.{ComputationEngineUtils, ComputaionEngineContant}
+import com.webank.wedatasphere.linkis.engineconn.core.executor.ExecutorManager
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnSyncEvent
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus
+import com.webank.wedatasphere.linkis.governance.common.exception.engineconn.{EngineConnExecutorErrorCode, EngineConnExecutorErrorException}
+import com.webank.wedatasphere.linkis.governance.common.protocol.task._
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.message.builder.ServiceMethodContext
+import com.webank.wedatasphere.linkis.protocol.message.RequestProtocol
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.rpc.utils.RPCUtils
+import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, IncompleteExecuteResponse, SubmitResponse}
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
+import javax.annotation.PostConstruct
+import org.apache.commons.lang.StringUtils
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Component
+
+import scala.collection.JavaConverters._
+
+
+@Component
+class TaskExecutionServiceImpl extends TaskExecutionService with Logging with ResultSetListener with LogListener with TaskProgressListener with TaskStatusListener {
+
+ private val executorManager = ExecutorManager.getInstance()
+ private val taskExecutedNum = new AtomicInteger(0)
+ private var lastTask: EngineConnTask = _
+ private var executorService: ExecutorService = _
+ private var lastTaskFuture: Future[_] = _
+
+ @Autowired
+ private var lockService: LockService = _
+ private val asyncListenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext.getEngineConnAsyncListenerBus
+ private val syncListenerBus = ExecutorListenerBusContext.getExecutorListenerBusContext.getEngineConnSyncListenerBus
+ private val taskIdCache: Cache[String, ComputationExecutor] = CacheBuilder.newBuilder().expireAfterAccess(EngineConnConf.ENGINE_TASK_EXPIRE_TIME.getValue, TimeUnit.MILLISECONDS)
+ .maximumSize(EngineConnConstant.MAX_TASK_NUM).build()
+
+ @PostConstruct
+ def init(): Unit = {
+ LogHelper.setLogListener(this)
+ syncListenerBus.addListener(this)
+ executorService = Executors.newSingleThreadExecutor()
+ }
+
+ private def sendToEntrance(task: EngineConnTask, msg: RequestProtocol): Unit = synchronized {
+ Utils.tryCatch {
+ var sender : Sender = null
+ if (null != task && null != task.getCallbackServiceInstance()) {
+ sender = Sender.getSender(task.getCallbackServiceInstance())
+ sender.send(msg)
+ } else {
+ // todo
+ debug("SendtoEntrance error, cannot find entrance instance.")
+ }
+ }{
+ t =>
+ val errorMsg = s"SendToEntrance error. $msg" + t.getCause
+ error(errorMsg, t)
+ throw new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.SEND_TO_ENTRANCE_ERROR, errorMsg)
+ }
+ }
+
+ @Receiver
+ override def execute(requestTask: RequestTask, smc: ServiceMethodContext): ExecuteResponse = {
+ // smc // todo get sender
+ // check lock
+ if (StringUtils.isBlank(requestTask.getLock)) {
+ error(s"Invalid lock : ${requestTask.getLock} , requestTask : " + BDPJettyServerHelper.gson.toJson(requestTask))
+ return ErrorExecuteResponse(s"Invalid lock : ${requestTask.getLock}.", new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_PARAMS, "Invalid lock or code(请获取到锁后再提交任务.)"))
+ }
+ if (!lockService.isLockExist(requestTask.getLock)) {
+ error(s"Lock ${requestTask.getLock} not exist, cannot execute.")
+ return ErrorExecuteResponse("Lock not exixt", new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_LOCK, "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)."))
+ }
+
+ if (StringUtils.isBlank(requestTask.getCode)) {
+ return IncompleteExecuteResponse("Your code is incomplete, it may be that only comments are selected for execution(您的代码不完整，可能是仅仅选中了注释进行执行)")
+ }
+
+ // 获取任务类型和任务代码，运行任务
+ val taskId: Int = taskExecutedNum.incrementAndGet()
+ val retryAble: Boolean = {
+ val retry = requestTask.getProperties.getOrDefault(ComputaionEngineContant.RETRYABLE_TYPE_NAME, null)
+ if (null != retry) retry.asInstanceOf[Boolean]
+ else false
+ }
+ val task = new CommonEngineConnTask(String.valueOf(taskId), retryAble)
+ task.setCode(requestTask.getCode)
+ task.setProperties(requestTask.getProperties)
+ task.data(ComputaionEngineContant.LOCK_TYPE_NAME, requestTask.getLock)
+ task.setStatus(ExecutionNodeStatus.Scheduled)
+ val labels = requestTask.getLabels.asScala.toArray
+ task.setLabels(labels)
+ val entranceServerInstance = RPCUtils.getServiceInstanceFromSender(smc.getSender)
+ task.setCallbackServiceInstance(entranceServerInstance)
+ val executor = executorManager.getExecutorByLabels(labels)
+ executor match {
+ case computationExecutor: ComputationExecutor =>
+ taskIdCache.put(task.getTaskId, computationExecutor)
+ submitTask(task, computationExecutor)
+ case o =>
+ val msg = "Invalid compuatitionExecutor : " + ComputationEngineUtils.GSON.toJson(o) + ", labels : " + ComputationEngineUtils.GSON.toJson(labels) + ", requestTask : " + ComputationEngineUtils.GSON.toJson(requestTask)
+ error(msg)
+ ErrorExecuteResponse("Invalid computationExecutor(生成无效的计算引擎，请联系管理员).",
+ new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_ENGINE_TYPE, msg))
+ }
+
+ }
+
+
+// override def taskStatus(taskID: String): ResponseTaskStatus = {
+// val task = taskIdCache.get(taskID)
+// ResponseTaskStatus(taskID, task.getStatus.id)
+// }
+
+ private def submitTask(task: CommonEngineConnTask, computationExecutor: ComputationExecutor) = {
+ val runTask = new Runnable {
+ override def run(): Unit = Utils.tryAndWarn {
+ val response = computationExecutor.execute(task)
+ response match {
+ case ErrorExecuteResponse(message, throwable) =>
+ sendToEntrance(task, ResponseTaskError(task.getTaskId, message))
+ error(message, throwable)
+ computationExecutor.transformTaskStatus(task, ExecutionNodeStatus.Failed)
+ case _ =>
+ }
+ }
+ }
+ lastTask = task
+ lastTaskFuture = executorService.submit(runTask)
+ info(s"Task ${task.getTaskId} was submited.")
+ SubmitResponse(task.getTaskId)
+ }
+
+ override def taskProgress(taskID: String): ResponseTaskProgress = {
+ var response = ResponseTaskProgress(taskID, -1, null)
+ if (StringUtils.isBlank(taskID)) response
+ val executor = taskIdCache.getIfPresent(taskID)
+ if (null != executor) {
+ val task = executor.getTaskById(taskID)
+ if (null != task) {
+ if (ExecutionNodeStatus.isCompleted(task.getStatus)) {
+ response = ResponseTaskProgress(taskID, 1.0f, null)
+ } else {
+ response = ResponseTaskProgress(taskID, executor.progress(), executor.getProgressInfo)
+ }
+ } else {
+ response = ResponseTaskProgress(taskID, -1, null)
+ }
+ } else {
+ error(s"Executor of taskId : $taskID is not cached.")
+ }
+ response
+ }
+
+ override def taskLog(taskID: String): ResponseTaskLog = {
+ // todo check
+ null
+ }
+
+// override def pauseTask(taskID: String): Unit = {
+// val task = taskIdCache.get(taskID)
+// // todo
+// }
+
+ override def killTask(taskID: String): Unit = {
+ val executor = taskIdCache.getIfPresent(taskID)
+ if (null != executor) {
+ executor.killTask(taskID)
+ } else {
+ error(s"Executor of taskId : $taskID is not cached.")
+ }
+ Utils.tryAndWarn (Thread.sleep(50))
+ if (null != lastTask && lastTask.getTaskId.equalsIgnoreCase(taskID)) {
+ if (null != lastTaskFuture && !lastTaskFuture.isDone) {
+ Utils.tryAndWarn(lastTaskFuture.cancel(true))
+ }
+ }
+ }
+
+ /*override def resumeTask(taskID: String): Unit = {
+ // todo
+ }*/
+
+ @Receiver
+ override def dealRequestTaskStatus(requestTaskStatus: RequestTaskStatus): ResponseTaskStatus = {
+ val task = getTaskByTaskId(requestTaskStatus.execId)
+ if (null != task) {
+ ResponseTaskStatus(task.getTaskId, task.getStatus)
+ } else {
+ val msg = "Task null! requestTaskStatus: " + ComputationEngineUtils.GSON.toJson(requestTaskStatus)
+ error(msg)
+ ResponseTaskStatus(requestTaskStatus.execId, ExecutionNodeStatus.Cancelled)
+ }
+ }
+
+ @Receiver
+ override def dealRequestTaskPause(requestTaskPause: RequestTaskPause): Unit = {
+ info(s"Pause is Not supported for task : " + requestTaskPause.execId )
+ }
+
+ @Receiver
+ override def dealRequestTaskKill(requestTaskKill: RequestTaskKill): Unit = {
+ val executor = taskIdCache.getIfPresent(requestTaskKill.execId)
+ if (null != executor) {
+ executor.killTask(requestTaskKill.execId)
+ info(s"TaskId : ${requestTaskKill.execId} was killed by user.")
+ } else {
+ error(s"Invalid executor : null for taskId : ${requestTaskKill.execId}")
+ }
+ }
+
+ @Receiver
+ override def dealRequestTaskResume(requestTaskResume: RequestTaskResume): Unit = {
+ info(s"Resume is Not support for task : " + requestTaskResume.execId )
+ }
+
+ override def onEvent(event: EngineConnSyncEvent): Unit = event match {
+ case taskStatusChangedEvent: TaskStatusChangedEvent => onTaskStatusChanged(taskStatusChangedEvent)
+ case taskProgressUpdateEvent: TaskProgressUpdateEvent => onProgressUpdate(taskProgressUpdateEvent)
+ case logUpdateEvent: TaskLogUpdateEvent => onLogUpdate(logUpdateEvent)
+ case taskResultCreateEvent: TaskResultCreateEvent => onResultSetCreated(taskResultCreateEvent)
+ case taskResultSizeCreatedEvent: TaskResultSizeCreatedEvent => onResultSizeCreated(taskResultSizeCreatedEvent)
+ case _ =>
+ warn("Unknown event : " + BDPJettyServerHelper.gson.toJson(event))
+ }
+
+ override def onLogUpdate(logUpdateEvent: TaskLogUpdateEvent): Unit = {
+ if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) {
+ if (null != logUpdateEvent && StringUtils.isNotBlank(logUpdateEvent.taskId)) {
+ val task = getTaskByTaskId(logUpdateEvent.taskId)
+ if (null != task) {
+ sendToEntrance(task, ResponseTaskLog(logUpdateEvent.taskId, logUpdateEvent.log))
+ } else {
+ error("Task cannot null! logupdateEvent: " + ComputationEngineUtils.GSON.toJson(logUpdateEvent))
+ }
+ } else {
+ val executor = ExecutorManager.getInstance().getDefaultExecutor
+ executor match {
+ case computationExecutor: ComputationExecutor =>
+ sendToEntrance(lastTask, ResponseTaskLog(lastTask.getTaskId, logUpdateEvent.log))
+ case _ =>
+ error("OnLogUpdate error. Invalid ComputationExecutor : " + ComputationEngineUtils.GSON.toJson(executor))
+ }
+ }
+
+ }
+ }
+
+ override def onTaskStatusChanged(taskStatusChangedEvent: TaskStatusChangedEvent): Unit = {
+ val task = getTaskByTaskId(taskStatusChangedEvent.taskId)
+ if (null != task) {
+ if (ExecutionNodeStatus.isCompleted(taskStatusChangedEvent.toStatus)) {
+ lastTask = task
+ LogHelper.pushAllRemainLogs()
+ }
+ sendToEntrance(task, ResponseTaskStatus(taskStatusChangedEvent.taskId, taskStatusChangedEvent.toStatus))
+ } else {
+ error("Task cannot null! taskStatusChangedEvent: " + ComputationEngineUtils.GSON.toJson(taskStatusChangedEvent))
+ }
+ }
+
+ override def onProgressUpdate(taskProgressUpdateEvent: TaskProgressUpdateEvent): Unit = {
+ if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) {
+ val task = getTaskByTaskId(taskProgressUpdateEvent.taskId)
+ if (null != task) {
+ sendToEntrance(task, ResponseTaskProgress(taskProgressUpdateEvent.taskId, taskProgressUpdateEvent.progress, taskProgressUpdateEvent.progressInfo))
+ } else {
+ error("Task cannot null! taskProgressUpdateEvent : " + ComputationEngineUtils.GSON.toJson(taskProgressUpdateEvent))
+ }
+ }
+ }
+
+ override def onResultSetCreated(taskResultCreateEvent: TaskResultCreateEvent): Unit = {
+ info(s"start to deal result event ${taskResultCreateEvent.taskId}")
+ val task = getTaskByTaskId(taskResultCreateEvent.taskId)
+ if (null != task) {
+ sendToEntrance(task, ResponseTaskResultSet(
+ taskResultCreateEvent.taskId,
+ taskResultCreateEvent.resStr,
+ taskResultCreateEvent.alias
+ ))
+ } else {
+ error("Task cannot null! taskResultCreateEvent: " + ComputationEngineUtils.GSON.toJson(taskResultCreateEvent))
+ }
+ info(s"Finished to deal result event ${taskResultCreateEvent.taskId}")
+ }
+
+ def getTaskByTaskId(taskId: String): EngineConnTask = {
+ if (StringUtils.isBlank(taskId)) return null
+ val executor = taskIdCache.getIfPresent(taskId)
+ if (null != executor) {
+ executor.getTaskById(taskId)
+ } else {
+ error(s"Executor of taskId : $taskId is not cached.")
+ null
+ }
+ }
+
+ override def onResultSizeCreated(taskResultSizeCreatedEvent: TaskResultSizeCreatedEvent): Unit = {
+ val task = getTaskByTaskId(taskResultSizeCreatedEvent.taskId)
+ if (null != task) {
+ sendToEntrance(task, ResponseTaskResultSize(
+ taskResultSizeCreatedEvent.taskId,
+ taskResultSizeCreatedEvent.resultSize
+ ))
+ } else {
+ error("Task cannot null! taskResultSizeCreatedEvent: " + ComputationEngineUtils.GSON.toJson(taskResultSizeCreatedEvent))
+ }
+ }
+
+ override def onEventError(event: Event, t: Throwable): Unit = {
+
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/utlis/ComputaionEngineContant.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/utlis/ComputaionEngineContant.scala
new file mode 100644
index 0000000000..40eefeff1d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/utlis/ComputaionEngineContant.scala
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.utlis
+
+
+object ComputaionEngineContant {
+
+ def RETRYABLE_TYPE_NAME = "taskRetryable"
+
+ def LOCK_TYPE_NAME = "lock"
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/utlis/ComputationEngineUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/utlis/ComputationEngineUtils.scala
new file mode 100644
index 0000000000..049bca958b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/com/webank/wedatasphere/linkis/engineconn/computation/executor/utlis/ComputationEngineUtils.scala
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.computation.executor.utlis
+
+import com.google.gson.{Gson, GsonBuilder}
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
+
+
+object ComputationEngineUtils {
+
+ def GSON: Gson = BDPJettyServerHelper.gson
+
+ def GSON_PRETTY: Gson = new GsonBuilder().setPrettyPrinting().create()
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/pom.xml
new file mode 100644
index 0000000000..1b707e875c
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/pom.xml
@@ -0,0 +1,91 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../../pom.xml
+
+ 4.0.0
+
+ linkis-engineconn-common
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-common
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-protocol
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-label-common
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-manager-common
+ ${linkis.version}
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConf.scala
new file mode 100644
index 0000000000..52dc90ed39
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConf.scala
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.conf
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars
+
+
+object EngineConnConf {
+
+ val ENGINE_EXECUTIONS = CommonVars("wds.linkis.engine.connector.executions", "com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.ComputationEngineConnExecution")
+
+ val ENGINE_CONN_HOOKS = CommonVars("wds.linkis.engine.connector.hooks", "com.webank.wedatasphere.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook")
+
+
+ val ENGINE_LAUNCH_CMD_PARAMS_USER_KEY = CommonVars("wds.linkis.engine.launch.cmd.params.user.key", "user")
+
+ val ENGINE_SUPPORT_PARALLELISM = CommonVars("wds.linkis.engine.parallelism.support.enabled", false)
+
+ val ENGINE_PUSH_LOG_TO_ENTRANCE = CommonVars("wds.linkis.engine.push.log.enable", true)
+
+ val ENGINECONN_PLUGIN_CLAZZ = CommonVars("wds.linkis.engineconn.plugin.default.clazz", "com.webank.wedatasphere.linkis.engineplugin.hive.HiveEngineConnPlugin")
+
+
+ val ENGINE_TASK_EXPIRE_TIME = CommonVars("wds.linkis.engine.task.expire.time", 1000 * 3600 * 24)
+
+ val ENGINE_LOCK_REFRESH_TIME = CommonVars("wds.linkis.engine.lock.refresh.time", 1000 * 60 * 3)
+
+ val ENGINE_CONN_LOCALPATH_PWD_KEY = CommonVars("wds.linkis.engine.localpath.pwd.key", "PWD")
+
+ val ENGINE_CONN_LOCAL_LOG_DIRS_KEY = CommonVars("wds.linkis.engine.logs.dir.key", "LOG_DIRS")
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConstant.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConstant.scala
new file mode 100644
index 0000000000..b5f7d6a8fb
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConstant.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.conf
+
+
+object EngineConnConstant {
+
+ val MAX_TASK_NUM = 10000
+
+ val SPRING_CONF_MAP_NAME = "SpringConfMap"
+
+ val MAX_EXECUTOR_ID_NAME = "MaxExecutorId"
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/creation/EngineCreationContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/creation/EngineCreationContext.scala
new file mode 100644
index 0000000000..281a3de469
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/creation/EngineCreationContext.scala
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.creation
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+
+trait EngineCreationContext {
+
+ def getUser: String
+
+ def setUser(user: String): Unit
+
+ def getTicketId: String
+
+ def setTicketId(ticketId: String): Unit
+
+ def getLabels(): util.List[Label[_]]
+
+ def setLabels(labels: util.List[Label[_]]): Unit
+
+ def getEMInstance: ServiceInstance
+
+ def setEMInstance(instance: ServiceInstance): Unit
+
+ def getOptions: util.Map[String, String]
+
+ def setOptions(options: util.Map[String, String])
+
+ def getExecutorId: Int
+
+ def setExecutorId(id: Int)
+
+ def setArgs(args: Array[String])
+
+ def getArgs: Array[String]
+}
+
+class DefaultEngineCreationContext extends EngineCreationContext {
+
+ private var user: String = _
+
+ private var ticketId: String = _
+
+ private var labels: util.List[Label[_]] = _
+
+ private var options: util.Map[String, String] = _
+
+ private var emInstance: ServiceInstance = _
+
+ private var executorId: Int = 0
+
+ private var args: Array[String] = null
+
+ override def getTicketId: String = this.ticketId
+
+ override def setTicketId(ticketId: String): Unit = this.ticketId = ticketId
+
+ override def getLabels(): util.List[Label[_]] = this.labels
+
+ override def setLabels(labels: util.List[Label[_]]): Unit = this.labels = labels
+
+ override def getEMInstance: ServiceInstance = this.emInstance
+
+ override def setEMInstance(instance: ServiceInstance): Unit = this.emInstance = instance
+
+ override def getOptions: util.Map[String, String] = this.options
+
+ override def setOptions(options: util.Map[String, String]): Unit = this.options = options
+
+ override def getUser: String = user
+
+ override def setUser(user: String): Unit = this.user = user
+
+ override def getExecutorId: Int = executorId
+
+ override def setExecutorId(id: Int): Unit = executorId = id
+
+ override def setArgs(args: Array[String]): Unit = this.args = args
+
+ override def getArgs: Array[String] = args
+
+
+ override def toString = s"DefaultEngineCreationContext(user-$user, ticketID-$ticketId, emInstance-$emInstance)"
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/engineconn/EngineConn.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/engineconn/EngineConn.scala
new file mode 100644
index 0000000000..c97339e101
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/engineconn/EngineConn.scala
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.engineconn
+
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+
+
+trait EngineConn {
+
+ /**
+ * 底层engine的类型比如：Spark、hive
+ *
+ * @return
+ */
+ def getEngineType(): String
+
+ def setEngineType(engineType: String): Unit
+
+ /**
+ * 底层计算存储Engine的具体连接信息，比如SparkSession,hive的sessionState
+ *
+ * @return
+ */
+ def getEngine(): Any
+
+ def setEngine(engine: Any): Unit
+
+ def getEngineCreationContext: EngineCreationContext
+
+
+}
+
+class DefaultEngineConn(engineCreationContext: EngineCreationContext) extends EngineConn {
+
+ var engineType: String = "spark"
+
+ var engine: Any = null
+
+
+ /**
+ * 底层engine的类型比如：Spark、hive
+ *
+ * @return
+ */
+ override def getEngineType(): String = engineType
+
+ override def setEngineType(engineType: String): Unit = this.engineType = engineType
+
+ /**
+ * 底层计算存储Engine的具体连接信息，比如SparkSession,hive的sessionState
+ *
+ * @return
+ */
+ override def getEngine(): Any = engine
+
+ override def setEngine(engine: Any): Unit = this.engine = engine
+
+ override def getEngineCreationContext: EngineCreationContext = engineCreationContext
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/exception/EngineConnException.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/exception/EngineConnException.scala
new file mode 100644
index 0000000000..51fa841b5f
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/exception/EngineConnException.scala
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.exception
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+
+
+case class EngineTypeVersionNotMatchException(msg: String) extends ErrorException(40010, msg)
+
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/execution/EngineExecution.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/execution/EngineExecution.scala
new file mode 100644
index 0000000000..b6e6252140
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/execution/EngineExecution.scala
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.execution
+
+import com.webank.wedatasphere.linkis.common.exception.FatalException
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.common.conf.EngineConnConf
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.common.engineconn.EngineConn
+import org.apache.commons.lang.StringUtils
+
+
+trait EngineExecution {
+
+ def execute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit
+}
+
+object EngineExecution extends Logging {
+
+
+ private val engineExecutions = initEngineExecutions
+
+ private def initEngineExecutions: Array[EngineExecution] = {
+
+ val executions = EngineConnConf.ENGINE_EXECUTIONS.getValue
+ if (StringUtils.isNotBlank(executions)) {
+ val clazzArr = executions.split(",")
+ if (null != clazzArr && !clazzArr.isEmpty) {
+ clazzArr.map { clazz =>
+ Utils.getClassInstance[EngineExecution](clazz)
+ }
+ } else {
+ throw new FatalException(0, "ENGINE_EXECUTIONS must be set")
+ }
+ } else {
+ throw new FatalException(0, "ENGINE_EXECUTIONS must be set")
+ }
+
+ }
+
+ def getEngineExecutions: Array[EngineExecution] = engineExecutions
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/hook/EngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/hook/EngineConnHook.scala
new file mode 100644
index 0000000000..dab0149ed9
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/hook/EngineConnHook.scala
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.hook
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.common.conf.EngineConnConf
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.common.engineconn.EngineConn
+import org.apache.commons.lang.StringUtils
+
+
+trait EngineConnHook {
+
+ def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit
+
+ /*def afterCreateEngineConn(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit*/
+
+ def beforeExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit
+
+ def afterExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit
+
+ def afterEngineServerStartFailed(engineCreationContext: EngineCreationContext, throwable: Throwable): Unit = {
+
+ }
+
+ def afterEngineServerStartSuccess(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {
+
+ }
+
+}
+
+object EngineConnHook extends Logging {
+
+ private val engineConnHooks = initEngineConnHooks
+
+ private def initEngineConnHooks: Array[EngineConnHook] = {
+
+ val hooks = EngineConnConf.ENGINE_CONN_HOOKS.getValue
+ if (StringUtils.isNotBlank(hooks)) {
+ val clazzArr = hooks.split(",")
+ if (null != clazzArr && clazzArr.nonEmpty) {
+ clazzArr.map { clazz =>
+ Utils.tryAndWarn(Utils.getClassInstance[EngineConnHook](clazz))
+ }.filter(_ != null)
+ } else {
+ Array.empty
+ }
+ } else {
+ Array.empty
+ }
+
+ }
+
+ def getEngineConnHooks: Array[EngineConnHook] = engineConnHooks
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/pom.xml
new file mode 100644
index 0000000000..cbdca0a76b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/pom.xml
@@ -0,0 +1,84 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../../pom.xml
+
+ 4.0.0
+
+ linkis-engineconn-core
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-common
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-computation-governance-common
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-plugin-core
+ ${linkis.version}
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/EngineConnObject.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/EngineConnObject.scala
new file mode 100644
index 0000000000..5b7b72cc86
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/EngineConnObject.scala
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.core
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.common.conf.EngineConnConf
+import com.webank.wedatasphere.linkis.engineconn.common.creation.{DefaultEngineCreationContext, EngineCreationContext}
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.EngineConnPlugin
+
+object EngineConnObject extends Logging {
+
+ private val engineConnPlugin: EngineConnPlugin = Utils.tryCatch {
+ Utils.getClassInstance[EngineConnPlugin](EngineConnConf.ENGINECONN_PLUGIN_CLAZZ.getValue)
+ } { case t: Throwable =>
+ error("Failed to create engineConnPlugin: " + EngineConnConf.ENGINECONN_PLUGIN_CLAZZ.getValue, t)
+ System.exit(1)
+ null
+ }
+ private var engineCreationContext: EngineCreationContext = _
+ private var ready = false
+
+ def isReady: Boolean = isReady
+
+ def setReady: Unit = this.ready = true
+
+ def getEngineConnPlugin: EngineConnPlugin = engineConnPlugin
+
+ def getEngineCreationContext: EngineCreationContext = this.engineCreationContext
+
+ def setEngineCreationContext( engineCreationContext: EngineCreationContext): Unit = this.engineCreationContext = engineCreationContext
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/engineconn/EngineConnManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/engineconn/EngineConnManager.scala
new file mode 100644
index 0000000000..6f417954ab
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/engineconn/EngineConnManager.scala
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.core.engineconn
+
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.common.engineconn.EngineConn
+import com.webank.wedatasphere.linkis.engineconn.core.EngineConnObject
+
+trait EngineConnManager {
+
+ def createEngineConn(engineCreationContext: EngineCreationContext): EngineConn
+
+ def getEngineConn(): EngineConn
+
+
+}
+
+object EngineConnManager {
+ val engineConnManager: EngineConnManager = new DefaultEngineConnManager
+
+ def getEngineConnManager = engineConnManager
+}
+
+class DefaultEngineConnManager extends EngineConnManager {
+
+ private var engineConn: EngineConn = _
+
+ override def createEngineConn(engineCreationContext: EngineCreationContext): EngineConn = {
+ this.engineConn = EngineConnObject.getEngineConnPlugin.getEngineConnFactory.createEngineConn(engineCreationContext)
+ this.engineConn
+ }
+
+ override def getEngineConn(): EngineConn = this.engineConn
+
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/executor/ExecutorManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/executor/ExecutorManager.scala
new file mode 100644
index 0000000000..a6ac558c79
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/executor/ExecutorManager.scala
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.core.executor
+
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicInteger
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+import com.webank.wedatasphere.linkis.engineconn.core.EngineConnObject
+import com.webank.wedatasphere.linkis.engineconn.core.engineconn.EngineConnManager
+import com.webank.wedatasphere.linkis.engineconn.core.util.EngineConnUtils
+import com.webank.wedatasphere.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.{Executor, LabelExecutor}
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.EngineConnPlugin
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.creation.{MultiExecutorEngineConnFactory, SingleExecutorEngineConnFactory}
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.exception.{EngineConnPluginErrorCode, EngineConnPluginErrorException}
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.manager.label.entity.engine.EngineRunTypeLabel
+
+import scala.collection.JavaConverters._
+
+trait ExecutorManager {
+
+ def getDefaultExecutor: Executor
+
+ def getExecutorByLabels(labels: Array[Label[_]]): Executor
+
+ def createExecutor(engineCreationContext: EngineCreationContext, labels: Array[Label[_]]): Executor
+
+ def getAllExecutorsMap(): util.Map[String, Executor]
+
+ def generateId(): Int
+
+}
+
+class ExecutorManagerImpl extends ExecutorManager with Logging {
+
+ private val executors: util.Map[String, Executor] = new ConcurrentHashMap[String, Executor](2)
+ private val GSON = EngineConnUtils.GSON
+ private var defaultExecutor: Executor = _
+
+ private val idCreator = new AtomicInteger()
+
+ override def getDefaultExecutor: Executor = {
+ if (null == defaultExecutor || defaultExecutor.isClosed()) {
+ val engineConn = EngineConnManager.getEngineConnManager.getEngineConn()
+ if (null == engineConn) {
+ error("Create default executor failed., engineConn not ready")
+ return null
+ }
+ Utils.tryCatch {
+ createExecutor(engineConn.getEngineCreationContext, null)
+ } {
+ case t: Throwable =>
+ error("Create default executor failed. Now will try again once.", t)
+ createExecutor(engineConn.getEngineCreationContext, null)
+ }
+ }
+ defaultExecutor
+ }
+
+ override def createExecutor(engineCreationContext: EngineCreationContext, labels: Array[Label[_]]): Executor = {
+ val engineConn = EngineConnManager.getEngineConnManager.getEngineConn()
+ val enginePlugin: EngineConnPlugin = EngineConnObject.getEngineConnPlugin
+ val isDefault: Boolean = if (null == labels || labels.isEmpty || null == defaultExecutor || defaultExecutor.isClosed()) true else false
+ var runType: String = null
+ if (!isDefault) {
+ runType = getRunTypeFromLables(labels)
+ if (null == runType) {
+ val msg = "Invalid RunType label. labels : " + GSON.toJson(labels)
+ error(msg)
+ throw new EngineConnPluginErrorException(EngineConnPluginErrorCode.INVALID_LABELS, msg)
+ }
+ }
+ val executor: Executor = {
+ enginePlugin.getEngineConnFactory match {
+ case engineConnFactory: SingleExecutorEngineConnFactory =>
+ engineConnFactory.createExecutor(engineCreationContext, engineConn)
+ case engineConnFactory: MultiExecutorEngineConnFactory =>
+ val executorFactories = engineConnFactory.getExecutorFactories
+ val chooseExecutorFactory = if (isDefault) {
+ info("use default executor")
+ Some(engineConnFactory.getDefaultExecutorFactory)
+ } else {
+ executorFactories.find(e => e.canCreate(labels))
+ }
+ if (chooseExecutorFactory.isEmpty) {
+ val msg = if (null == labels) {
+ "Cannot get default executorFactory. EngineCreation labels: " + GSON.toJson(engineCreationContext.getLabels())
+ } else {
+ "Cannot get valid executorFactory. EngineCreation labels: " + GSON.toJson(labels)
+ }
+ error(msg)
+ throw new EngineConnPluginErrorException(EngineConnPluginErrorCode.INVALID_LABELS, msg)
+ } else {
+ chooseExecutorFactory.get.createExecutor(engineCreationContext, engineConn, labels)
+ }
+ case o =>
+ error("Invalid ExecutorFactory " + GSON.toJson(o))
+ null
+ }
+ }
+ info(s"Finished create executor ${executor.getId()}")
+ executor.init()
+ info(s"Finished init executor ${executor.getId()}")
+ executor match {
+ case labelExecutor: LabelExecutor =>
+ runType = getRunTypeFromLables(labelExecutor.getExecutorLabels().asScala.toArray)
+ case _ =>
+ }
+ executors.put(runType, executor)
+ if (isDefault) {
+ defaultExecutor = executor
+ }
+ executor
+ }
+
+ override def generateId(): Int = idCreator.getAndIncrement()
+
+ override def getExecutorByLabels(labels: Array[Label[_]]): Executor = {
+ var runType: String = null
+ labels.foreach(l => l match {
+ case label: EngineRunTypeLabel =>
+ runType = label.getRunType
+ case _ =>
+ })
+ if (null == runType) {
+ error("Invalid RunType Label. labels: " + GSON.toJson(labels))
+ return null
+ }
+ if (!executors.containsKey(runType)) {
+ val engineConn = EngineConnManager.getEngineConnManager.getEngineConn()
+ createExecutor(engineConn.getEngineCreationContext, labels)
+ }
+ executors.get(runType)
+ }
+
+ override def getAllExecutorsMap(): util.Map[String, Executor] = executors
+
+ private def getRunTypeFromLables(labels: Array[Label[_]]): String = {
+
+ labels.foreach {
+ case label: EngineRunTypeLabel =>
+ return label.getRunType
+ case _ =>
+ }
+ null
+ }
+}
+
+object ExecutorManager {
+
+ private val executorManager: ExecutorManager = new ExecutorManagerImpl
+ Utils.getClassInstance[ExecutorManager](EngineConnExecutorConfiguration.EXECUTOR_MANAGER_SERVICE_CLAZZ.getValue)
+
+ def getInstance(): ExecutorManager = executorManager
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/hook/ShutdownHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/hook/ShutdownHook.scala
new file mode 100644
index 0000000000..9e226fe3ec
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/hook/ShutdownHook.scala
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.core.hook
+
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.locks.ReentrantLock
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+
+class ShutdownHook extends Logging {
+
+ var exitCode = 0
+
+ private val lock = new ReentrantLock()
+ private val condition = lock.newCondition()
+
+ // Guarded by "lock"
+ private var error: Throwable = null
+
+ // Guarded by "lock"
+ private var stopped: Boolean = false
+
+ def notifyError(e: Throwable): Unit = {
+ lock.lock()
+ try {
+ setExitCode(1)
+ error = e
+ error("process exit reason: ", e)
+ condition.signalAll()
+ } finally {
+ lock.unlock()
+ }
+ }
+
+ def notifyStop(): Unit = {
+ lock.lock()
+ try {
+ setExitCode(0)
+ stopped = true
+ condition.signalAll()
+ } finally {
+ lock.unlock()
+ }
+ }
+
+ /**
+ * Return `true` if it's stopped; or throw the reported error if `notifyError` has been called; or
+ * `false` if the waiting time detectably elapsed before return from the method.
+ */
+ def waitForStopOrError(timeout: Long = -1): Boolean = {
+ lock.lock()
+ try {
+ if (timeout < 0) {
+ while (!stopped && error == null) {
+ condition.await()
+ }
+ } else {
+ var nanos = TimeUnit.MILLISECONDS.toNanos(timeout)
+ while (!stopped && error == null && nanos > 0) {
+ nanos = condition.awaitNanos(nanos)
+ }
+ }
+ // If already had error, then throw it
+ if (error != null) throw error
+ // already stopped or timeout
+ stopped
+ } finally {
+ lock.unlock()
+ }
+ }
+
+ def setExitCode(exitCode: Int): Unit = this.exitCode = exitCode
+
+ def getExitCode(): Int = exitCode
+
+}
+
+object ShutdownHook {
+
+ private val shutdownHook = new ShutdownHook
+
+ def getShutdownHook: ShutdownHook = shutdownHook
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/util/EngineConnUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/util/EngineConnUtils.scala
new file mode 100644
index 0000000000..e1d4e5ba83
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/core/util/EngineConnUtils.scala
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.core.util
+
+import com.google.gson.Gson
+
+object EngineConnUtils {
+
+ val GSON = new Gson()
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/pom.xml
new file mode 100644
index 0000000000..1b6b563f61
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/pom.xml
@@ -0,0 +1,99 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../../pom.xml
+
+ 4.0.0
+
+ linkis-accessible-executor
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-executor-core
+ ${linkis.version}
+
+
+ reflections
+ org.reflections
+
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-launch
+ ${linkis.version}
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-message-scheduler
+ ${linkis.version}
+
+
+ reflections
+ org.reflections
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ ${basedir}/src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/AbstractLogCache.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/AbstractLogCache.java
new file mode 100644
index 0000000000..c60eacd8c0
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/AbstractLogCache.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.log;
+
+import com.webank.wedatasphere.linkis.common.log.LogUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.LogEvent;
+
+public abstract class AbstractLogCache implements LogCache {
+ protected String generateLog(LogEvent event) {
+ if (event.getLevel() == Level.INFO) {
+ return LogUtils.generateInfo(event.getMessage().toString());
+ } else if (event.getLevel() == Level.WARN) {
+ return LogUtils.generateWarn(event.getMessage().toString());
+ } else if (event.getLevel() == Level.ERROR) {
+ return LogUtils.generateERROR(event.getMessage().toString());
+ } else if (event.getLevel() == Level.FATAL) {
+ return LogUtils.generateSystemError(event.getMessage().toString());
+ }
+ return "";
+ }
+}
diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/LogCache.java
similarity index 86%
rename from ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java
rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/LogCache.java
index 63b67fba23..4000f33e51 100644
--- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/LogCache.java
@@ -14,26 +14,24 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.engine.log;
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.log;
import java.util.List;
-/**
- * created by enjoyyin on 2018/11/27
- * Description:
- */
public interface LogCache {
/**
* Store logs in the cache for subsequent delivery
* 将日志存储到缓存当中，用于后续的发送
+ *
* @param log 日志信息
*/
void cacheLog(String log);
/**
* Get the log of the number of num（获取num数目的日志）
- * @param num The number of logs you want to get（希望获取的日志的数量）
+ *
+ * @param num The number of logs you want to get（希望获取的日志的数量）
* @return Log List Collection（日志List集合）
*/
List getLog(int num);
@@ -41,6 +39,7 @@ public interface LogCache {
/**
* Take all the logs in the log cache for the engine to take out all the logs after performing a task
* 将日志缓存中的所有日志都取出，用于engine执行完一个任务之后将日志全部取出
+ *
* @return
*/
List getRemain();
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/MountLogCache.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/MountLogCache.java
new file mode 100644
index 0000000000..4058910ad9
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/MountLogCache.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.log;
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MountLogCache extends AbstractLogCache {
+
+ private static final Logger logger = LoggerFactory.getLogger(MountLogCache.class);
+
+ class CircularQueue {
+
+ private int max;
+ private String[] elements;
+ private int front, rear, count;
+
+ CircularQueue() {
+ this((Integer) AccessibleExecutorConfiguration.ENGINECONN_LOG_CACHE_NUM().getValue());
+ }
+
+ CircularQueue(int max) {
+ this.max = max;
+ this.elements = new String[max];
+ }
+
+ public boolean isEmpty() {
+ return count == 0;
+ }
+
+ public synchronized void enqueue(String value) {
+ if (count == max) {
+ logger.warn("Queue is full, log: {} needs to be dropped", value);
+ } else {
+ rear = (rear + 1) % max;
+ elements[rear] = value;
+ count++;
+ }
+ }
+
+ public String dequeue() {
+ if (count == 0) {
+ logger.debug("Queue is empty, nothing to get");
+ return null;
+ } else {
+ front = (front + 1) % max;
+ count--;
+ return elements[front];
+ }
+ }
+
+ public List dequeue(int num) {
+ List list = new ArrayList<>();
+ int index = 0;
+ while (index < num) {
+ String tempLog = dequeue();
+ if (StringUtils.isNotEmpty(tempLog)) {
+ list.add(tempLog);
+ } else if (tempLog == null) {
+ break;
+ }
+ index++;
+ }
+ return list;
+ }
+
+ public synchronized List getRemain() {
+ List list = new ArrayList<>();
+ while (!isEmpty()) {
+ list.add(dequeue());
+ }
+ return list;
+ }
+
+ public int size() {
+ return count;
+ }
+ }
+
+
+ private CircularQueue logs;
+
+
+ public MountLogCache(int loopMax) {
+ this.logs = new CircularQueue(loopMax);
+ }
+
+ @Override
+ public void cacheLog(String log) {
+ logs.enqueue(log);
+ }
+
+ @Override
+ public List getLog(int num) {
+ return logs.dequeue(num);
+ }
+
+ @Override
+ public synchronized List getRemain() {
+ return logs.getRemain();
+ }
+
+ @Override
+ public int size() {
+ return logs.size();
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/SendAppender.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/SendAppender.java
new file mode 100644
index 0000000000..ec6069cc7d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/SendAppender.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.log;
+
+import com.webank.wedatasphere.linkis.common.utils.Utils;
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration;
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.TaskLogUpdateEvent;
+import com.webank.wedatasphere.linkis.engineconn.core.EngineConnObject;
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnSyncListenerBus;
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginElement;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+@Plugin(name = "Send", category = "Core", elementType = "appender", printObject = true)
+public class SendAppender extends AbstractAppender {
+
+ /**
+ * @fields serialVersionUID
+ */
+ private static final long serialVersionUID = -830237775522429777L;
+
+ private static EngineConnSyncListenerBus engineConnSyncListenerBus = ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnSyncListenerBus();
+
+ private LogCache logCache;
+ private static final Logger logger = LoggerFactory.getLogger(SendAppender.class);
+
+ private static final String IGNORE_WORDS = AccessibleExecutorConfiguration.ENGINECONN_IGNORE_WORDS().getValue();
+
+ private static final String[] IGNORE_WORD_ARR = IGNORE_WORDS.split(",");
+
+ private static final String PASS_WORDS = AccessibleExecutorConfiguration.ENGINECONN_PASS_WORDS().getValue();
+
+ private static final String[] PASS_WORDS_ARR = PASS_WORDS.split(",");
+
+
+ class SendThread implements Runnable {
+ @Override
+ public void run() {
+ if (engineConnSyncListenerBus == null) {
+ //ignore
+ } else {
+ if (logCache == null) {
+ logger.warn("logCache is null");
+ return;
+ }
+ List logs = logCache.getRemain();
+ if (logs.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ for (String log : logs) {
+ sb.append(log).append("\n");
+ }
+ if (EngineConnObject.isReady()) {
+ engineConnSyncListenerBus.postToAll(new TaskLogUpdateEvent(null, sb.toString()));
+ }
+ }
+ }
+ }
+ }
+
+
+ public SendAppender(final String name, final Filter filter, final Layout layout,
+ final boolean ignoreExceptions) {
+ super(name, filter, layout, ignoreExceptions);
+ this.logCache = LogHelper.logCache();
+ SendThread thread = new SendThread();
+ Utils.defaultScheduler().scheduleAtFixedRate(thread, 10, (Integer) AccessibleExecutorConfiguration.ENGINECONN_LOG_SEND_TIME_INTERVAL().getValue(), TimeUnit.MILLISECONDS);
+ }
+
+
+
+ @Override
+ public void append(LogEvent event) {
+ if (engineConnSyncListenerBus == null) {
+ return;
+ }
+ String logStr = new String(getLayout().toByteArray(event));
+ if (event.getLevel().intLevel() == Level.INFO.intLevel()) {
+ boolean flag = false;
+ for (String ignoreLog : IGNORE_WORD_ARR) {
+ if (logStr.contains(ignoreLog)) {
+ flag = true;
+ break;
+ }
+ }
+ for (String word : PASS_WORDS_ARR) {
+ if (logStr.contains(word)) {
+ flag = false;
+ break;
+ }
+ }
+ if (!flag) {
+ logCache.cacheLog(logStr);
+ }
+ } else {
+ logCache.cacheLog(logStr);
+ }
+ }
+
+ @PluginFactory
+ public static SendAppender createAppender(@PluginAttribute("name") String name,
+ @PluginElement("Filter") final Filter filter,
+ @PluginElement("Layout") Layout layout,
+ @PluginAttribute("ignoreExceptions") boolean ignoreExceptions) {
+ if (name == null) {
+ LOGGER.error("No name provided for SendAppender");
+ return null;
+ }
+ if (layout == null) {
+ layout = PatternLayout.createDefaultLayout();
+ }
+ return new SendAppender(name, filter, layout, ignoreExceptions);
+ }
+
+}
\ No newline at end of file
diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/TimeLogCache.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/TimeLogCache.java
similarity index 84%
rename from ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/TimeLogCache.java
rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/TimeLogCache.java
index 0da73e4a42..68ff59454c 100644
--- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/TimeLogCache.java
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/TimeLogCache.java
@@ -14,17 +14,14 @@
* limitations under the License.
*/
-package com.webank.wedatasphere.linkis.engine.log;
-
-import org.apache.logging.log4j.core.LogEvent;
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.log;
import java.util.List;
/**
- * created by enjoyyin on 2018/11/27
* Description: Cache with time as storage unit（以时间作为存储单位的缓存方式）
*/
-public class TimeLogCache extends AbstractLogCache{
+public class TimeLogCache extends AbstractLogCache {
@Override
public void cacheLog(String log) {
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala
new file mode 100644
index 0000000000..88a9d378e7
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf
+
+import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType}
+
+object AccessibleExecutorConfiguration {
+
+ val ENGINECONN_LOG_CACHE_NUM = CommonVars("wds.linkis.engineconn.log.cache.default", 500)
+
+
+ val ENGINECONN_IGNORE_WORDS = CommonVars("wds.linkis.engineconn.ignore.words", "org.apache.spark.deploy.yarn.Client")
+
+ val ENGINECONN_PASS_WORDS = CommonVars("wds.linkis.engineconn.pass.words", "org.apache.hadoop.hive.ql.exec.Task")
+
+ val ENGINECONN_LOG_NUM_SEND_ONCE = CommonVars("wds.linkis.engineconn.log.send.once", 100)
+
+ val ENGINECONN_LOG_SEND_TIME_INTERVAL = CommonVars("wds.linkis.engineconn.log.send.time.interval", 3)
+
+
+ val ENGINECONN_MAX_FREE_TIME = CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("1h"))
+
+
+ val ENGINECONN_SUPPORT_PARALLELISM = CommonVars("wds.linkis.engineconn.support.parallelism", false)
+
+
+ val ENGINECONN_HEARTBEAT_TIME = CommonVars("wds.linkis.engineconn.heartbeat.time", new TimeType("3m"))
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/conf/AccessibleExecutorSpringConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/conf/AccessibleExecutorSpringConfiguration.scala
new file mode 100644
index 0000000000..2a0f157730
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/conf/AccessibleExecutorSpringConfiguration.scala
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.info.{DefaultNodeOverLoadInfoManager, NodeOverLoadInfoManager}
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.service.{EngineConnConcurrentLockService, EngineConnTimedLockService, LockService}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
+import org.springframework.context.annotation.{Bean, Configuration}
+
+@Configuration
+class AccessibleExecutorSpringConfiguration extends Logging {
+
+ private val asyncListenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus
+
+ @Bean(Array("lockService"))
+ @ConditionalOnMissingBean
+ def createLockManager(): LockService = {
+
+ val lockService = if (AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM.getValue) new EngineConnConcurrentLockService
+ else new EngineConnTimedLockService
+ asyncListenerBusContext.addListener(lockService)
+ lockService
+ }
+
+ @Bean
+ @ConditionalOnMissingBean(Array(classOf[NodeOverLoadInfoManager]))
+ def createNodeOverLoadInfoManager(): NodeOverLoadInfoManager = {
+
+ new DefaultNodeOverLoadInfoManager
+ }
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/entity/AccessibleExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/entity/AccessibleExecutor.scala
new file mode 100644
index 0000000000..277ad5eafb
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/entity/AccessibleExecutor.scala
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.entity
+
+import com.webank.wedatasphere.linkis.common.utils.Utils
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.ExecutorStatusChangedEvent
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.SensibleExecutor
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.scheduler.exception.SchedulerErrorException
+
+abstract class AccessibleExecutor extends SensibleExecutor {
+
+ private var isExecutorClosed = false
+
+ def isIdle = NodeStatus.isIdle(getStatus)
+
+ def isBusy = NodeStatus.isLocked(getStatus)
+
+ def whenBusy[A](f: => A) = whenStatus(NodeStatus.Busy, f)
+
+ def whenIdle[A](f: => A) = whenStatus(NodeStatus.Idle, f)
+
+ def whenStatus[A](_status: NodeStatus, f: => A) = if (getStatus == _status) f
+
+ def ensureBusy[A](f: => A): A = {
+ lastActivityTime = System.currentTimeMillis
+ if (isBusy) synchronized {
+ if (isBusy) return f
+ }
+ throw new SchedulerErrorException(20001, "%s is in status %s." format(toString, getStatus))
+ }
+
+ def ensureIdle[A](f: => A): A = ensureIdle(f, true)
+
+ def ensureIdle[A](f: => A, transitionState: Boolean): A = {
+ if (isIdle) synchronized {
+ if (isIdle) {
+ if (transitionState) transition(NodeStatus.Busy)
+ return Utils.tryFinally(f) {
+ if (transitionState) transition(NodeStatus.Idle)
+ callback()
+ }
+ }
+ }
+ throw new SchedulerErrorException(20001, "%s is in status %s." format(toString, getStatus))
+ }
+
+
+ def ensureAvailable[A](f: => A): A = {
+ if (NodeStatus.isAvailable(getStatus)) synchronized {
+ if (NodeStatus.isAvailable(getStatus)) return Utils.tryFinally(f)(callback())
+ }
+ throw new SchedulerErrorException(20001, "%s is in status %s." format(toString, getStatus))
+ }
+
+ def whenAvailable[A](f: => A): A = {
+ if (NodeStatus.isAvailable(getStatus)) return Utils.tryFinally(f)(callback())
+ throw new SchedulerErrorException(20001, "%s is in status %s." format(toString, getStatus))
+ }
+
+ protected def callback(): Unit
+
+ def supportCallBackLogs(): Boolean
+
+ protected override def onStatusChanged(fromStatus: NodeStatus, toStatus: NodeStatus): Unit = {
+ ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus.post(ExecutorStatusChangedEvent(this, fromStatus, toStatus))
+ }
+
+ override def close(): Unit = {
+ isExecutorClosed = true
+ super.close()
+ }
+
+ override def isClosed(): Boolean = isExecutorClosed
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/entity/ExecutorStatusInfo.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/entity/ExecutorStatusInfo.scala
new file mode 100644
index 0000000000..3633d4039a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/entity/ExecutorStatusInfo.scala
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.entity
+
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.metrics.{NodeHealthyInfo, NodeOverLoadInfo}
+
+import scala.beans.BeanProperty
+
+class ExecutorStatusInfo {
+
+ @BeanProperty
+ var status: NodeStatus = NodeStatus.Starting
+
+ @BeanProperty
+ var healthyInfo: NodeHealthyInfo = null
+
+ @BeanProperty
+ var overLoadInfo: NodeOverLoadInfo = null
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeHealthyInfoManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeHealthyInfoManager.scala
new file mode 100644
index 0000000000..8238ef8d57
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeHealthyInfoManager.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.info
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeHealthy
+import com.webank.wedatasphere.linkis.manager.common.entity.metrics.NodeHealthyInfo
+import org.springframework.stereotype.Component
+
+trait NodeHealthyInfoManager {
+
+ def getNodeHealthyInfo(): NodeHealthyInfo
+
+
+}
+
+
+@Component
+class DefaultNodeHealthyInfoManager extends NodeHealthyInfoManager with Logging {
+
+ override def getNodeHealthyInfo(): NodeHealthyInfo = {
+ val nodeHealthyInfo = new NodeHealthyInfo
+ nodeHealthyInfo.setMsg("")
+ nodeHealthyInfo.setNodeHealthy(NodeHealthy.Healthy)
+ nodeHealthyInfo
+ }
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeHeartbeatMsgManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeHeartbeatMsgManager.scala
new file mode 100644
index 0000000000..f99cd8f836
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeHeartbeatMsgManager.scala
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.info
+
+trait NodeHeartbeatMsgManager {
+
+ def getHeartBeatMsg(): String
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeOverLoadInfoManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeOverLoadInfoManager.scala
new file mode 100644
index 0000000000..ed8b256785
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/info/NodeOverLoadInfoManager.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.info
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, OverloadUtils}
+import com.webank.wedatasphere.linkis.manager.common.entity.metrics.NodeOverLoadInfo
+import org.springframework.stereotype.Component
+
+trait NodeOverLoadInfoManager {
+ // TODO linkis/windows/docker overload
+ def getNodeOverLoadInfo: NodeOverLoadInfo
+
+}
+
+//@ConditionalOnMissingBean(name=Array("nodeOverLoadInfoManager"))
+@Component
+class DefaultNodeOverLoadInfoManager extends NodeOverLoadInfoManager with Logging {
+
+ override def getNodeOverLoadInfo: NodeOverLoadInfo = {
+ val nodeOverLoadInfo = new NodeOverLoadInfo
+ nodeOverLoadInfo.setMaxMemory(OverloadUtils.getProcessMaxMemory)
+ nodeOverLoadInfo.setUsedMemory(OverloadUtils.getProcessUsedMemory)
+ nodeOverLoadInfo.setSystemCPUUsed(OverloadUtils.getSystemCPUUsed)
+ nodeOverLoadInfo.setSystemLeftMemory(OverloadUtils.getSystemFreeMemory)
+ nodeOverLoadInfo
+ }
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/ExecutorLockListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/ExecutorLockListener.scala
new file mode 100644
index 0000000000..b5b6b27546
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/ExecutorLockListener.scala
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener
+
+import com.webank.wedatasphere.linkis.common.listener.Event
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.{ExecutorLockEvent, ExecutorUnLockEvent}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnAsyncListener
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnAsyncEvent
+
+trait ExecutorLockListener extends EngineConnAsyncListener {
+
+ def onAddLock(addLockEvent: ExecutorLockEvent): Unit
+
+ def onReleaseLock(releaseLockEvent: ExecutorUnLockEvent): Unit
+
+ override def onEvent(event: EngineConnAsyncEvent): Unit = event match {
+ case addLockEvent: ExecutorLockEvent => onAddLock(addLockEvent)
+ case releaseLockEvent: ExecutorUnLockEvent => onReleaseLock(releaseLockEvent)
+ case _ =>
+ }
+
+ override def onEventError(event: Event, t: Throwable): Unit = {
+
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/ExecutorStatusListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/ExecutorStatusListener.scala
new file mode 100644
index 0000000000..22139eecb2
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/ExecutorStatusListener.scala
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener
+
+import com.webank.wedatasphere.linkis.common.listener.Event
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.{ExecutorCompletedEvent, ExecutorCreateEvent, ExecutorStatusChangedEvent}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnAsyncListener
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnAsyncEvent
+
+trait ExecutorStatusListener extends EngineConnAsyncListener {
+
+ def onExecutorCreated(executorCreateEvent: ExecutorCreateEvent): Unit
+
+ def onExecutorCompleted(executorCompletedEvent: ExecutorCompletedEvent): Unit
+
+ def onExecutorStatusChanged(executorStatusChangedEvent: ExecutorStatusChangedEvent): Unit
+
+ override def onEvent(event: EngineConnAsyncEvent): Unit = event match {
+ case executorCreateEvent: ExecutorCreateEvent => onExecutorCreated(executorCreateEvent)
+ case executorCompletedEvent: ExecutorCompletedEvent => onExecutorCompleted(executorCompletedEvent)
+ case executorStatusChangedEvent: ExecutorStatusChangedEvent => onExecutorStatusChanged(executorStatusChangedEvent)
+ case _ =>
+ }
+
+ override def onEventError(event: Event, t: Throwable): Unit = {
+
+ }
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/LogListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/LogListener.scala
new file mode 100644
index 0000000000..6d15d9ff65
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/LogListener.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.TaskLogUpdateEvent
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnSyncListener
+
+trait LogListener extends EngineConnSyncListener {
+
+ def onLogUpdate(logUpdateEvent: TaskLogUpdateEvent)
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/NodeHealthyListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/NodeHealthyListener.scala
new file mode 100644
index 0000000000..d6244d49c9
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/NodeHealthyListener.scala
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener
+
+import com.webank.wedatasphere.linkis.common.listener.Event
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.NodeHealthyUpdateEvent
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.EngineConnAsyncListener
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnAsyncEvent
+
+trait NodeHealthyListener extends EngineConnAsyncListener {
+
+
+ def onNodeHealthyUpdate(nodeHealthyUpdateEvent: NodeHealthyUpdateEvent): Unit
+
+ override def onEvent(event: EngineConnAsyncEvent): Unit = event match {
+
+ case nodeHealthyUpdateEvent: NodeHealthyUpdateEvent => onNodeHealthyUpdate(nodeHealthyUpdateEvent)
+ case _ =>
+ }
+
+ override def onEventError(event: Event, t: Throwable): Unit = {
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/AccessibleExecutorConnAsyncEvent.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/AccessibleExecutorConnAsyncEvent.scala
new file mode 100644
index 0000000000..b60c4a4d4b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/AccessibleExecutorConnAsyncEvent.scala
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event
+
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.Executor
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnAsyncEvent
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.entity.metrics.NodeHealthyInfo
+
+trait AccessibleExecutorConnAsyncEvent extends EngineConnAsyncEvent {
+
+}
+
+case class ExecutorCreateEvent(executor: Executor) extends AccessibleExecutorConnAsyncEvent
+
+case class ExecutorCompletedEvent(executor: Executor, message: String) extends AccessibleExecutorConnAsyncEvent
+
+case class ExecutorStatusChangedEvent(executor: Executor, fromStatus: NodeStatus, toStatus: NodeStatus) extends AccessibleExecutorConnAsyncEvent
+
+case class ExecutorLockEvent(executor: Executor, lock: String) extends AccessibleExecutorConnAsyncEvent
+
+case class ExecutorUnLockEvent(executor: Executor, lock: String) extends AccessibleExecutorConnAsyncEvent
+
+case class NodeHealthyUpdateEvent(nodeHealthyInfo: NodeHealthyInfo) extends AccessibleExecutorConnAsyncEvent
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/LogUpdateEvent.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/LogUpdateEvent.scala
new file mode 100644
index 0000000000..5623ed7551
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/LogUpdateEvent.scala
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event
+
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnSyncEvent
+
+case class LogUpdateEvent(taskId: String, log: String) extends EngineConnSyncEvent
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/TaskEvent.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/TaskEvent.scala
new file mode 100644
index 0000000000..92b3d66126
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/listener/event/TaskEvent.scala
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event
+
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnSyncEvent
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus
+import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
+
+trait TaskEvent extends EngineConnSyncEvent {
+
+}
+
+case class TaskProgressUpdateEvent(taskId: String, progress: Float, progressInfo: Array[JobProgressInfo]) extends TaskEvent
+
+case class TaskLogUpdateEvent(taskId: String, log: String) extends TaskEvent
+
+case class TaskStatusChangedEvent(taskId: String, fromStatus: ExecutionNodeStatus, toStatus: ExecutionNodeStatus) extends TaskEvent
+
+case class TaskResultCreateEvent(taskId: String, resStr: String, alias: String) extends TaskEvent
+
+case class TaskResultSizeCreatedEvent(taskId: String, resultSize: Int) extends TaskEvent
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala
new file mode 100644
index 0000000000..ae0b60471b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.lock
+
+import java.util.concurrent.{ScheduledFuture, ScheduledThreadPoolExecutor, Semaphore, TimeUnit}
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.entity.AccessibleExecutor
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.ExecutorUnLockEvent
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+
+class EngineConnTimedLock(private var timeout: Long) extends TimedLock with Logging {
+
+ var lock = new Semaphore(1)
+ val releaseScheduler = new ScheduledThreadPoolExecutor(1)
+ var releaseTask: ScheduledFuture[_] = null
+ var lastLockTime: Long = 0
+ var lockedBy: AccessibleExecutor = null
+
+ override def acquire(executor: AccessibleExecutor): Unit = {
+ lock.acquire()
+ lastLockTime = System.currentTimeMillis()
+ lockedBy = executor
+ scheduleTimeout
+ }
+
+ override def tryAcquire(executor: AccessibleExecutor): Boolean = {
+ if (null == executor) return false
+ val succeed = lock.tryAcquire()
+ debug("try to lock for succeed is " + succeed.toString)
+ if (succeed) {
+ lastLockTime = System.currentTimeMillis()
+ lockedBy = executor
+ debug("try to lock for add time out task ! Locked by thread : " + lockedBy.getId)
+ scheduleTimeout
+ }
+ succeed
+ }
+
+ // Unlock callback is not called in release method, because release method is called actively
+ override def release(): Unit = {
+ debug("try to release for lock," + lockedBy + ",current thread " + Thread.currentThread().getName)
+ if (lockedBy != null) {
+ //&& lockedBy == Thread.currentThread() Inconsistent thread(线程不一致)
+ debug("try to release for lockedBy and thread ")
+ if (releaseTask != null) {
+ releaseTask.cancel(true)
+ releaseTask = null
+ releaseScheduler.purge()
+ }
+ debug("try to release for lock release success")
+ lockedBy = null
+ }
+ resetLock()
+ }
+
+ private def resetLock(): Unit = {
+ lock.release()
+ lock = new Semaphore(1)
+ }
+
+ override def forceRelease(): Unit = {
+ if (isAcquired()) {
+ if (releaseTask != null) {
+ releaseTask.cancel(true)
+ releaseTask = null
+ releaseScheduler.purge()
+ }
+ lock.release()
+ lockedBy = null
+ }
+ resetLock()
+ }
+
+ private def scheduleTimeout: Unit = {
+ synchronized {
+ if (null == releaseTask || releaseTask.isDone) {
+ releaseTask = releaseScheduler.schedule(new Runnable {
+ override def run(): Unit = {
+ synchronized {
+ if (isExpired()) {
+ lock.release()
+ // unlockCallback depends on lockedBy, so lockedBy cannot be set null before unlockCallback
+ unlockCallback(lock.toString)
+ info(s"Lock : [${lock.toString} was released due to timeout." )
+ resetLock()
+ }
+ }
+ }
+ }, timeout, TimeUnit.MILLISECONDS)
+ }
+ }
+ }
+
+ override def isAcquired(): Boolean = {
+ lock.availablePermits() < 1
+ }
+
+ override def isExpired(): Boolean = {
+ if (lastLockTime == 0) return false
+ System.currentTimeMillis() - lastLockTime > timeout
+ }
+
+
+ override def numOfPending(): Int = {
+ lock.getQueueLength
+ }
+
+ override def renew(): Boolean = {
+ if (lockedBy != null && lockedBy == Thread.currentThread()) {
+ if (isAcquired && releaseTask != null) {
+ if (releaseTask.cancel(false)) {
+ releaseScheduler.purge()
+ scheduleTimeout
+ lastLockTime = System.currentTimeMillis()
+ return true
+ }
+ }
+ }
+ false
+ }
+
+ override def resetTimeout(timeout: Long): Unit = synchronized {
+ if (isAcquired()) {
+ if (null != releaseTask && !isExpired()) {
+ releaseTask.cancel(true)
+ this.timeout = timeout
+ }
+ scheduleTimeout
+ } else {
+ error("Lock is not acquired, so cannot be reset-Timeout")
+ }
+ }
+
+ private def unlockCallback(lockStr: String): Unit = {
+ if (null != lockedBy) {
+ lockedBy.transition(NodeStatus.Unlock)
+ }
+ ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus.post(ExecutorUnLockEvent(null, lockStr.toString))
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/lock/TimedLock.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/lock/TimedLock.scala
new file mode 100644
index 0000000000..8ecdb30ffd
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/lock/TimedLock.scala
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.lock
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.entity.AccessibleExecutor
+
+trait TimedLock {
+
+ @throws[Exception]
+ def acquire(executor: AccessibleExecutor): Unit
+
+ def tryAcquire(executor: AccessibleExecutor): Boolean
+
+ @throws[Exception]
+ def release(): Unit
+
+ @throws[Exception]
+ def forceRelease(): Unit
+
+ def numOfPending(): Int
+
+ def isAcquired(): Boolean
+
+ def isExpired(): Boolean
+
+ @throws[Exception]
+ def renew(): Boolean
+
+ def resetTimeout(timeout: Long): Unit
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/LogHelper.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/LogHelper.scala
new file mode 100644
index 0000000000..e6590c35c1
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/log/LogHelper.scala
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.log
+
+import java.util
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.LogListener
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.TaskLogUpdateEvent
+import org.slf4j.LoggerFactory
+
+object LogHelper {
+ private val logger = LoggerFactory.getLogger(getClass)
+ val logCache = new MountLogCache(AccessibleExecutorConfiguration.ENGINECONN_LOG_CACHE_NUM.getValue)
+
+ private var logListener: LogListener = _
+
+ def setLogListener(logListener: LogListener): Unit = this.logListener = logListener
+
+ def pushAllRemainLogs(): Unit = {
+// logger.info(s"start to push all remain logs")
+ Thread.sleep(30)
+ //logCache.synchronized{
+ if (logListener == null) {
+ logger.warn("logListener is null, can not push remain logs")
+ //return
+ } else {
+ var logs: util.List[String] = null
+ logCache.synchronized {
+ logs = logCache.getRemain
+ }
+ if (logs != null && logs.size > 0) {
+ val sb: StringBuilder = new StringBuilder
+ import scala.collection.JavaConversions._
+ logs map (log => log + "\n") foreach sb.append
+ logger.info(s"remain logs is ${sb.toString()}")
+ logListener.onLogUpdate(TaskLogUpdateEvent(null, sb.toString))
+ }
+ }
+ logger.info("end to push all remain logs")
+ // }
+ // if (sendAppender == null){
+ // logger.error("SendAppender has not been initialized")
+ // }else{
+ // val logCache = sendAppender.getLogCache
+ // val logListener = SendAppender.getLogListener
+ // logCache.synchronized{
+ // val logs: util.List[String] = logCache.getRemain
+ // if (logs.size > 0) {
+ // val sb: StringBuilder = new StringBuilder
+ // import scala.collection.JavaConversions._
+ // logs map (log => log + "\n") foreach sb.append
+ // logListener.onLogUpdate(null, sb.toString)
+ // }
+ // }
+ // }
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/AccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/AccessibleService.scala
new file mode 100644
index 0000000000..66bb93b6fa
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/AccessibleService.scala
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.ExecutorStatusListener
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineSuicideRequest
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{RequestNodeStatus, ResponseNodeStatus}
+import com.webank.wedatasphere.linkis.message.builder.ServiceMethodContext
+
+trait AccessibleService extends ExecutorStatusListener {
+
+ def stopExecutor: Unit
+
+ def pauseExecutor: Unit
+
+ def reStartExecutor: Boolean
+
+ def dealEngineStopRequest(engineSuicideRequest: EngineSuicideRequest, smc: ServiceMethodContext): Unit
+
+
+ /**
+ * service 需要加定时任务判断Executor是否空闲很久，然后调用该方法进行释放
+ */
+ def requestManagerReleaseExecutor(msg: String): Unit
+
+
+ def dealRequestNodeStatus(requestNodeStatus: RequestNodeStatus): ResponseNodeStatus
+
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala
new file mode 100644
index 0000000000..adf6dc19ad
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.entity.AccessibleExecutor
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.{ExecutorCompletedEvent, ExecutorCreateEvent, ExecutorStatusChangedEvent}
+import com.webank.wedatasphere.linkis.engineconn.core.EngineConnObject
+import com.webank.wedatasphere.linkis.engineconn.core.engineconn.EngineConnManager
+import com.webank.wedatasphere.linkis.engineconn.core.executor.ExecutorManager
+import com.webank.wedatasphere.linkis.engineconn.core.hook.ShutdownHook
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.{Executor, SensibleExecutor}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.engineconn.executor.service.ManagerService
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.{EngineConnReleaseRequest, EngineSuicideRequest}
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{RequestNodeStatus, ResponseNodeStatus}
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.message.builder.ServiceMethodContext
+import com.webank.wedatasphere.linkis.rpc.Sender
+import javax.annotation.PostConstruct
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Service
+
+import scala.collection.JavaConverters._
+
+
+@Service
+class DefaultAccessibleService extends AccessibleService with Logging {
+
+ @Autowired
+ private var executorHeartbeatService: ExecutorHeartbeatService = _
+
+ private val asyncListenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext.getEngineConnAsyncListenerBus
+
+ @Receiver
+ override def dealEngineStopRequest(engineSuicideRequest: EngineSuicideRequest, smc: ServiceMethodContext): Unit = {
+ // todo check user
+ if (DataWorkCloudApplication.getServiceInstance.equals(engineSuicideRequest.getServiceInstance)) {
+ stopEngine()
+ info(s"engine will suiside now.")
+ ShutdownHook.getShutdownHook.notifyStop()
+ } else {
+ if (null != engineSuicideRequest.getServiceInstance) {
+ error(s"Invalid serviceInstance : ${engineSuicideRequest.getServiceInstance.toString}, will not suicide.")
+ } else {
+ error("Invalid empty serviceInstance.")
+ }
+ }
+ }
+
+ private def executorShutDownHook(): Unit = {
+ var executor: Executor = ExecutorManager.getInstance().getDefaultExecutor
+ if (null != executor) {
+ executor.asInstanceOf[SensibleExecutor].transition(NodeStatus.ShuttingDown)
+ } else {
+ executor = SensibleExecutor.getDefaultErrorSensibleExecutor()
+ }
+ executorHeartbeatService.reportHeartBeatMsg(executor)
+ info("Reported status shuttingDown to manager.")
+ }
+
+ override def stopExecutor: Unit = {
+ // todo
+ }
+
+
+ override def pauseExecutor: Unit = {
+
+ }
+
+ override def reStartExecutor: Boolean = {
+ true
+ }
+
+ /**
+ * Service启动后则启动定时任务 空闲释放
+ */
+ @PostConstruct
+ private def init(): Unit = {
+ val context = EngineConnObject.getEngineCreationContext
+ val maxFreeTimeVar = AccessibleExecutorConfiguration.ENGINECONN_MAX_FREE_TIME.getValue(context.getOptions)
+ val maxFreeTimeStr = maxFreeTimeVar.toString
+ val maxFreeTime = maxFreeTimeVar.toLong
+ Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
+ override def run(): Unit = Utils.tryAndWarn {
+ val executor = ExecutorManager.getInstance().getDefaultExecutor
+ if (null == executor || !executor.isInstanceOf[AccessibleExecutor]) {
+ warn("Default is not AccessibleExecutor,do noting")
+ return
+ }
+ info("maxFreeTimeMills is " + maxFreeTime)
+ val accessibleExecutor = ExecutorManager.getInstance().getDefaultExecutor.asInstanceOf[AccessibleExecutor]
+ if (NodeStatus.isCompleted(accessibleExecutor.getStatus)) {
+ error(s"${accessibleExecutor.getId()} has completed with status ${accessibleExecutor.getStatus}, now stop it.")
+ ShutdownHook.getShutdownHook.notifyStop()
+ } else if (accessibleExecutor.getStatus == NodeStatus.ShuttingDown) {
+ warn(s"${accessibleExecutor.getId()} is ShuttingDown...")
+ ShutdownHook.getShutdownHook.notifyStop()
+ } else if (maxFreeTime > 0 && NodeStatus.isIdle(accessibleExecutor.getStatus) && System.currentTimeMillis - accessibleExecutor.getLastActivityTime > maxFreeTime) {
+ warn(s"${accessibleExecutor.getId()} has not been used for $maxFreeTimeStr, now try to shutdown it.")
+ requestManagerReleaseExecutor(" idle release")
+ }
+ }
+ }, 10 * 60 * 1000, AccessibleExecutorConfiguration.ENGINECONN_HEARTBEAT_TIME.getValue.toLong, TimeUnit.MILLISECONDS)
+ asyncListenerBusContext.addListener(this)
+ Utils.addShutdownHook(executorShutDownHook())
+ }
+
+ private def stopEngine(): Unit = {
+ Utils.tryAndWarn {
+ ExecutorManager.getInstance().getDefaultExecutor.asInstanceOf[SensibleExecutor].transition(NodeStatus.ShuttingDown)
+ val executors = ExecutorManager.getInstance().getAllExecutorsMap()
+ executors.asScala.map{
+ kv => kv._2.tryShutdown()
+ }
+ executors.clear()
+ }
+ }
+
+ /**
+ * service 需要加定时任务判断Executor是否空闲很久，然后调用该方法进行释放
+ */
+
+ override def requestManagerReleaseExecutor(msg: String): Unit = {
+ val engineReleaseRequest = new EngineConnReleaseRequest(Sender.getThisServiceInstance, Utils.getJvmUser, msg, EngineConnManager.getEngineConnManager.getEngineConn().getEngineCreationContext.getTicketId)
+ ManagerService.getManagerService.requestReleaseEngineConn(engineReleaseRequest)
+ }
+
+ @Receiver
+ override def dealRequestNodeStatus(requestNodeStatus: RequestNodeStatus): ResponseNodeStatus = {
+ val status = ExecutorManager.getInstance().getDefaultExecutor match {
+ case executor: SensibleExecutor =>
+ executor.getStatus
+ case _ => NodeStatus.Starting
+ }
+ val responseNodeStatus = new ResponseNodeStatus
+ responseNodeStatus.setNodeStatus(status)
+ responseNodeStatus
+ }
+
+ override def onExecutorCreated(executorCreateEvent: ExecutorCreateEvent): Unit = {
+
+ }
+
+ override def onExecutorCompleted(executorCompletedEvent: ExecutorCompletedEvent): Unit = {
+
+ }
+
+ override def onExecutorStatusChanged(executorStatusChangedEvent: ExecutorStatusChangedEvent): Unit = {
+ if (!ExecutorManager.getInstance().getDefaultExecutor.isInstanceOf[AccessibleExecutor]) {
+ warn("Default is not AccessibleExecutor,do noting")
+ return
+ }
+ val executor = ExecutorManager.getInstance().getDefaultExecutor.asInstanceOf[AccessibleExecutor]
+ executorHeartbeatService.reportHeartBeatMsg(executor)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala
new file mode 100644
index 0000000000..f135cae735
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.info.{NodeHealthyInfoManager, NodeHeartbeatMsgManager, NodeOverLoadInfoManager}
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.NodeHealthyListener
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.NodeHealthyUpdateEvent
+import com.webank.wedatasphere.linkis.engineconn.core.executor.ExecutorManager
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.{Executor, ResourceExecutor, SensibleExecutor}
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.engineconn.executor.service.ManagerService
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHeartbeatRequest}
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.rpc.Sender
+import javax.annotation.PostConstruct
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Service
+
+@Service
+class DefaultExecutorHeartbeatService extends ExecutorHeartbeatService with NodeHealthyListener with Logging {
+
+
+ @Autowired
+ private var nodeHealthyInfoManager: NodeHealthyInfoManager = _
+
+ @Autowired
+ private var nodeOverLoadInfoManager: NodeOverLoadInfoManager = _
+
+ @Autowired(required = false)
+ private var nodeHeartbeatMsgManager: NodeHeartbeatMsgManager = _
+
+ private val asyncListenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext.getEngineConnAsyncListenerBus
+
+ @PostConstruct
+ private def init(): Unit = {
+ asyncListenerBusContext.addListener(this)
+ val heartbeatTime = AccessibleExecutorConfiguration.ENGINECONN_HEARTBEAT_TIME.getValue.toLong
+ Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
+ override def run(): Unit = Utils.tryAndWarn {
+ val executor = ExecutorManager.getInstance().getDefaultExecutor
+ reportHeartBeatMsg(executor)
+ }
+ }, 3 * 60 * 1000, heartbeatTime, TimeUnit.MILLISECONDS)
+ }
+
+ /**
+ * 定时上报心跳信息，依据Executor不同进行实现
+ *
+ * @param executor
+ */
+ override def reportHeartBeatMsg(executor: Executor): Unit = {
+ ManagerService.getManagerService.heartbeatReport(generateHeartBeatMsg(executor))
+ }
+
+
+ @Receiver
+ override def dealNodeHeartbeatRequest(nodeHeartbeatRequest: NodeHeartbeatRequest): NodeHeartbeatMsg = {
+ val executor = ExecutorManager.getInstance().getDefaultExecutor
+ generateHeartBeatMsg(executor)
+ }
+
+
+ override def onNodeHealthyUpdate(nodeHealthyUpdateEvent: NodeHealthyUpdateEvent): Unit = {
+ warn(s"node healthy update, tiger heartbeatReport")
+ val executor = ExecutorManager.getInstance().getDefaultExecutor
+ reportHeartBeatMsg(executor)
+ }
+
+ override def generateHeartBeatMsg(executor: Executor): NodeHeartbeatMsg = {
+ val nodeHeartbeatMsg = new NodeHeartbeatMsg
+
+ nodeHeartbeatMsg.setServiceInstance(Sender.getThisServiceInstance)
+ nodeHeartbeatMsg.setOverLoadInfo(nodeOverLoadInfoManager.getNodeOverLoadInfo)
+ nodeHeartbeatMsg.setHealthyInfo(nodeHealthyInfoManager.getNodeHealthyInfo())
+ executor match {
+ case sensibleExecutor: SensibleExecutor =>
+ nodeHeartbeatMsg.setStatus(sensibleExecutor.getStatus)
+ case _ =>
+ }
+ executor match {
+ case resourceExecutor: ResourceExecutor =>
+ nodeHeartbeatMsg.setNodeResource(resourceExecutor.getCurrentNodeResource())
+ case _ =>
+ }
+ if (null != nodeHeartbeatMsgManager) {
+ nodeHeartbeatMsg.setHeartBeatMsg(nodeHeartbeatMsgManager.getHeartBeatMsg())
+ }
+ nodeHeartbeatMsg
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala
new file mode 100644
index 0000000000..7500d9f22b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import java.util
+
+import com.webank.wedatasphere.linkis.engineconn.executor.service.ManagerService
+import com.webank.wedatasphere.linkis.governance.common.conf.GovernanceCommonConf
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineConnReleaseRequest
+import com.webank.wedatasphere.linkis.manager.common.protocol.label.LabelReportRequest
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, ResponseNodeStatus}
+import com.webank.wedatasphere.linkis.manager.common.protocol.resource.ResourceUsedProtocol
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.rpc.Sender
+
+class DefaultManagerService extends ManagerService {
+
+
+ protected def getManagerSender: Sender = {
+ Sender.getSender(GovernanceCommonConf.MANAGER_SPRING_NAME.getValue)
+ }
+
+ protected def getEngineConnManagerSender: Sender = {
+ Sender.getSender(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue)
+ }
+
+
+ override def labelReport(labels: util.List[Label[_]]): Unit = {
+ val labelReportRequest = LabelReportRequest(labels, Sender.getThisServiceInstance)
+ getManagerSender.send(labelReportRequest)
+ }
+
+
+ override def statusReport(status: NodeStatus): Unit = {
+ val responseNodeStatus = new ResponseNodeStatus
+ responseNodeStatus.setNodeStatus(status)
+ getManagerSender.send(responseNodeStatus)
+ }
+
+ override def requestReleaseEngineConn(engineConnReleaseRequest: EngineConnReleaseRequest): Unit = {
+ getManagerSender.send(engineConnReleaseRequest)
+ }
+
+ override def heartbeatReport(nodeHeartbeatMsg: NodeHeartbeatMsg): Unit = {
+ getManagerSender.send(nodeHeartbeatMsg)
+ }
+
+ override def reportUsedResource(resourceUsedProtocol: ResourceUsedProtocol): Unit = {
+ getManagerSender.send(resourceUsedProtocol)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
new file mode 100644
index 0000000000..e12c0650ce
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.entity.AccessibleExecutor
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.event.{ExecutorLockEvent, ExecutorUnLockEvent}
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.lock.EngineConnTimedLock
+import com.webank.wedatasphere.linkis.engineconn.core.executor.ExecutorManager
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.ExecutorListenerBusContext
+import com.webank.wedatasphere.linkis.governance.common.exception.engineconn.{EngineConnExecutorErrorCode, EngineConnExecutorErrorException}
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineLockType
+import com.webank.wedatasphere.linkis.manager.common.protocol.{RequestEngineLock, RequestEngineUnlock, ResponseEngineLock, ResponseEngineUnlock}
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
+import org.apache.commons.lang.StringUtils
+
+
+class EngineConnTimedLockService extends LockService with Logging {
+
+ private var engineConnLock: EngineConnTimedLock = null
+ private var lockString: String = null
+ private var lockType: EngineLockType = EngineLockType.Timed
+
+
+ private def isSupportParallelism: Boolean = AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM.getValue
+
+ /**
+ *
+ *
+ * @param lock
+ * @return
+ */
+ override def isLockExist(lock: String = null): Boolean = synchronized {
+ if (isSupportParallelism) true
+ else if (null != engineConnLock && engineConnLock.isAcquired()) {
+ if (StringUtils.isNotBlank(lock)) {
+ if (lock.equals(lockString)) true
+ else false
+ } else {
+ false
+ }
+ } else {
+ false
+ }
+
+ }
+
+ /**
+ * Try to lock an Executor in the ExecutorManager. If the lock is successful, it will return the Executor ID as the ID.
+ * 尝试去锁住ExecutorManager里的一个Executor，如果锁成功的话，将返回Executor ID作为标识
+ *
+ * @return
+ */
+ @throws[EngineConnExecutorErrorException]
+ override def tryLock(requestEngineLock: RequestEngineLock): Option[String] = synchronized {
+ if (null != engineConnLock && engineConnLock.isAcquired()) return None
+ this.lockType = requestEngineLock.lockType
+ lockType match {
+ case EngineLockType.Always =>
+ timedLock(-1)
+ case EngineLockType.Timed =>
+ timedLock(requestEngineLock.timeout)
+ case o: Any =>
+ error("Invalid lockType : " + BDPJettyServerHelper.gson.toJson(o))
+ return Some(null)
+ }
+
+ }
+
+ private def timedLock(timeout: Long): Option[String] = {
+
+ // Lock is binded to engineconn, so choose default executor
+ ExecutorManager.getInstance().getDefaultExecutor match {
+ case accessibleExecutor: AccessibleExecutor =>
+ debug("try to lock for executor state is " + accessibleExecutor.getStatus)
+ debug("try to lock for executor id is " + accessibleExecutor.getId)
+ if (null == engineConnLock) {
+ engineConnLock = new EngineConnTimedLock(timeout)
+ debug("try to lock for executor get new lock " + engineConnLock)
+ }
+ if (engineConnLock.tryAcquire(accessibleExecutor)) {
+ debug("try to lock for tryAcquire is true ")
+ this.lockString = engineConnLock.lock.toString
+ ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus.post(ExecutorLockEvent(accessibleExecutor, lockString))
+ accessibleExecutor.transition(NodeStatus.Idle)
+ Some(lockString)
+ } else None
+ case _ =>
+ val msg = s"Invalid executor or not instance of SensibleEngine."
+ error(msg)
+ throw new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_ENGINE_TYPE, msg)
+ }
+ }
+
+ /**
+ * Unlock(解锁)
+ *
+ * @param lock
+ */
+ override def unlock(lock: String): Boolean = synchronized {
+ info("try to unlock for lockEntity is " + engineConnLock.toString + ",and lock is " + lock + ",acquired is " + engineConnLock.isAcquired().toString)
+ if (isLockExist(lock)) {
+ info(s"try to unlock lockEntity : lockString=${lockString},lockedBy=${engineConnLock.lockedBy.getId()}")
+ engineConnLock.release()
+ this.lockString = null
+ ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus.post(ExecutorUnLockEvent(null, lock))
+ ExecutorManager.getInstance().getDefaultExecutor match {
+ case accessibleExecutor: AccessibleExecutor =>
+ accessibleExecutor.transition(NodeStatus.Unlock)
+ case _ =>
+ val msg = s"Invalid executor or not instance of SensibleEngine."
+ error(msg)
+ }
+ true
+ } else {
+ false
+ }
+ }
+
+ @Receiver
+ override def requestUnLock(requestEngineUnlock: RequestEngineUnlock): ResponseEngineUnlock = {
+ if (StringUtils.isBlank(requestEngineUnlock.lock)) {
+ error("Invalid requestEngineUnlock: ")
+ ResponseEngineUnlock(false)
+ } else {
+ ResponseEngineUnlock(unlock(requestEngineUnlock.lock))
+ }
+ }
+
+ override def onAddLock(addLockEvent: ExecutorLockEvent): Unit = {
+
+ }
+
+ override def onReleaseLock(releaseLockEvent: ExecutorUnLockEvent): Unit = {
+
+ }
+
+
+ @Receiver
+ override def requestLock(requestEngineLock: RequestEngineLock): ResponseEngineLock = {
+ super.requestLock(requestEngineLock)
+ }
+}
+
+class EngineConnConcurrentLockService extends LockService {
+
+ override def isLockExist(lock: String): Boolean = true
+
+ override def tryLock(requestEngineLock: RequestEngineLock): Option[String] = {
+ /*ExecutorManager.getInstance().getDefaultExecutor match {
+ case accessibleExecutor: AccessibleExecutor =>
+ // Concurrent Engine don't change status when get lock, status will change in other rules
+// accessibleExecutor.transition(NodeStatus.Idle)
+ case _ =>
+ }*/
+ Some("lock")
+ }
+
+ /**
+ * Unlock(解锁)
+ *
+ * @param lock
+ */
+ override def unlock(lock: String): Boolean = {
+ /*ExecutorManager.getInstance().getDefaultExecutor match {
+ case accessibleExecutor: AccessibleExecutor =>
+ accessibleExecutor.transition(NodeStatus.Unlock)
+ case _ =>
+ }*/
+ true
+ }
+
+ @Receiver
+ override def requestUnLock(requestEngineUnlock: RequestEngineUnlock): ResponseEngineUnlock = ResponseEngineUnlock(true)
+
+ override def onAddLock(addLockEvent: ExecutorLockEvent): Unit = {}
+
+ override def onReleaseLock(releaseLockEvent: ExecutorUnLockEvent): Unit = {}
+
+ @Receiver
+ override def requestLock(requestEngineLock: RequestEngineLock): ResponseEngineLock = {
+ super.requestLock(requestEngineLock)
+ }
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/ExecutorHeartbeatService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/ExecutorHeartbeatService.scala
new file mode 100644
index 0000000000..21a16cc922
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/ExecutorHeartbeatService.scala
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.Executor
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHeartbeatRequest}
+
+trait ExecutorHeartbeatService {
+
+ /**
+ * 定时上报心跳信息，依据Executor不同进行实现
+ *
+ * @param executor
+ */
+ def reportHeartBeatMsg(executor: Executor): Unit
+
+ def generateHeartBeatMsg(executor: Executor): NodeHeartbeatMsg
+
+
+ def dealNodeHeartbeatRequest(nodeHeartbeatRequest: NodeHeartbeatRequest): NodeHeartbeatMsg
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/LockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/LockService.scala
new file mode 100644
index 0000000000..9f82f1345e
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/LockService.scala
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.listener.ExecutorLockListener
+import com.webank.wedatasphere.linkis.manager.common.protocol.{RequestEngineLock, RequestEngineUnlock, ResponseEngineLock, ResponseEngineUnlock}
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
+import org.apache.commons.lang.StringUtils
+
+trait LockService extends ExecutorLockListener with Logging {
+
+ def isLockExist(lock: String): Boolean
+
+ def tryLock(requestEngineLock: RequestEngineLock): Option[String]
+
+ /**
+ * Unlock(解锁)
+ *
+ * @param lock
+ */
+ def unlock(lock: String): Boolean
+
+ def requestLock(requestEngineLock: RequestEngineLock): ResponseEngineLock = {
+ var response: ResponseEngineLock = null
+ tryLock(requestEngineLock) match {
+ case Some(lockStr) =>
+ // Engine can be locked
+ if (!StringUtils.isBlank(lockStr)) {
+ // lock success
+ response = ResponseEngineLock(true, lockStr, s"Lock for ${requestEngineLock.timeout} ms")
+ } else {
+ // lock failed
+ response = ResponseEngineLock(false, lockStr, "lock str is blank")
+ }
+ case None =>
+ // Engine is busy
+ response = ResponseEngineLock(false, null, "Engine is busy.")
+ }
+ info ("RequestLock : " + BDPJettyServerHelper.gson.toJson(requestEngineLock) + "\nResponseLock : " + BDPJettyServerHelper.gson.toJson(response))
+ response
+ }
+
+ def requestUnLock(requestEngineUnlock: RequestEngineUnlock): ResponseEngineUnlock
+
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/LogService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/LogService.scala
new file mode 100644
index 0000000000..18e2e84969
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/service/LogService.scala
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.service
+
+import com.webank.wedatasphere.linkis.engineconn.acessible.executor.log.LogCache
+
+trait LogService {
+
+
+ def getLogCache(): LogCache
+
+ def sendLog(): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/utils/AccessableExecutorUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/utils/AccessableExecutorUtils.scala
new file mode 100644
index 0000000000..9c7e4b4654
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/com/webank/wedatasphere/linkis/engineconn/acessible/executor/utils/AccessableExecutorUtils.scala
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.acessible.executor.utils
+
+object AccessibleExecutorUtils {
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/pom.xml
new file mode 100644
index 0000000000..bff3c83f7d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/pom.xml
@@ -0,0 +1,83 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+
+ 4.0.0
+
+ linkis-callback-service
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-executor-core
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-message-scheduler
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-core
+ ${linkis.version}
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala
new file mode 100644
index 0000000000..2d4cd3f14b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.callback.service
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+
+class EngineConnAfterStartCallback(emInstance: ServiceInstance) extends AbstractEngineConnStartUpCallback(emInstance) {
+
+ override def callback(): Unit = {
+
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnCallback.scala
new file mode 100644
index 0000000000..a19be12c8a
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnCallback.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.callback.service
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.protocol.message.RequestProtocol
+import com.webank.wedatasphere.linkis.rpc.Sender
+
+trait EngineConnCallback {
+
+ protected def getEMSender: Sender
+
+ def callback(): Unit
+
+}
+
+abstract class AbstractEngineConnStartUpCallback(emInstance: ServiceInstance) extends EngineConnCallback {
+
+ override protected def getEMSender: Sender = {
+ Sender.getSender(emInstance)
+ }
+
+
+ def callback(protocol: RequestProtocol): Unit = {
+ getEMSender.send(protocol)
+ }
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnPidCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnPidCallback.scala
new file mode 100644
index 0000000000..31f8a3a2d1
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnPidCallback.scala
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.callback.service
+
+import java.lang.management.ManagementFactory
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.engineconn.core.EngineConnObject
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.ResponseEngineConnPid
+import com.webank.wedatasphere.linkis.rpc.Sender
+
+
+class EngineConnPidCallback(emInstance: ServiceInstance) extends AbstractEngineConnStartUpCallback(emInstance) {
+
+ override def callback(): Unit = {
+ val pid = ManagementFactory.getRuntimeMXBean.getName.split("@")(0)
+ val instance = Sender.getThisServiceInstance
+ val context = EngineConnObject.getEngineCreationContext
+ callback(ResponseEngineConnPid(instance, pid, context.getTicketId))
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnTimedCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnTimedCallback.scala
new file mode 100644
index 0000000000..318615644f
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/com/webank/wedatasphere/linkis/engineconn/callback/service/EngineConnTimedCallback.scala
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.callback.service
+
+/**
+ * 常规callBack
+ */
+trait EngineConnTimedCallback extends EngineConnCallback {
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/pom.xml
new file mode 100644
index 0000000000..a2f59278e9
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/pom.xml
@@ -0,0 +1,89 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+
+ 4.0.0
+
+ linkis-executor-core
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-common
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-protocol
+ ${linkis.version}
+ provided
+
+
+ com.webank.wedatasphere.linkis
+ linkis-manager-common
+ ${linkis.version}
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-common
+ ${linkis.version}
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/conf/EngineConnExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/conf/EngineConnExecutorConfiguration.scala
new file mode 100644
index 0000000000..88ab804753
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/conf/EngineConnExecutorConfiguration.scala
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.conf
+
+import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType}
+
+object EngineConnExecutorConfiguration {
+
+
+ val TMP_PATH = CommonVars("wds.linkis.dataworkclod.engine.tmp.path", "file:///tmp/")
+
+ val ENGINE_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.engine.application.name", "")
+
+ val ENGINE_LOG_PREFIX = CommonVars("wds.linkis.log4j2.prefix", "/appcom/logs/linkis/" + ENGINE_SPRING_APPLICATION_NAME.getValue)
+
+ val CLEAR_LOG = CommonVars("wds.linkis.log.clear", false)
+
+
+ val ENGINE_IGNORE_WORDS = CommonVars("wds.linkis.engine.ignore.words", "org.apache.spark.deploy.yarn.Client")
+
+ val ENGINE_PASS_WORDS = CommonVars("wds.linkis.engine.pass.words", "org.apache.hadoop.hive.ql.exec.Task")
+
+
+
+
+ val ENTRANCE_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.entrance.application.name", "linkis-cg-entrance")
+
+ val ENGINE_SERVER_LISTENER_ASYNC_QUEUE_CAPACITY = CommonVars("wds.linkis.engine.listener.async.queue.size.max", 300)
+
+ val ENGINE_SERVER_LISTENER_ASYNC_CONSUMER_THREAD_MAX = CommonVars("wds.linkis.engine.listener.async.consumer.thread.max", 5)
+
+ val ENGINE_SERVER_LISTENER_ASYNC_CONSUMER_THREAD_FREE_TIME_MAX = CommonVars("wds.linkis.engine.listener.async.consumer.freetime.max", new TimeType("5000ms"))
+
+ // todo better to rename
+ val EXECUTOR_MANAGER_SERVICE_CLAZZ = CommonVars("wds.linkis.engineconn.executor.manager.service.clazz", "com.webank.wedatasphere.linkis.engineconn.acessible.executor.service.DefaultManagerService")
+
+ /*val EXECUTOR_MANAGER_CLAZZ = CommonVars("wds.linkis.engineconn.executor.manager.claazz", "")*/
+
+ val DEFAULT_EXECUTOR_NAME = CommonVars("wds.linkis.engineconn.executor.default.name", "ComputationExecutor")
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/Executor.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/Executor.scala
new file mode 100644
index 0000000000..f9e0619b97
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/Executor.scala
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.entity
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+
+trait Executor extends Logging {
+
+ def getId(): String
+
+ def init(): Unit
+
+ def tryReady(): Boolean
+
+ def tryShutdown(): Boolean
+
+ def tryFailed(): Boolean
+
+ /**
+ * 仅用于Kill Executor
+ * EngineConn kill 在AccessibleService
+ */
+ def close(): Unit = {
+ warn(s"executor ${getId()} exit by close")
+ }
+
+ def isClosed(): Boolean
+
+}
+
+trait ConcurrentExecutor extends Executor {
+
+ def getConcurrentLimit: Int
+
+ def killAll(): Unit
+
+}
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/LabelExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/LabelExecutor.scala
new file mode 100644
index 0000000000..e95bdd6cb9
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/LabelExecutor.scala
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.entity
+
+import java.util
+
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+trait LabelExecutor extends Executor {
+
+ def getExecutorLabels(): util.List[Label[_]]
+
+ def setExecutorLabels(labels: util.List[Label[_]]): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/ResourceExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/ResourceExecutor.scala
new file mode 100644
index 0000000000..8ec322dab6
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/ResourceExecutor.scala
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.entity
+
+import com.webank.wedatasphere.linkis.manager.common.entity.resource.NodeResource
+
+trait ResourceExecutor extends Executor {
+
+
+ def requestExpectedResource(expectedResource: NodeResource): NodeResource
+
+ def getCurrentNodeResource(): NodeResource
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/SensibleExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/SensibleExecutor.scala
new file mode 100644
index 0000000000..83ee4d39c1
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/SensibleExecutor.scala
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.entity
+
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+
+import scala.beans.BeanProperty
+
+trait SensibleExecutor extends Executor {
+
+ protected var status: NodeStatus = NodeStatus.Starting
+
+ @BeanProperty
+ var lastActivityTime = System.currentTimeMillis
+
+ def getStatus: NodeStatus = status
+
+ protected def onStatusChanged(fromStatus: NodeStatus, toStatus: NodeStatus): Unit
+
+ def transition(toStatus: NodeStatus) = this synchronized {
+ lastActivityTime = System.currentTimeMillis
+ this.status match {
+ case NodeStatus.Failed | NodeStatus.Success =>
+ warn(s"$toString attempt to change status ${this.status} => $toStatus, ignore it.")
+ case NodeStatus.ShuttingDown =>
+ toStatus match {
+ case NodeStatus.Failed | NodeStatus.Success =>
+ val oldState = status
+ this.status = toStatus
+ onStatusChanged(oldState, toStatus)
+ case _ => warn(s"$toString attempt to change a Executor.ShuttingDown session to $toStatus, ignore it.")
+ }
+ case _ =>
+
+ }
+ info(s"$toString change status ${status} => $toStatus.")
+ val oldState = status
+ this.status = toStatus
+ onStatusChanged(oldState, toStatus)
+ }
+
+}
+
+object SensibleExecutor {
+
+ lazy val defaultErrorSensibleExecutor: SensibleExecutor = new SensibleExecutor {
+ override def getStatus: NodeStatus = NodeStatus.ShuttingDown
+
+ override protected def onStatusChanged(fromStatus: NodeStatus, toStatus: NodeStatus): Unit = {}
+
+ override def getId(): String = "0"
+
+ override def init(): Unit = {}
+
+ override def tryReady(): Boolean = false
+
+ override def tryShutdown(): Boolean = true
+
+ override def tryFailed(): Boolean = true
+
+ override def isClosed(): Boolean = true
+ }
+
+ def getDefaultErrorSensibleExecutor(): SensibleExecutor = defaultErrorSensibleExecutor
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/YarnExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/YarnExecutor.scala
new file mode 100644
index 0000000000..368ad78027
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/entity/YarnExecutor.scala
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.entity
+
+import scala.beans.BeanProperty
+
+trait YarnExecutor extends Executor {
+
+ @BeanProperty
+ var applicationId: String
+
+ @BeanProperty
+ var applicationURL: String
+
+ @BeanProperty
+ var yarnMode: String
+
+ @BeanProperty
+ var queue: String
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnAsyncListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnAsyncListener.scala
new file mode 100644
index 0000000000..5d68f71eef
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnAsyncListener.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.listener
+
+import com.webank.wedatasphere.linkis.common.listener.EventListener
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnAsyncEvent
+
+trait EngineConnAsyncListener extends EventListener {
+
+ def onEvent(event: EngineConnAsyncEvent): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnAsyncListenerBus.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnAsyncListenerBus.scala
new file mode 100644
index 0000000000..160f4b7a1c
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnAsyncListenerBus.scala
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.listener
+
+import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus
+import com.webank.wedatasphere.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnAsyncEvent
+
+class EngineConnAsyncListenerBus(eventQueueCapacity: Int, name: String,
+ listenerConsumerThreadSize: Int,
+ listenerThreadMaxFreeTime: Long)
+ extends ListenerEventBus[EngineConnAsyncListener, EngineConnAsyncEvent](eventQueueCapacity, name)(listenerConsumerThreadSize, listenerThreadMaxFreeTime) {
+
+ /**
+ * Post an event to the specified listener. `onPostEvent` is guaranteed to be called in the same
+ * thread for all listeners.
+ */
+ override protected def doPostEvent(listener: EngineConnAsyncListener, event: EngineConnAsyncEvent): Unit = {
+ listener.onEvent(event)
+ }
+
+}
+
+object EngineConnAsyncListenerBus {
+
+ def NAME: String = "EngineServerAsyncListenerBus"
+
+ lazy val getInstance = new EngineConnAsyncListenerBus(
+ EngineConnExecutorConfiguration.ENGINE_SERVER_LISTENER_ASYNC_QUEUE_CAPACITY.getValue,
+ NAME,
+ EngineConnExecutorConfiguration.ENGINE_SERVER_LISTENER_ASYNC_CONSUMER_THREAD_MAX.getValue,
+ EngineConnExecutorConfiguration.ENGINE_SERVER_LISTENER_ASYNC_CONSUMER_THREAD_FREE_TIME_MAX.getValue.toLong)
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnSyncListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnSyncListener.scala
new file mode 100644
index 0000000000..e40263eeae
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnSyncListener.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.listener
+
+import com.webank.wedatasphere.linkis.common.listener.EventListener
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnSyncEvent
+
+trait EngineConnSyncListener extends EventListener {
+
+ def onEvent(event: EngineConnSyncEvent): Unit
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnSyncListenerBus.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnSyncListenerBus.scala
new file mode 100644
index 0000000000..9a147375f0
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/EngineConnSyncListenerBus.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.listener
+
+import com.webank.wedatasphere.linkis.common.listener.ListenerBus
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.engineconn.executor.listener.event.EngineConnSyncEvent
+
+class EngineConnSyncListenerBus extends ListenerBus[EngineConnSyncListener, EngineConnSyncEvent] with Logging {
+
+ /**
+ * Post an event to the specified listener. `onPostEvent` is guaranteed to be called in the same
+ * thread for all listeners.
+ */
+ override protected def doPostEvent(listener: EngineConnSyncListener, event: EngineConnSyncEvent): Unit = {
+ debug(s"$listener start to deal event $event")
+ listener.onEvent(event)
+ debug(s"$listener Finished to deal event $event")
+ }
+}
+
+object EngineConnSyncListenerBus {
+
+ def NAME: String = "EngineServerSyncListenerBus"
+
+ lazy val getInstance = new EngineConnSyncListenerBus()
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/ExecutorListenerBusContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/ExecutorListenerBusContext.scala
new file mode 100644
index 0000000000..cfb2837210
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/ExecutorListenerBusContext.scala
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.listener
+
+import com.webank.wedatasphere.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration
+
+class ExecutorListenerBusContext {
+
+ private val engineConnAsyncListenerBus: EngineConnAsyncListenerBus = new EngineConnAsyncListenerBus(
+ EngineConnExecutorConfiguration.ENGINE_SERVER_LISTENER_ASYNC_QUEUE_CAPACITY.getValue,
+ "EngineConn-Asyn-Thread",
+ EngineConnExecutorConfiguration.ENGINE_SERVER_LISTENER_ASYNC_CONSUMER_THREAD_MAX.getValue,
+ EngineConnExecutorConfiguration.ENGINE_SERVER_LISTENER_ASYNC_CONSUMER_THREAD_FREE_TIME_MAX.getValue.toLong
+ )
+
+ private val engineConnSyncListenerBus: EngineConnSyncListenerBus = new EngineConnSyncListenerBus
+
+ def getEngineConnAsyncListenerBus: EngineConnAsyncListenerBus = {
+ engineConnAsyncListenerBus
+ }
+
+ def getEngineConnSyncListenerBus: EngineConnSyncListenerBus = engineConnSyncListenerBus
+
+}
+
+object ExecutorListenerBusContext {
+
+ private val executorListenerBusContext = new ExecutorListenerBusContext
+
+ def getExecutorListenerBusContext() = executorListenerBusContext
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/event/EngineConnAsyncEvent.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/event/EngineConnAsyncEvent.scala
new file mode 100644
index 0000000000..6161fa2212
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/event/EngineConnAsyncEvent.scala
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.listener.event
+
+import com.webank.wedatasphere.linkis.common.listener.Event
+
+trait EngineConnAsyncEvent extends Event {
+
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/event/EngineConnSyncEvent.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/event/EngineConnSyncEvent.scala
new file mode 100644
index 0000000000..4c9a56801b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/listener/event/EngineConnSyncEvent.scala
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.listener.event
+
+import com.webank.wedatasphere.linkis.common.listener.Event
+
+trait EngineConnSyncEvent extends Event {
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/service/LabelService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/service/LabelService.scala
new file mode 100644
index 0000000000..30ae088f6d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/service/LabelService.scala
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.service
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.engineconn.executor.entity.Executor
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+trait LabelService {
+
+ /*def labelUpdate(labelUpdateRequest: LabelUpdateRequest): Unit*/
+
+ def labelReport(labels: util.List[Label[_]], executor: Executor): Unit
+
+}
+
+class DefaultLabelService extends LabelService with Logging {
+
+
+ /*override def labelUpdate(labelUpdateRequest: LabelUpdateRequest): Unit = ???*/
+
+ override def labelReport(labels: util.List[Label[_]], executor: Executor): Unit = {
+ info(s"executor ${executor.getId()} prepare to report Labels ")
+ ManagerService.getManagerService.labelReport(labels)
+ info(s"executor ${executor.getId()} end to report Labels ")
+ }
+
+
+}
+
+object LabelService {
+
+ private val labelService = new DefaultLabelService
+
+ def getLabelService: LabelService = this.labelService
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/service/ManagerService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/service/ManagerService.scala
new file mode 100644
index 0000000000..a79ae003eb
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/com/webank/wedatasphere/linkis/engineconn/executor/service/ManagerService.scala
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.executor.service
+
+import com.webank.wedatasphere.linkis.common.utils.Utils
+import com.webank.wedatasphere.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.protocol.engine.EngineConnReleaseRequest
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.NodeHeartbeatMsg
+import com.webank.wedatasphere.linkis.manager.common.protocol.resource.ResourceUsedProtocol
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+trait ManagerService {
+
+ def labelReport(labels: java.util.List[Label[_]]): Unit
+
+ def statusReport(status: NodeStatus): Unit
+
+ def requestReleaseEngineConn(engineConnReleaseRequest: EngineConnReleaseRequest): Unit
+
+ def heartbeatReport(nodeHeartbeatMsg: NodeHeartbeatMsg): Unit
+
+ def reportUsedResource(resourceUsedProtocol: ResourceUsedProtocol): Unit
+
+}
+
+object ManagerService {
+
+ private val managerService: ManagerService = Utils.getClassInstance[ManagerService](EngineConnExecutorConfiguration.EXECUTOR_MANAGER_SERVICE_CLAZZ.getValue)
+
+ def getManagerService = managerService
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/resource-executor/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/resource-executor/pom.xml
new file mode 100644
index 0000000000..383a4f6d4b
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/resource-executor/pom.xml
@@ -0,0 +1,70 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+
+ 4.0.0
+
+ linkis-resource-executor
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-executor-core
+ ${linkis.version}
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/pom.xml
new file mode 100644
index 0000000000..df14b7a979
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/pom.xml
@@ -0,0 +1,72 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+ ../../../pom.xml
+
+ 4.0.0
+
+ linkis-engineconn-launch
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-engineconn-core
+ ${linkis.version}
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/src/main/scala/com/webank/wedatasphere/linkis/engineconn/launch/EngineConnServer.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/src/main/scala/com/webank/wedatasphere/linkis/engineconn/launch/EngineConnServer.scala
new file mode 100644
index 0000000000..4b8fb5de9f
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/src/main/scala/com/webank/wedatasphere/linkis/engineconn/launch/EngineConnServer.scala
@@ -0,0 +1,98 @@
+package com.webank.wedatasphere.linkis.engineconn.launch
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.common.conf.CommonVars
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engineconn.common.creation.{DefaultEngineCreationContext, EngineCreationContext}
+import com.webank.wedatasphere.linkis.engineconn.common.execution.EngineExecution
+import com.webank.wedatasphere.linkis.engineconn.common.hook.EngineConnHook
+import com.webank.wedatasphere.linkis.engineconn.core.EngineConnObject
+import com.webank.wedatasphere.linkis.engineconn.core.engineconn.EngineConnManager
+import com.webank.wedatasphere.linkis.engineconn.core.util.EngineConnUtils
+import com.webank.wedatasphere.linkis.engineconn.core.hook.ShutdownHook
+import com.webank.wedatasphere.linkis.governance.common.conf.GovernanceCommonConf
+import com.webank.wedatasphere.linkis.governance.common.utils.EngineConnArgumentsParser
+import com.webank.wedatasphere.linkis.manager.engineplugin.common.launch.process.Environment
+import com.webank.wedatasphere.linkis.manager.label.builder.factory.{LabelBuilderFactory, StdLabelBuilderFactory}
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
+
+
+object EngineConnServer extends Logging {
+
+
+ private val engineCreationContext: EngineCreationContext = new DefaultEngineCreationContext
+ private val labelBuilderFactory: LabelBuilderFactory = new StdLabelBuilderFactory
+
+ var isReady = false
+
+ def main(args: Array[String]): Unit = {
+ info("<<---------------------EngineConnServer Start --------------------->>")
+
+ Utils.tryCatch {
+ // 1. 封装EngineCreationContext
+ init(args)
+ info("Finished to create EngineCreationContext")
+ EngineConnHook.getEngineConnHooks.foreach(_.beforeCreateEngineConn(getEngineCreationContext))
+ info("Finished to execute hook of beforeCreateEngineConn")
+ //2. 创建EngineConn
+ val engineConn = getEngineConnManager.createEngineConn(getEngineCreationContext)
+ info(s"Finished to create engineConn that type is ${engineConn.getEngineType()}")
+ EngineConnHook.getEngineConnHooks.foreach(_.beforeExecutionExecute(getEngineCreationContext, engineConn))
+ info("Finished to execute hook of beforeExecutionExecute")
+ //3. 注册的executions 执行
+ getEngineExecutions.foreach(_.execute(getEngineCreationContext, engineConn))
+ EngineConnObject.setReady
+ info("Finished to execute executions")
+ EngineConnHook.getEngineConnHooks.foreach(_.afterExecutionExecute(getEngineCreationContext, engineConn))
+ info("Finished to execute hook of afterExecutionExecute")
+ EngineConnHook.getEngineConnHooks.foreach(_.afterEngineServerStartSuccess(getEngineCreationContext, engineConn))
+ } { t =>
+ EngineConnHook.getEngineConnHooks.foreach(_.afterEngineServerStartFailed(getEngineCreationContext, t))
+ }
+
+ //4. 等待Executions执行完毕
+ ShutdownHook.getShutdownHook.waitForStopOrError()
+ info("<<---------------------EngineConnServer Exit --------------------->>")
+ System.exit(ShutdownHook.getShutdownHook.getExitCode())
+ }
+
+ /**
+ *
+ * @param args
+ */
+ private def init(args: Array[String]): Unit = {
+ val arguments = EngineConnArgumentsParser.getEngineConnArgumentsParser.parseToObj(args)
+ val engineConf = arguments.getEngineConnConfMap
+ this.engineCreationContext.setUser(engineConf.getOrElse("user", Utils.getJvmUser))
+ this.engineCreationContext.setTicketId(engineConf.getOrElse("ticketId", ""))
+ val host = CommonVars(Environment.ECM_HOST.toString, "127.0.0.1").getValue
+ val port = CommonVars(Environment.ECM_PORT.toString, "80").getValue
+ this.engineCreationContext.setEMInstance(ServiceInstance(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue, s"$host:$port"))
+ val labels = new ArrayBuffer[Label[_]]
+ val labelArgs = engineConf.filter(_._1.startsWith(EngineConnArgumentsParser.LABEL_PREFIX))
+ if (labelArgs.nonEmpty) {
+ labelArgs.foreach { case (key, value) =>
+ labels += labelBuilderFactory.createLabel[Label[_]](key.replace(EngineConnArgumentsParser.LABEL_PREFIX, ""), value)
+ }
+ engineCreationContext.setLabels(labels.toList)
+ }
+ val jMap = new java.util.HashMap[String, String](engineConf.size)
+ jMap.putAll(engineConf)
+ this.engineCreationContext.setOptions(jMap)
+ this.engineCreationContext.setArgs(args)
+ EngineConnObject.setEngineCreationContext(this.engineCreationContext)
+ info("Finished to init engineCreationContext" + EngineConnUtils.GSON.toJson(engineCreationContext))
+ }
+
+ def getEngineCreationContext: EngineCreationContext = this.engineCreationContext
+
+ private def getEngineConnManager: EngineConnManager = EngineConnManager.getEngineConnManager
+
+ private def getEngineExecutions: Array[EngineExecution] = EngineExecution.getEngineExecutions
+
+ private def getShutdownHook: ShutdownHook = ShutdownHook.getShutdownHook
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/pom.xml b/linkis-computation-governance/linkis-engineconn/pom.xml
new file mode 100644
index 0000000000..07d1c6d4a0
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/pom.xml
@@ -0,0 +1,42 @@
+
+
+
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+ 4.0.0
+
+ linkis-engineconn
+ pom
+
+ linkis-engineconn-common
+ linkis-engineconn-core
+ linkis-engineconn-executor/executor-core
+ linkis-engineconn-launch
+ linkis-engineconn-executor/accessible-executor
+ linkis-engineconn-executor/resource-executor
+ linkis-engineconn-executor/callback-service
+ linkis-computation-engineconn
+
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/pom.xml b/linkis-computation-governance/linkis-entrance-client/pom.xml
new file mode 100644
index 0000000000..e88dff6d64
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance-client/pom.xml
@@ -0,0 +1,73 @@
+
+
+
+
+ 4.0.0
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+
+ linkis-entrance-client
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-entrance
+ ${linkis.version}
+
+
+ com.webank.wedatasphere.linkis
+ linkis-io-entrance
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-rpc
+ ${linkis.version}
+ provided
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+
+ ${basedir}/src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
+
\ No newline at end of file
diff --git a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java
similarity index 97%
rename from ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java
index 06c3415bb1..231eca71f5 100644
--- a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java
@@ -27,9 +27,6 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-/**
- * Created by johnnwang on 2019/2/15.
- */
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Bean(value = ClientEngineBuilderBeanAnnotation.BEAN_NAME)
diff --git a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java
similarity index 97%
rename from ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java
index daf155ebd0..98f3c0f3fd 100644
--- a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java
@@ -27,9 +27,6 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-/**
- * Created by johnnwang on 2019/2/15.
- */
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Bean(value = ClientEngineRequesterBeanAnnotation.BEAN_NAME)
diff --git a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java
similarity index 97%
rename from ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java
index 6bf9968452..6a51a80923 100644
--- a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java
@@ -27,9 +27,6 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-/**
- * Created by johnnwang on 2019/2/15.
- */
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Bean(value = ClientEngineSelectorBeanAnnotation.BEAN_NAME)
diff --git a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java
similarity index 97%
rename from ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java
index daf074b80a..94a0735f51 100644
--- a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java
@@ -27,9 +27,6 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-/**
- * Created by johnnwang on 2019/2/15.
- */
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Bean(value = ClientEntranceParserBeanAnnotation.BEAN_NAME)
diff --git a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java
similarity index 97%
rename from ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java
index be9f298ff4..9f1d949062 100644
--- a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java
@@ -27,9 +27,6 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-/**
- * Created by johnnwang on 2019/2/15.
- */
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Bean(value = ClientGroupFactoryBeanAnnotation.BEAN_NAME)
diff --git a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java
similarity index 97%
rename from ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java
index 4ac97bece0..6b7a58f2c1 100644
--- a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java
@@ -27,9 +27,6 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-/**
- * Created by johnnwang on 2019/2/15.
- */
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Bean(value = ClientInterceptorsBeanAnnotation.BEAN_NAME)
diff --git a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java
similarity index 97%
rename from ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java
index 5a606c0290..f327db1982 100644
--- a/ujes/entranceclient/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java
@@ -27,9 +27,6 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-/**
- * Created by johnnwang on 2019/2/15.
- */
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Bean(value = DefaultEntranceClientBeanAnnotation.BEAN_NAME)
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/conf/ClientSpringConfiguration.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/conf/ClientSpringConfiguration.java
new file mode 100644
index 0000000000..ac0fbb3c3c
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/java/com/webank/wedatasphere/linkis/entranceclient/conf/ClientSpringConfiguration.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entranceclient.conf;
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication;
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceSpringConfiguration;
+import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor;
+import com.webank.wedatasphere.linkis.entrance.scheduler.EntranceSchedulerContext;
+import com.webank.wedatasphere.linkis.entranceclient.EntranceClient;
+import com.webank.wedatasphere.linkis.entranceclient.EntranceClientImpl;
+import com.webank.wedatasphere.linkis.entranceclient.annotation.ClientEntranceParserBeanAnnotation;
+import com.webank.wedatasphere.linkis.entranceclient.annotation.ClientInterceptorsBeanAnnotation;
+import com.webank.wedatasphere.linkis.entranceclient.annotation.DefaultEntranceClientBeanAnnotation;
+import com.webank.wedatasphere.linkis.entranceclient.context.ClientEntranceParser;
+import com.webank.wedatasphere.linkis.rpc.RPCReceiveRestful;
+import com.webank.wedatasphere.linkis.rpc.conf.RPCConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.AutoConfigureBefore;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@AutoConfigureBefore({EntranceSpringConfiguration.class, RPCReceiveRestful.class})
+class ClientSpringConfiguration {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ {
+ logger.info("start a multi-entrance application...");
+ }
+
+ @ClientEntranceParserBeanAnnotation
+ @ConditionalOnMissingBean(name = {ClientEntranceParserBeanAnnotation.BEAN_NAME})
+ public ClientEntranceParser generateClientEntranceParser() {
+ return new ClientEntranceParser();
+ }
+
+
+ @ClientInterceptorsBeanAnnotation
+ @ConditionalOnMissingBean(name = {ClientInterceptorsBeanAnnotation.BEAN_NAME})
+ public EntranceInterceptor[] generateEntranceInterceptor() {
+ return new EntranceInterceptor[0];
+ }
+
+
+ @DefaultEntranceClientBeanAnnotation
+ @Conditional(MultiEntranceCondition.class)
+ public EntranceClient generateEntranceClient(@ClientEntranceParserBeanAnnotation.ClientEntranceParserAutowiredAnnotation ClientEntranceParser clientEntranceParser,
+ @ClientInterceptorsBeanAnnotation.ClientInterceptorsAutowiredAnnotation EntranceInterceptor[] clientInterceptors,
+ @Autowired EntranceSchedulerContext entranceSchedulerContext) {
+ logger.warn("try to check the rpc receiver consumer threadPool...");
+ setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY_FOR_CLIENT());
+ setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX_FOR_CLIENT());
+ EntranceClientImpl client = EntranceClientImpl.apply(ClientConfiguration.CLIENT_DEFAULT_NAME());
+ logger.warn("Multi-entrance application is ready to initial EntranceClient " + client.getEntranceClientName());
+ client.init(clientEntranceParser, entranceSchedulerContext, clientInterceptors, ClientConfiguration.CLIENT_DEFAULT_PARALLELISM_USERS().getValue());
+ return client;
+ }
+
+// @EventListener
+// public void onApplicationStarting(ApplicationStartingEvent applicationStartingEvent) {
+// if(MultiEntranceCondition.isMultiEntranceApplication()) {
+// logger.warn("start a multi-entrance application, now try to check the rpc receiver consumer threadPool...");
+// setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY_FOR_CLIENT());
+// setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX_FOR_CLIENT());
+// setClientValue(EntranceConfiguration.CONCURRENT_ENGINE_MAX_PARALLELISM(), ClientConfiguration.CONCURRENT_ENGINE_MAX_PARALLELISM_FOR_CLIENT());
+// }
+// }
+
+ private void setClientValue(CommonVars fromConf, CommonVars toConf) {
+ int queueSize = Integer.parseInt(fromConf.getValue().toString());
+ int queueSizeValue = toConf.getValue();
+ if (queueSize < queueSizeValue) {
+ String key = fromConf.key();
+ DataWorkCloudApplication.setProperty(key, String.valueOf(queueSizeValue));
+ logger.warn("Multi-entrance application set " + key + "=" + queueSizeValue);
+ }
+ }
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/resources/META-INF/spring.factories b/linkis-computation-governance/linkis-entrance-client/src/main/resources/META-INF/spring.factories
new file mode 100644
index 0000000000..715812457a
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/resources/META-INF/spring.factories
@@ -0,0 +1,3 @@
+org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
+com.webank.wedatasphere.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,\
+com.webank.wedatasphere.linkis.entranceclient.conf.ClientSpringConfiguration
\ No newline at end of file
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EngineApplicationNameFactory.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EngineApplicationNameFactory.scala
similarity index 97%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EngineApplicationNameFactory.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EngineApplicationNameFactory.scala
index cb3d225d3e..f4a4ce35c8 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EngineApplicationNameFactory.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EngineApplicationNameFactory.scala
@@ -18,9 +18,6 @@ package com.webank.wedatasphere.linkis.entranceclient
import com.webank.wedatasphere.linkis.entranceclient.conf.ClientConfiguration.{CLIENT_ENGINE_MANAGER_SPRING_APPLICATION_NAME, CLIENT_ENGINE_SPRING_APPLICATION_NAME}
-/**
- * Created by johnnwang on 2019/1/22.
- */
trait EngineApplicationNameFactory {
private var engineApplicationName: String = CLIENT_ENGINE_SPRING_APPLICATION_NAME.getValue
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClient.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClient.scala
similarity index 96%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClient.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClient.scala
index ef47c59060..d0cd9fb041 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClient.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClient.scala
@@ -18,9 +18,6 @@ package com.webank.wedatasphere.linkis.entranceclient
import com.webank.wedatasphere.linkis.entranceclient.execute.ClientJob
-/**
- * Created by johnnwang on 2019/1/22.
- */
trait EntranceClient {
def getEntranceClientName: String
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClientImpl.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClientImpl.scala
new file mode 100644
index 0000000000..fe4923a63f
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/EntranceClientImpl.scala
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entranceclient
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.entrance.context.DefaultEntranceContext
+import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
+import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
+import com.webank.wedatasphere.linkis.entrance.scheduler.EntranceSchedulerContext
+import com.webank.wedatasphere.linkis.entrance.server.DefaultEntranceServer
+import com.webank.wedatasphere.linkis.entrance.{EntranceContext, EntranceServer}
+import com.webank.wedatasphere.linkis.entranceclient.execute._
+import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
+import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelScheduler
+import com.webank.wedatasphere.linkis.server.JMap
+import org.apache.commons.lang.StringUtils
+
+class EntranceClientImpl private() extends EntranceClient with EngineApplicationNameFactory
+ with EngineManagerApplicationNameFactory with Logging {
+
+ private var entranceServer: EntranceServer = _
+ private var clientName: String = _
+
+
+ def setClientName(clientName: String): Unit = this.clientName = clientName
+
+
+ def init(entranceServer: EntranceServer): Unit = {
+ this.entranceServer = entranceServer
+ }
+
+ def init(entranceContext: EntranceContext): Unit = init(new DefaultEntranceServer(entranceContext))
+
+ def init(clientEntranceParser: context.ClientEntranceParser, schedulerContext: EntranceSchedulerContext, interceptors: Array[EntranceInterceptor],
+ maxParallelismUsers: Int): Unit = if (entranceServer == null) synchronized {
+ if (entranceServer == null) {
+ val scheduler = new ParallelScheduler(schedulerContext)
+ scheduler.init()
+ scheduler.start()
+ val entranceContext = new DefaultEntranceContext(clientEntranceParser, new context.ClientPersistenceManager, new context.ClientLogManager,
+ scheduler, interceptors, null, Array.empty)
+ init(entranceContext)
+ }
+ }
+
+ private def executeUntil[T](code: String, user: String, creator: String,
+ params: java.util.Map[String, Any], op: ClientJob => T): T = {
+ val execId = executeJob(code, user, creator, params)
+ entranceServer.getJob(execId).foreach {
+ case job: ClientJob =>
+ job.waitForComplete()
+ return op(job)
+ }
+ throw new EntranceErrorException(50003, s"execute failed, cannot find the job $execId.")
+ }
+
+ override def getEntranceClientName: String = if(StringUtils.isNotEmpty(clientName)) clientName else entranceServer.getName
+
+ override def execute(code: String, user: String, creator: String): Boolean = execute(code, user, creator, null)
+
+ override def execute(code: String, user: String, creator: String,
+ params: java.util.Map[String, Any]): Boolean =
+ executeUntil(code, user, creator, params, job =>
+ if (job.isSucceed) true else false)
+
+ override def executeJob(code: String, user: String, creator: String): String = executeJob(code, user, creator, null)
+
+ override def executeJob(code: String, user: String, creator: String, params: util.Map[String, Any]): String = {
+ val requestMap = new JMap[String, Any]
+ requestMap.put(TaskConstant.EXECUTIONCODE, code)
+ requestMap.put(TaskConstant.UMUSER, user)
+ requestMap.put(TaskConstant.REQUESTAPPLICATIONNAME, creator)
+ if(params != null && !params.isEmpty) {
+ if(params.containsKey(EntranceServer.DO_NOT_PRINT_PARAMS_LOG)) {
+ requestMap.put(EntranceServer.DO_NOT_PRINT_PARAMS_LOG, params.get(EntranceServer.DO_NOT_PRINT_PARAMS_LOG))
+ params.remove(EntranceServer.DO_NOT_PRINT_PARAMS_LOG)
+ }
+ if(!params.isEmpty) requestMap.put(TaskConstant.PARAMS, params)
+ }
+ entranceServer.execute(requestMap)
+ }
+
+ override def getJob(jobId: String): Option[ClientJob] = entranceServer.getJob(jobId).map { case job: ClientJob => job }
+
+ override def executeResult(code: String, user: String, creator: String): Array[String] =
+ executeResult(code, user, creator, null)
+
+ override def executeResult(code: String, user: String, creator: String,
+ params: java.util.Map[String, Any]): Array[String] =
+ executeUntil(code, user, creator, params, job =>
+ if(job.isSucceed) job.getResultSets
+ else if(job.getErrorResponse != null) {
+ val exception = new EntranceErrorException(22001, job.getErrorResponse.message)
+ if(job.getErrorResponse.t != null) exception.initCause(job.getErrorResponse.t)
+ throw exception
+ } else throw new EntranceErrorException(22002, "execute failed, unknown reason."))
+
+}
+object EntranceClientImpl {
+
+ private val clientNameToEntranceClient = new JMap[String, EntranceClientImpl]
+
+ def apply(clientName: String): EntranceClientImpl = {
+ if(!clientNameToEntranceClient.containsKey(clientName)) synchronized {
+ if(!clientNameToEntranceClient.containsKey(clientName)) {
+ val client = new EntranceClientImpl
+ client.setClientName(clientName)
+ clientNameToEntranceClient.put(clientName, client)
+ }
+ }
+ clientNameToEntranceClient.get(clientName)
+ }
+
+ def getClientNames = clientNameToEntranceClient.keySet()
+}
\ No newline at end of file
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientConfiguration.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientConfiguration.scala
similarity index 97%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientConfiguration.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientConfiguration.scala
index ad46e5caad..68e381de44 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientConfiguration.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientConfiguration.scala
@@ -18,9 +18,6 @@ package com.webank.wedatasphere.linkis.entranceclient.conf
import com.webank.wedatasphere.linkis.common.conf.CommonVars
-/**
- * Created by johnnwang on 2019/1/22.
- */
object ClientConfiguration {
val CLIENT_ENGINE_MANAGER_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.client.enginemanager.application.name.default", "IOEngineManager")
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientForEntranceSpringConfiguration.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientForEntranceSpringConfiguration.scala
new file mode 100644
index 0000000000..93a0342932
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/ClientForEntranceSpringConfiguration.scala
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entranceclient.conf
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+//import com.webank.wedatasphere.linkis.entrance.annotation.EntranceServerBeanAnnotation.EntranceServerAutowiredAnnotation
+//import com.webank.wedatasphere.linkis.entrance.annotation._
+//import com.webank.wedatasphere.linkis.entrance.conf.EntranceSpringConfiguration
+//import com.webank.wedatasphere.linkis.entrance.execute._
+//import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
+//import com.webank.wedatasphere.linkis.entrance.log.LogManager
+//import com.webank.wedatasphere.linkis.entrance.persistence.PersistenceManager
+//import com.webank.wedatasphere.linkis.entrance.{EntranceParser, EntranceServer}
+//import com.webank.wedatasphere.linkis.entranceclient.annotation.ClientEngineSelectorBeanAnnotation.ClientEngineSelectorAutowiredAnnotation
+//import com.webank.wedatasphere.linkis.entranceclient.annotation.ClientEntranceParserBeanAnnotation.ClientEntranceParserAutowiredAnnotation
+//import com.webank.wedatasphere.linkis.entranceclient.annotation.ClientInterceptorsBeanAnnotation.ClientInterceptorsAutowiredAnnotation
+//import com.webank.wedatasphere.linkis.entranceclient.annotation.DefaultEntranceClientBeanAnnotation
+//import com.webank.wedatasphere.linkis.entranceclient.{EntranceClient, EntranceClientImpl, _}
+//import com.webank.wedatasphere.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser}
+//import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
+//import org.springframework.boot.autoconfigure.{AutoConfigureAfter, AutoConfigureBefore}
+//import org.springframework.context.annotation.{Conditional, Configuration}
+
+/*@Configuration
+@AutoConfigureBefore(Array(classOf[EntranceSpringConfiguration]))
+@AutoConfigureAfter(Array(classOf[ClientSpringConfiguration]))
+@Conditional(Array(classOf[SingleEntranceCondition]))*/
+class ClientForEntranceSpringConfiguration extends Logging {
+
+ warn(s"start a single-entrance application...")
+
+ /*@DefaultEntranceClientBeanAnnotation
+ def generateEntranceClient(@EntranceServerAutowiredAnnotation entranceServer: EntranceServer): EntranceClient = {
+ val client = EntranceClientImpl(ClientConfiguration.CLIENT_DEFAULT_NAME)
+ warn(s"ready to initial EntranceClient ${client.getEntranceClientName}...")
+ client.init(entranceServer)
+ client
+ }*/
+/*
+
+ @ReceiverChooserBeanAnnotation
+ def generateEntranceReceiverChooser(): ReceiverChooser = new ReceiverChooser{
+ override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = None
+ }
+
+ @PersistenceManagerBeanAnnotation
+ @ConditionalOnMissingBean(value = Array(classOf[context.ClientPersistenceManager]))
+ def generatePersistenceManager(): PersistenceManager = new context.ClientPersistenceManager
+
+ @EntranceParserBeanAnnotation
+ def generateEntranceParser(@ClientEntranceParserAutowiredAnnotation clientEntranceParser: context.ClientEntranceParser): EntranceParser = clientEntranceParser
+
+ @LogManagerBeanAnnotation
+ @ConditionalOnMissingBean(value = Array(classOf[context.ClientLogManager]))
+ def generateLogManager(): LogManager = new context.ClientLogManager
+
+
+ @EntranceInterceptorBeanAnnotation
+ def generateEntranceInterceptor(@ClientInterceptorsAutowiredAnnotation clientInterceptors: Array[EntranceInterceptor]): Array[EntranceInterceptor] = clientInterceptors
+*/
+
+
+// @EngineSelectorBeanAnnotation
+// def generateEngineSelector(@ClientEngineSelectorAutowiredAnnotation clientEngineSelector: EngineSelector): EngineSelector = clientEngineSelector
+
+
+}
\ No newline at end of file
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/MultiEntranceCondition.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/MultiEntranceCondition.scala
similarity index 86%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/MultiEntranceCondition.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/MultiEntranceCondition.scala
index 90d1077584..f3bac7e5f0 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/MultiEntranceCondition.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/conf/MultiEntranceCondition.scala
@@ -18,13 +18,9 @@ package com.webank.wedatasphere.linkis.entranceclient.conf
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entranceclient
-import org.apache.commons.lang.StringUtils
import org.springframework.context.annotation.{Condition, ConditionContext}
import org.springframework.core.`type`.AnnotatedTypeMetadata
-/**
- * Created by johnnwang on 2019/1/22.
- */
class MultiEntranceCondition extends Condition {
override def matches(conditionContext: ConditionContext, annotatedTypeMetadata: AnnotatedTypeMetadata): Boolean =
entranceclient.conf.MultiEntranceCondition.isMultiEntranceApplication
@@ -34,5 +30,5 @@ class SingleEntranceCondition extends Condition {
!entranceclient.conf.MultiEntranceCondition.isMultiEntranceApplication
}
object MultiEntranceCondition {
- def isMultiEntranceApplication: Boolean = StringUtils.isNotBlank(EntranceConfiguration.ENGINE_MANAGER_SPRING_APPLICATION_NAME.getValue)
+ def isMultiEntranceApplication: Boolean = EntranceConfiguration.MULTI_ENTRANCE_CONDITION.getValue
}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientEntranceParser.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientEntranceParser.scala
new file mode 100644
index 0000000000..e12fcf7e49
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientEntranceParser.scala
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entranceclient.context
+
+import java.util
+
+import com.webank.wedatasphere.linkis.entrance.parser.CommonEntranceParser
+import com.webank.wedatasphere.linkis.entranceclient.execute
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.task.Task
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+
+class ClientEntranceParser extends CommonEntranceParser {
+
+ override def parseToTask(job: Job): Task = job match {
+ case j: execute.ClientJob => j.getTask
+ }
+
+ override def parseToJob(task: Task): Job = {
+ val job = new execute.ClientJob
+ task match {
+ case t: RequestPersistTask =>
+ job.setTask(t)
+ job.setCreator(t.getRequestApplicationName)
+ job.setUser(t.getUmUser)
+ if (t.getParams != null) job.setParams(t.getParams.asInstanceOf[util.Map[String, Any]])
+ }
+ job
+ }
+}
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientLogManager.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientLogManager.scala
similarity index 97%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientLogManager.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientLogManager.scala
index fd15742125..1b1b4d5bc7 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientLogManager.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientLogManager.scala
@@ -24,9 +24,6 @@ import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import org.apache.commons.io.input.NullInputStream
import org.apache.commons.io.output.NullOutputStream
-/**
- * Created by johnnwang on 2018/10/30.
- */
class ClientLogManager extends CacheLogManager {
override def getLogReader(execId: String): LogReader = {
new CacheLogReader("", ServerConfiguration.BDP_SERVER_ENCODING.getValue,
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceEngine.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceEngine.scala
similarity index 97%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceEngine.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceEngine.scala
index 1f5e31d503..d69fe21aea 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceEngine.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceEngine.scala
@@ -21,9 +21,6 @@ import java.lang
import com.webank.wedatasphere.linkis.entrance.persistence.PersistenceEngine
import com.webank.wedatasphere.linkis.protocol.task.Task
-/**
- * Created by johnnwang on 2018/10/30.
- */
class ClientPersistenceEngine extends PersistenceEngine {
override def persist(task: Task): Unit = {}
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceManager.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceManager.scala
similarity index 97%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceManager.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceManager.scala
index c9254c4fb3..c8320f9a76 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceManager.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/context/ClientPersistenceManager.scala
@@ -23,9 +23,6 @@ import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
import com.webank.wedatasphere.linkis.scheduler.executer.OutputExecuteResponse
import com.webank.wedatasphere.linkis.scheduler.queue.Job
-/**
- * Created by johnnwang on 2018/10/30.
- */
class ClientPersistenceManager extends PersistenceManager {
private val persistenceEngine = new ClientPersistenceEngine
override def getEntranceContext: EntranceContext = null
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/exception/ClientErrorException.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/exception/ClientErrorException.scala
similarity index 100%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/exception/ClientErrorException.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/exception/ClientErrorException.scala
diff --git a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/execute/ClientJob.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/execute/ClientJob.scala
similarity index 80%
rename from ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/execute/ClientJob.scala
rename to linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/execute/ClientJob.scala
index 58332dad65..23834cf411 100644
--- a/ujes/entranceclient/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/execute/ClientJob.scala
+++ b/linkis-computation-governance/linkis-entrance-client/src/main/scala/com/webank/wedatasphere/linkis/entranceclient/execute/ClientJob.scala
@@ -17,27 +17,25 @@
package com.webank.wedatasphere.linkis.entranceclient.execute
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
-import com.webank.wedatasphere.linkis.entranceclient.context.ClientTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, JobExecuteRequest}
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState
import com.webank.wedatasphere.linkis.scheduler.queue.{JobInfo, SchedulerEventState}
import scala.collection.mutable.ArrayBuffer
-/**
- * Created by johnnwang on 2018/10/30.
- */
class ClientJob extends EntranceJob {
private val resultSets = ArrayBuffer[String]()
def addResultSet(resultSet: String): Unit = resultSets += resultSet
+
def getResultSets: Array[String] = resultSets.toArray
def waitForComplete(): Unit = {
- if(SchedulerEventState.isCompleted(this.getState)) return
+ if (SchedulerEventState.isCompleted(this.getState)) return
resultSets synchronized {
- while(!SchedulerEventState.isCompleted(this.getState)) resultSets.wait()
+ while (!SchedulerEventState.isCompleted(this.getState)) resultSets.wait()
}
}
@@ -51,11 +49,11 @@ class ClientJob extends EntranceJob {
override def init(): Unit = {}
override protected def jobToExecuteRequest: ExecuteRequest = new ExecuteRequest with JobExecuteRequest {
- override val code: String = getTask.asInstanceOf[ClientTask].getCode
+ override val code: String = getTask.asInstanceOf[RequestPersistTask].getCode
override val jobId: String = getId
}
- override def getName: String = getTask.asInstanceOf[ClientTask].getCreator + "_" + getTask.asInstanceOf[ClientTask].getUser + "_" + getId
+ override def getName: String = getTask.asInstanceOf[RequestPersistTask].getRequestApplicationName + "_" + getTask.asInstanceOf[RequestPersistTask].getUmUser + "_" + getId
override def getJobInfo: JobInfo = null
diff --git a/linkis-computation-governance/linkis-entrance/pom.xml b/linkis-computation-governance/linkis-entrance/pom.xml
new file mode 100644
index 0000000000..c9e6678a1e
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/pom.xml
@@ -0,0 +1,142 @@
+
+
+
+
+ 4.0.0
+
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+ linkis-entrance
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-scheduler
+ ${linkis.version}
+
+
+ com.webank.wedatasphere.linkis
+ linkis-common
+
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-rpc
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-storage
+ ${linkis.version}
+ provided
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-cs-engine-support
+ ${linkis.version}
+
+
+
+ org.apache.httpcomponents
+ httpclient
+ ${httpclient.version}
+ provided
+
+
+
+ junit
+ junit
+ 4.12
+ test
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-computation-governance-common
+ ${linkis.version}
+
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-orchestrator-ecm-plugin
+ ${linkis.version}
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ 2.3
+ false
+
+
+ make-assembly
+ package
+
+ single
+
+
+
+ src/main/assembly/distribution.xml
+
+
+
+
+
+ false
+ out
+ false
+ false
+
+ src/main/assembly/distribution.xml
+
+
+
+
+
+
+ ${basedir}/src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance/src/main/assembly/distribution.xml b/linkis-computation-governance/linkis-entrance/src/main/assembly/distribution.xml
new file mode 100644
index 0000000000..6781dc2891
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/assembly/distribution.xml
@@ -0,0 +1,319 @@
+
+
+
+ linkis-ujes-spark-entrance
+
+ dir
+ zip
+
+ false
+ linkis-ujes-linkis-entrance
+
+
+
+
+
+ lib
+ true
+ true
+ false
+ false
+ true
+
+
+ antlr:antlr:jar
+ aopalliance:aopalliance:jar
+ asm:asm:jar
+ cglib:cglib:jar
+ com.amazonaws:aws-java-sdk-autoscaling:jar
+ com.amazonaws:aws-java-sdk-core:jar
+ com.amazonaws:aws-java-sdk-ec2:jar
+ com.amazonaws:aws-java-sdk-route53:jar
+ com.amazonaws:aws-java-sdk-sts:jar
+ com.amazonaws:jmespath-java:jar
+ com.fasterxml.jackson.core:jackson-annotations:jar
+ com.fasterxml.jackson.core:jackson-core:jar
+ com.fasterxml.jackson.core:jackson-databind:jar
+ com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar
+ com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar
+ com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar
+ com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar
+ com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar
+ com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar
+ com.fasterxml.jackson.module:jackson-module-parameter-names:jar
+ com.fasterxml.jackson.module:jackson-module-paranamer:jar
+ com.fasterxml.jackson.module:jackson-module-scala_2.11:jar
+ com.github.andrewoma.dexx:dexx-collections:jar
+ com.github.vlsi.compactmap:compactmap:jar
+ com.google.code.findbugs:annotations:jar
+ com.google.code.findbugs:jsr305:jar
+ com.google.code.gson:gson:jar
+ com.google.guava:guava:jar
+ com.google.inject:guice:jar
+ com.google.protobuf:protobuf-java:jar
+ com.netflix.archaius:archaius-core:jar
+ com.netflix.eureka:eureka-client:jar
+ com.netflix.eureka:eureka-core:jar
+ com.netflix.hystrix:hystrix-core:jar
+ com.netflix.netflix-commons:netflix-commons-util:jar
+ com.netflix.netflix-commons:netflix-eventbus:jar
+ com.netflix.netflix-commons:netflix-infix:jar
+ com.netflix.netflix-commons:netflix-statistics:jar
+ com.netflix.ribbon:ribbon:jar
+ com.netflix.ribbon:ribbon-core:jar
+ com.netflix.ribbon:ribbon-eureka:jar
+ com.netflix.ribbon:ribbon-httpclient:jar
+ com.netflix.ribbon:ribbon-loadbalancer:jar
+ com.netflix.ribbon:ribbon-transport:jar
+ com.netflix.servo:servo-core:jar
+ com.ning:async-http-client:jar
+ com.sun.jersey.contribs:jersey-apache-client4:jar
+ com.sun.jersey:jersey-client:jar
+ com.sun.jersey:jersey-core:jar
+ com.sun.jersey:jersey-json:jar
+ com.sun.jersey:jersey-server:jar
+ com.sun.jersey:jersey-servlet:jar
+ com.sun.xml.bind:jaxb-impl:jar
+ com.thoughtworks.paranamer:paranamer:jar
+ com.thoughtworks.xstream:xstream:jar
+ com.webank.wedatasphere.linkis:linkis-common:jar
+ com.webank.wedatasphere.linkis:linkis-module:jar
+ commons-beanutils:commons-beanutils:jar
+ commons-beanutils:commons-beanutils-core:jar
+ commons-cli:commons-cli:jar
+ commons-codec:commons-codec:jar
+ commons-collections:commons-collections:jar
+ commons-configuration:commons-configuration:jar
+ commons-daemon:commons-daemon:jar
+ commons-dbcp:commons-dbcp:jar
+ commons-digester:commons-digester:jar
+ commons-httpclient:commons-httpclient:jar
+ commons-io:commons-io:jar
+ commons-jxpath:commons-jxpath:jar
+ commons-lang:commons-lang:jar
+ commons-logging:commons-logging:jar
+ commons-net:commons-net:jar
+ commons-pool:commons-pool:jar
+ io.micrometer:micrometer-core:jar
+ io.netty:netty:jar
+ io.netty:netty-all:jar
+ io.netty:netty-buffer:jar
+ io.netty:netty-codec:jar
+ io.netty:netty-codec-http:jar
+ io.netty:netty-common:jar
+ io.netty:netty-handler:jar
+ io.netty:netty-transport:jar
+ io.netty:netty-transport-native-epoll:jar
+ io.reactivex:rxjava:jar
+ io.reactivex:rxnetty:jar
+ io.reactivex:rxnetty-contexts:jar
+ io.reactivex:rxnetty-servo:jar
+ javax.activation:activation:jar
+ javax.annotation:javax.annotation-api:jar
+ javax.inject:javax.inject:jar
+ javax.servlet:javax.servlet-api:jar
+ javax.servlet.jsp:jsp-api:jar
+ javax.validation:validation-api:jar
+ javax.websocket:javax.websocket-api:jar
+ javax.ws.rs:javax.ws.rs-api:jar
+ javax.xml.bind:jaxb-api:jar
+ javax.xml.stream:stax-api:jar
+ joda-time:joda-time:jar
+ log4j:log4j:jar
+ mysql:mysql-connector-java:jar
+ net.databinder.dispatch:dispatch-core_2.11:jar
+ net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar
+ org.antlr:antlr-runtime:jar
+ org.antlr:stringtemplate:jar
+ org.apache.commons:commons-compress:jar
+ org.apache.commons:commons-math:jar
+ org.apache.commons:commons-math3:jar
+ org.apache.curator:curator-client:jar
+ org.apache.curator:curator-framework:jar
+ org.apache.curator:curator-recipes:jar
+ org.apache.directory.api:api-asn1-api:jar
+ org.apache.directory.api:api-util:jar
+ org.apache.directory.server:apacheds-i18n:jar
+ org.apache.directory.server:apacheds-kerberos-codec:jar
+ org.apache.hadoop:hadoop-annotations:jar
+ org.apache.hadoop:hadoop-auth:jar
+ org.apache.hadoop:hadoop-common:jar
+ org.apache.hadoop:hadoop-hdfs:jar
+ org.apache.htrace:htrace-core:jar
+ org.apache.httpcomponents:httpclient:jar
+ org.apache.httpcomponents:httpcore:jar
+ org.apache.logging.log4j:log4j-api:jar
+ org.apache.logging.log4j:log4j-core:jar
+ org.apache.logging.log4j:log4j-jul:jar
+ org.apache.logging.log4j:log4j-slf4j-impl:jar
+ org.apache.zookeeper:zookeeper:jar
+ org.aspectj:aspectjweaver:jar
+ org.bouncycastle:bcpkix-jdk15on:jar
+ org.bouncycastle:bcprov-jdk15on:jar
+ org.codehaus.jackson:jackson-jaxrs:jar
+ org.codehaus.jackson:jackson-xc:jar
+ org.codehaus.jettison:jettison:jar
+ org.codehaus.woodstox:stax2-api:jar
+ org.codehaus.woodstox:woodstox-core-asl:jar
+ org.eclipse.jetty:jetty-annotations:jar
+ org.eclipse.jetty:jetty-client:jar
+ org.eclipse.jetty:jetty-continuation:jar
+ org.eclipse.jetty:jetty-http:jar
+ org.eclipse.jetty:jetty-io:jar
+ org.eclipse.jetty:jetty-jndi:jar
+ org.eclipse.jetty:jetty-plus:jar
+ org.eclipse.jetty:jetty-security:jar
+ org.eclipse.jetty:jetty-server:jar
+ org.eclipse.jetty:jetty-servlet:jar
+ org.eclipse.jetty:jetty-servlets:jar
+ org.eclipse.jetty:jetty-util:jar
+ org.eclipse.jetty:jetty-webapp:jar
+ org.eclipse.jetty:jetty-xml:jar
+ org.eclipse.jetty.websocket:javax-websocket-client-impl:jar
+ org.eclipse.jetty.websocket:javax-websocket-server-impl:jar
+ org.eclipse.jetty.websocket:websocket-api:jar
+ org.eclipse.jetty.websocket:websocket-client:jar
+ org.eclipse.jetty.websocket:websocket-common:jar
+ org.eclipse.jetty.websocket:websocket-server:jar
+ org.eclipse.jetty.websocket:websocket-servlet:jar
+ org.fusesource.leveldbjni:leveldbjni-all:jar
+ org.glassfish.hk2:class-model:jar
+ org.glassfish.hk2:config-types:jar
+ org.glassfish.hk2.external:aopalliance-repackaged:jar
+ org.glassfish.hk2.external:asm-all-repackaged:jar
+ org.glassfish.hk2.external:bean-validator:jar
+ org.glassfish.hk2.external:javax.inject:jar
+ org.glassfish.hk2:hk2:jar
+ org.glassfish.hk2:hk2-api:jar
+ org.glassfish.hk2:hk2-config:jar
+ org.glassfish.hk2:hk2-core:jar
+ org.glassfish.hk2:hk2-locator:jar
+ org.glassfish.hk2:hk2-runlevel:jar
+ org.glassfish.hk2:hk2-utils:jar
+ org.glassfish.hk2:osgi-resource-locator:jar
+ org.glassfish.hk2:spring-bridge:jar
+ org.glassfish.jersey.bundles:jaxrs-ri:jar
+ org.glassfish.jersey.bundles.repackaged:jersey-guava:jar
+ org.glassfish.jersey.containers:jersey-container-servlet:jar
+ org.glassfish.jersey.containers:jersey-container-servlet-core:jar
+ org.glassfish.jersey.core:jersey-client:jar
+ org.glassfish.jersey.core:jersey-common:jar
+ org.glassfish.jersey.core:jersey-server:jar
+ org.glassfish.jersey.ext:jersey-entity-filtering:jar
+ org.glassfish.jersey.ext:jersey-spring3:jar
+ org.glassfish.jersey.media:jersey-media-jaxb:jar
+ org.glassfish.jersey.media:jersey-media-json-jackson:jar
+ org.glassfish.jersey.media:jersey-media-multipart:jar
+ org.hdrhistogram:HdrHistogram:jar
+ org.javassist:javassist:jar
+ org.json4s:json4s-ast_2.11:jar
+ org.json4s:json4s-core_2.11:jar
+ org.json4s:json4s-jackson_2.11:jar
+ org.jsoup:jsoup:jar
+ org.jvnet.mimepull:mimepull:jar
+ org.jvnet:tiger-types:jar
+ org.latencyutils:LatencyUtils:jar
+ org.mortbay.jasper:apache-el:jar
+ org.mortbay.jetty:jetty:jar
+ org.mortbay.jetty:jetty-util:jar
+ org.ow2.asm:asm-analysis:jar
+ org.ow2.asm:asm-commons:jar
+ org.ow2.asm:asm-tree:jar
+ org.reflections:reflections:jar
+ org.scala-lang.modules:scala-parser-combinators_2.11:jar
+ org.scala-lang.modules:scala-xml_2.11:jar
+ org.scala-lang:scala-compiler:jar
+ org.scala-lang:scala-library:jar
+ org.scala-lang:scala-reflect:jar
+ org.scala-lang:scalap:jar
+ org.slf4j:jul-to-slf4j:jar
+ org.slf4j:slf4j-api:jar
+ org.springframework.boot:spring-boot:jar
+ org.springframework.boot:spring-boot-actuator:jar
+ org.springframework.boot:spring-boot-actuator-autoconfigure:jar
+ org.springframework.boot:spring-boot-autoconfigure:jar
+ org.springframework.boot:spring-boot-starter:jar
+ org.springframework.boot:spring-boot-starter-actuator:jar
+ org.springframework.boot:spring-boot-starter-aop:jar
+ org.springframework.boot:spring-boot-starter-jetty:jar
+ org.springframework.boot:spring-boot-starter-json:jar
+ org.springframework.boot:spring-boot-starter-log4j2:jar
+ org.springframework.boot:spring-boot-starter-web:jar
+ org.springframework.cloud:spring-cloud-commons:jar
+ org.springframework.cloud:spring-cloud-config-client:jar
+ org.springframework.cloud:spring-cloud-context:jar
+ org.springframework.cloud:spring-cloud-netflix-archaius:jar
+ org.springframework.cloud:spring-cloud-netflix-core:jar
+ org.springframework.cloud:spring-cloud-netflix-eureka-client:jar
+ org.springframework.cloud:spring-cloud-netflix-ribbon:jar
+ org.springframework.cloud:spring-cloud-starter:jar
+ org.springframework.cloud:spring-cloud-starter-config:jar
+ org.springframework.cloud:spring-cloud-starter-eureka:jar
+ org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar
+ org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar
+ org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar
+ org.springframework.security:spring-security-crypto:jar
+ org.springframework.security:spring-security-rsa:jar
+ org.springframework:spring-aop:jar
+ org.springframework:spring-beans:jar
+ org.springframework:spring-context:jar
+ org.springframework:spring-core:jar
+ org.springframework:spring-expression:jar
+ org.springframework:spring-jcl:jar
+ org.springframework:spring-web:jar
+ org.springframework:spring-webmvc:jar
+ org.tukaani:xz:jar
+ org.yaml:snakeyaml:jar
+ software.amazon.ion:ion-java:jar
+ xerces:xercesImpl:jar
+ xmlenc:xmlenc:jar
+ xmlpull:xmlpull:jar
+ xpp3:xpp3_min:jar
+
+
+
+
+
+
+ ${basedir}/src/main/resources
+
+ *
+
+ 0777
+ conf
+ unix
+
+
+
+
+
+
diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/LinkisEntranceApplication.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/LinkisEntranceApplication.java
new file mode 100644
index 0000000000..ac80d687bb
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/LinkisEntranceApplication.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance;
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+
+
+public class LinkisEntranceApplication {
+
+ private static final Log logger = LogFactory.getLog(DataWorkCloudApplication.class);
+
+ public static void main(String[] args) throws ReflectiveOperationException {
+ logger.info("Start to running LinkisManagerApplication");
+ DataWorkCloudApplication.main(args);
+ }
+}
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/BackGroundServiceBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/BackGroundServiceBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/BackGroundServiceBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/BackGroundServiceBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ConsumerManagerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ConsumerManagerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ConsumerManagerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ConsumerManagerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineBuilderBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineBuilderBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineBuilderBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineBuilderBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineManagerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineManagerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineManagerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineManagerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineRequesterBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineRequesterBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineRequesterBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineRequesterBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineSelectorBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineSelectorBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineSelectorBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EngineSelectorBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceContextBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceContextBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceContextBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceContextBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
similarity index 97%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
index 902733be5f..38659f6e1f 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
@@ -35,7 +35,7 @@
@Bean(value = EntranceExecutorManagerBeanAnnotation.BEAN_NAME)
@Component(value = EntranceExecutorManagerBeanAnnotation.BEAN_NAME)
public @interface EntranceExecutorManagerBeanAnnotation {
- String BEAN_NAME = "entranceExecutorManager";
+ String BEAN_NAME = "executorManager";
@AliasFor(annotation = Component.class)
String value() default BEAN_NAME;
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceInterceptorBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceInterceptorBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceInterceptorBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceInterceptorBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceListenerBusBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceListenerBusBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceListenerBusBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceListenerBusBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceParserBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceParserBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceParserBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceParserBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceServerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceServerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceServerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceServerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeListenerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeListenerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeListenerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeListenerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeManagerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeManagerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeManagerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ErrorCodeManagerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/GroupFactoryBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/GroupFactoryBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/GroupFactoryBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/GroupFactoryBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/LogManagerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/LogManagerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/LogManagerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/LogManagerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/NewEngineBroadcastListenerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/NewEngineBroadcastListenerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/NewEngineBroadcastListenerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/NewEngineBroadcastListenerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceEngineBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceEngineBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceEngineBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceEngineBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceManagerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceManagerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceManagerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/PersistenceManagerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ReceiverChooserBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ReceiverChooserBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ReceiverChooserBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ReceiverChooserBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResponseEngineStatusChangedBroadcastListenerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResponseEngineStatusChangedBroadcastListenerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResponseEngineStatusChangedBroadcastListenerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResponseEngineStatusChangedBroadcastListenerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResultSetEngineBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResultSetEngineBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResultSetEngineBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/ResultSetEngineBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerContextBeanAnnotation.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerContextBeanAnnotation.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerContextBeanAnnotation.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/SchedulerContextBeanAnnotation.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
similarity index 84%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
index 360e8ffa1f..d9e27e299c 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,15 +13,15 @@
package com.webank.wedatasphere.linkis.entrance.conf;
-import com.webank.wedatasphere.linkis.entrance.EntranceContext;
import com.webank.wedatasphere.linkis.entrance.EntranceParser;
+import com.webank.wedatasphere.linkis.entrance.EntranceServer;
import com.webank.wedatasphere.linkis.entrance.annotation.*;
import com.webank.wedatasphere.linkis.entrance.background.BackGroundService;
import com.webank.wedatasphere.linkis.entrance.event.EntranceEvent;
import com.webank.wedatasphere.linkis.entrance.event.EntranceEventListener;
import com.webank.wedatasphere.linkis.entrance.event.EntranceEventListenerBus;
-import com.webank.wedatasphere.linkis.entrance.execute.*;
-import com.webank.wedatasphere.linkis.entrance.execute.impl.*;
+import com.webank.wedatasphere.linkis.entrance.execute.EntranceExecutionService;
+import com.webank.wedatasphere.linkis.entrance.execute.impl.EntranceExecutorManagerImpl;
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor;
import com.webank.wedatasphere.linkis.entrance.interceptor.impl.*;
import com.webank.wedatasphere.linkis.entrance.log.*;
@@ -32,26 +29,29 @@
import com.webank.wedatasphere.linkis.entrance.persistence.*;
import com.webank.wedatasphere.linkis.entrance.scheduler.EntranceGroupFactory;
import com.webank.wedatasphere.linkis.entrance.scheduler.EntranceSchedulerContext;
-import com.webank.wedatasphere.linkis.rpc.ReceiverChooser;
+import com.webank.wedatasphere.linkis.orchestrator.ecm.EngineConnManagerBuilder;
+import com.webank.wedatasphere.linkis.orchestrator.ecm.EngineConnManagerBuilder$;
+import com.webank.wedatasphere.linkis.orchestrator.ecm.entity.Policy;
import com.webank.wedatasphere.linkis.scheduler.Scheduler;
import com.webank.wedatasphere.linkis.scheduler.SchedulerContext;
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorManager;
import com.webank.wedatasphere.linkis.scheduler.queue.ConsumerManager;
import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory;
import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager;
import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.AutoConfigureBefore;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Configuration;
import static com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration.ENTRANCE_SCHEDULER_MAX_PARALLELISM_USERS;
/**
- * created by enjoyyin on 2018/10/16
* Description:This configuration class is used to generate some singleton classes in the entity module.(该配置类用于生成entrance模块中的一些单例类)
*/
@Configuration
+@AutoConfigureBefore({EntranceServer.class, EntranceExecutionService.class})
public class EntranceSpringConfiguration {
private Logger logger = LoggerFactory.getLogger(getClass());
@@ -59,11 +59,7 @@ public class EntranceSpringConfiguration {
logger.info("load the ujes-entrance spring configuration.");
}
- @EntranceParserBeanAnnotation
- @ConditionalOnMissingBean(name = {EntranceParserBeanAnnotation.BEAN_NAME})
- public EntranceParser generateEntranceParser(){
- return new CommonEntranceParser();
- }
+
@PersistenceEngineBeanAnnotation
@ConditionalOnMissingBean(name = {PersistenceEngineBeanAnnotation.BEAN_NAME})
@@ -81,12 +77,18 @@ public ResultSetEngine generateResultSetEngine(){
@ConditionalOnMissingBean(name = {PersistenceManagerBeanAnnotation.BEAN_NAME})
public PersistenceManager generatePersistenceManager(@PersistenceEngineBeanAnnotation.PersistenceEngineAutowiredAnnotation PersistenceEngine persistenceEngine,
@ResultSetEngineBeanAnnotation.ResultSetEngineAutowiredAnnotation ResultSetEngine resultSetEngine){
+ logger.info("init PersistenceManager.");
QueryPersistenceManager persistenceManager = new QueryPersistenceManager();
persistenceManager.setPersistenceEngine(persistenceEngine);
persistenceManager.setResultSetEngine(resultSetEngine);
return persistenceManager;
}
+ @EntranceParserBeanAnnotation
+ @ConditionalOnMissingBean(name = {EntranceParserBeanAnnotation.BEAN_NAME})
+ public EntranceParser generateEntranceParser(){
+ return new CommonEntranceParser();
+ }
@EntranceListenerBusBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceListenerBusBeanAnnotation.BEAN_NAME})
@@ -104,7 +106,7 @@ public EntranceEventListenerBus generateEn
@EntranceInterceptorBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceInterceptorBeanAnnotation.BEAN_NAME})
public EntranceInterceptor[] generateEntranceInterceptors() {
- return new EntranceInterceptor[]{new CSEntranceInterceptor(), new PythonCodeCheckInterceptor(), new DBInfoCompleteInterceptor(), new SparkCodeCheckInterceptor(),
+ return new EntranceInterceptor[]{new CSEntranceInterceptor(), new PythonCodeCheckInterceptor(), new DBInfoCompleteInterceptor(), new SparkCodeCheckInterceptor(),
new SQLCodeCheckInterceptor(), new VarSubstitutionInterceptor(), new LogPathCreateInterceptor(),
new StorePathEntranceInterceptor(), new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(), new CommentInterceptor(),
};
@@ -152,18 +154,18 @@ public ConsumerManager generateConsumerManager(){
@SchedulerContextBeanAnnotation
@ConditionalOnMissingBean(name = {SchedulerContextBeanAnnotation.BEAN_NAME})
public SchedulerContext generateSchedulerContext(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory,
- @EntranceExecutorManagerBeanAnnotation.EntranceExecutorManagerAutowiredAnnotation EntranceExecutorManager executorManager,
+ @EntranceExecutorManagerBeanAnnotation.EntranceExecutorManagerAutowiredAnnotation ExecutorManager executorManager,
@ConsumerManagerBeanAnnotation.ConsumerManagerAutowiredAnnotation ConsumerManager consumerManager) {
return new EntranceSchedulerContext(groupFactory, consumerManager, executorManager);
}
- @EngineRequesterBeanAnnotation
+ /* @EngineRequesterBeanAnnotation
@ConditionalOnMissingBean(name = {EngineRequesterBeanAnnotation.BEAN_NAME})
public EngineRequester generateEngineRequester(){
return new EngineRequesterImpl();
}
-
- @EngineSelectorBeanAnnotation
+*/
+ /* @EngineSelectorBeanAnnotation
@ConditionalOnMissingBean(name = {EngineSelectorBeanAnnotation.BEAN_NAME})
public EngineSelector generateEngineSelector(@EntranceListenerBusBeanAnnotation.EntranceListenerBusAutowiredAnnotation
EntranceEventListenerBus entranceEventListenerBus) {
@@ -171,8 +173,8 @@ public EngineSelector generateEngineSelector(@EntranceListenerBusBeanAnnotation.
singleEngineSelector.setEntranceEventListenerBus(entranceEventListenerBus);
return singleEngineSelector;
}
-
- @EngineBuilderBeanAnnotation
+*/
+ /*@EngineBuilderBeanAnnotation
@ConditionalOnMissingBean(name = {EngineBuilderBeanAnnotation.BEAN_NAME})
public EngineBuilder generateEngineBuilder(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory) {
return new AbstractEngineBuilder(groupFactory) {
@@ -181,49 +183,39 @@ public EntranceEngine createEngine(long id) {
return new SingleEntranceEngine(id);
}
};
- }
+ }*/
- @EngineManagerBeanAnnotation
+ /* @EngineManagerBeanAnnotation
@ConditionalOnMissingBean(name = {EngineManagerBeanAnnotation.BEAN_NAME})
public EngineManager generateEngineManager() {
return new EngineManagerImpl();
- }
+ }*/
@EntranceExecutorManagerBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceExecutorManagerBeanAnnotation.BEAN_NAME})
- public EntranceExecutorManager generateExecutorManager(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory,
- @EngineBuilderBeanAnnotation.EngineBuilderAutowiredAnnotation EngineBuilder engineBuilder,
- @EngineRequesterBeanAnnotation.EngineRequesterAutowiredAnnotation EngineRequester engineRequester,
- @EngineSelectorBeanAnnotation.EngineSelectorAutowiredAnnotation EngineSelector engineSelector,
- @EngineManagerBeanAnnotation.EngineManagerAutowiredAnnotation EngineManager engineManager,
- @Autowired EntranceExecutorRuler[] entranceExecutorRulers){
- return new EntranceExecutorManagerImpl(groupFactory, engineBuilder,
- engineRequester, engineSelector, engineManager, entranceExecutorRulers);
+ public ExecutorManager generateExecutorManager(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory) {
+ EngineConnManagerBuilder engineConnManagerBuilder = EngineConnManagerBuilder$.MODULE$.builder();
+ engineConnManagerBuilder.setPolicy(Policy.Process);
+ return new EntranceExecutorManagerImpl(groupFactory, engineConnManagerBuilder.build());
}
@SchedulerBeanAnnotation
@ConditionalOnMissingBean(name = {SchedulerBeanAnnotation.BEAN_NAME})
- public Scheduler generateScheduler(@SchedulerContextBeanAnnotation.SchedulerContextAutowiredAnnotation SchedulerContext schedulerContext){
+ public Scheduler generateScheduler(@SchedulerContextBeanAnnotation.SchedulerContextAutowiredAnnotation SchedulerContext schedulerContext) {
Scheduler scheduler = new ParallelScheduler(schedulerContext);
scheduler.init();
scheduler.start();
return scheduler;
}
- @ReceiverChooserBeanAnnotation
- @ConditionalOnMissingBean(name = {ReceiverChooserBeanAnnotation.BEAN_NAME})
- public ReceiverChooser generateEntranceReceiverChooser(@EntranceContextBeanAnnotation.EntranceContextAutowiredAnnotation
- EntranceContext entranceContext) {
- return new EntranceReceiverChooser(entranceContext);
- }
@BackGroundServiceBeanAnnotation
@ConditionalOnMissingBean(name = {BackGroundServiceBeanAnnotation.BEAN_NAME})
- public BackGroundService[] generateBackGroundService(){
+ public BackGroundService[] generateBackGroundService() {
return new BackGroundService[]{};
}
- @NewEngineBroadcastListenerBeanAnnotation
+ /* @NewEngineBroadcastListenerBeanAnnotation
@ConditionalOnMissingBean(name = {NewEngineBroadcastListenerBeanAnnotation.BEAN_NAME})
public NewEngineBroadcastListener generateNewEngineBroadcastListener(@EntranceExecutorManagerBeanAnnotation.EntranceExecutorManagerAutowiredAnnotation
EntranceExecutorManager entranceExecutorManager) {
@@ -239,5 +231,5 @@ public ResponseEngineStatusChangedBroadcastListener generateResponseEngineStatus
ResponseEngineStatusChangedBroadcastListener broadcastListener = new ResponseEngineStatusChangedBroadcastListener();
broadcastListener.setEntranceExecutorManager(entranceExecutorManager);
return broadcastListener;
- }
+ }*/
}
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/context/DefaultEntranceContext.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/context/DefaultEntranceContext.java
similarity index 92%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/context/DefaultEntranceContext.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/context/DefaultEntranceContext.java
index e0ad6babb9..7bb61e9f0c 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/context/DefaultEntranceContext.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/context/DefaultEntranceContext.java
@@ -29,16 +29,18 @@
import com.webank.wedatasphere.linkis.entrance.event.EntranceEvent;
import com.webank.wedatasphere.linkis.entrance.event.EntranceEventListener;
import com.webank.wedatasphere.linkis.entrance.event.EntranceEventListenerBus;
-import com.webank.wedatasphere.linkis.entrance.execute.EntranceExecutorManager;
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor;
import com.webank.wedatasphere.linkis.entrance.log.LogManager;
import com.webank.wedatasphere.linkis.entrance.persistence.PersistenceManager;
import com.webank.wedatasphere.linkis.scheduler.Scheduler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.annotation.PostConstruct;
@EntranceContextBeanAnnotation
public class DefaultEntranceContext extends EntranceContext {
+ private static Logger logger = LoggerFactory.getLogger(DefaultEntranceContext.class);
@EntranceParserBeanAnnotation.EntranceParserAutowiredAnnotation
private EntranceParser entranceParser;
@@ -79,11 +81,12 @@ public DefaultEntranceContext() {
@PostConstruct
public void init() {
entranceParser.setEntranceContext(this);
+ logger.info("Finished init entranceParser from postConstruct end!");
persistenceManager.setEntranceContext(this);
logManager.setEntranceContext(this);
- if(scheduler.getSchedulerContext().getOrCreateExecutorManager() instanceof EntranceExecutorManager) {
+ /* if(scheduler.getSchedulerContext().getOrCreateExecutorManager() instanceof EntranceExecutorManager) {
listenerBus.addListener(((EntranceExecutorManager) scheduler.getSchedulerContext().getOrCreateExecutorManager()).getOrCreateEngineManager());
- }
+ }*/
}
@Override
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorCode.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorCode.java
similarity index 86%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorCode.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorCode.java
index dbf48ab111..e6b144283d 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorCode.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorCode.java
@@ -27,7 +27,10 @@ public enum EntranceErrorCode {
*
*/
CACHE_NOT_READY(200, "shared cache not ready"),
- ENTRANCE_CAST_FAIL(20002, "class cast failed")
+ ENTRANCE_CAST_FAIL(20002, "class cast failed"),
+ PARAM_CANNOT_EMPTY(20008, "params cannot be empty "),
+ LABEL_PARAMS_INVALID(20009, "Label params invalid. ")
+
;
private int errCode;
private String desc;
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
similarity index 84%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
index 82da5b66b5..8497c9edc1 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
@@ -18,15 +18,14 @@
import com.webank.wedatasphere.linkis.common.exception.ErrorException;
-/**
- * created by enjoyyin on 2018/10/8
- * Description:
- */
+
public class EntranceIllegalParamException extends ErrorException {
public EntranceIllegalParamException(int errCode, String message){
super(errCode, message);
}
-
+ public EntranceIllegalParamException(EntranceErrorCode entranceErrorCode) {
+ super(entranceErrorCode.getErrCode(), entranceErrorCode.getDesc());
+ }
}
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceRPCException.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceRPCException.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceRPCException.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceRPCException.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/QueryFailedException.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/QueryFailedException.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/QueryFailedException.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/QueryFailedException.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
similarity index 92%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
index 1f5585b500..1a7c36b1c3 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
@@ -21,8 +21,9 @@
import com.webank.wedatasphere.linkis.entrance.execute.*;
import com.webank.wedatasphere.linkis.entrance.log.*;
import com.webank.wedatasphere.linkis.entrance.persistence.HaPersistenceTask;
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask;
+import com.webank.wedatasphere.linkis.manager.label.entity.Label;
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant;
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask;
import com.webank.wedatasphere.linkis.protocol.task.Task;
import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils;
import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest;
@@ -37,12 +38,10 @@
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.List;
import java.util.Map;
-/**
- * created by enjoyyin on 2018/10/17
- * Description:
- */
+
public class EntranceExecutionJob extends EntranceJob implements LogHandler {
private LogReader logReader;
@@ -51,14 +50,15 @@ public class EntranceExecutionJob extends EntranceJob implements LogHandler {
private WebSocketLogWriter webSocketLogWriter;
private static final Logger logger = LoggerFactory.getLogger(EntranceExecutionJob.class);
- public class EntranceExecuteRequest implements ExecuteRequest, LockExecuteRequest, JobExecuteRequest, RuntimePropertiesExecuteRequest {
+ public class EntranceExecuteRequest implements ExecuteRequest, LabelExecuteRequest, JobExecuteRequest, RuntimePropertiesExecuteRequest {
private String executionCode;
- public void setExecutionCode(){
+
+ public void setExecutionCode() {
Task task = getTask();
- if (task instanceof RequestPersistTask){
- this.executionCode = ((RequestPersistTask) task).getExecutionCode();
+ if (task instanceof RequestPersistTask) {
+ this.executionCode = ((RequestPersistTask) task).getExecutionCode();
}
}
@@ -67,24 +67,29 @@ public String code() {
return this.executionCode;
}
- @Override
- public String lock() {
- return getLock();
- }
-
@Override
public String jobId() {
return getId();
}
+
@Override
public Map properties() {
Map properties = TaskUtils.getRuntimeMap(getParams());
- if(getTask() instanceof RequestPersistTask) {
+ if (getTask() instanceof RequestPersistTask) {
properties.put(TaskConstant.RUNTYPE, ((RequestPersistTask) getTask()).getRunType());
}
return properties;
}
+
+ @Override
+ public List> labels() {
+ Task task = getTask();
+ if (task instanceof RequestPersistTask) {
+ return ((RequestPersistTask) task).getLabels();
+ }
+ return null;
+ }
}
public class StorePathEntranceExecuteRequest extends EntranceExecuteRequest implements StorePathExecuteRequest {
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
similarity index 79%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
index 78a2777917..c911e481c3 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
@@ -21,19 +21,19 @@
import com.webank.wedatasphere.linkis.entrance.exception.EntranceIllegalParamException;
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob;
import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob;
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask;
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask;
+import com.webank.wedatasphere.linkis.governance.common.utils.GovernanceConstant;
import com.webank.wedatasphere.linkis.protocol.task.Task;
+import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils;
import com.webank.wedatasphere.linkis.scheduler.queue.Job;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Date;
+import java.util.Map;
+
-/**
- * created by enjoyyin on 2018/10/11
- * Description:
- */
public abstract class AbstractEntranceParser extends EntranceParser {
private EntranceContext entranceContext;
@@ -61,26 +61,21 @@ public Task parseToTask(Job job) throws EntranceIllegalParamException{
if (job == null){
throw new EntranceIllegalParamException(20002, "job can't be null");
}
- Task task = ((EntranceJob)job).getTask();
+ Task task = ((EntranceJob) job).getTask();
if(StringUtils.isEmpty(task.getExecId())) {
task.setExecId(job.getId());
}
- if (task instanceof RequestPersistTask){
+ if (task instanceof RequestPersistTask) {
((RequestPersistTask) task).setProgress(job.getProgress());
((RequestPersistTask) task).setStatus(job.getState().toString());
((RequestPersistTask) task).setUpdatedTime(new Date(System.currentTimeMillis()));
((RequestPersistTask) task).setProgress(job.getProgress());
- if(job.isCompleted() && !job.isSucceed() && job.getErrorResponse() != null
+ if (job.isCompleted() && !job.isSucceed() && job.getErrorResponse() != null
&& StringUtils.isBlank(((RequestPersistTask) task).getErrDesc())
- && StringUtils.isNotEmpty(job.getErrorResponse().message())) {
+ && StringUtils.isNotEmpty(job.getErrorResponse().message())) {
((RequestPersistTask) task).setErrDesc(job.getErrorResponse().message());
}
- //if job is successful, errCode and errDesc needs to be null
- if (job.isSucceed()){
- ((RequestPersistTask) task).setErrCode(null);
- ((RequestPersistTask) task).setErrDesc(null);
- }
- }else{
+ } else {
logger.warn("not supported task type");
}
return task;
@@ -92,25 +87,27 @@ protected EntranceJob createEntranceJob() {
/**
* Parse a task into an executable job(将一个task解析成一个可执行的job)
+ *
* @param task
* @return
*/
@Override
public Job parseToJob(Task task) throws EntranceIllegalParamException {
- if (task == null){
+ if (task == null) {
throw new EntranceIllegalParamException(20001, "task can't be null");
}
EntranceJob job = null;
- if (task instanceof RequestPersistTask){
+ if (task instanceof RequestPersistTask) {
job = createEntranceJob();
job.setTask(task);
job.setUser(((RequestPersistTask) task).getUmUser());
job.setCreator(((RequestPersistTask) task).getRequestApplicationName());
+
job.setParams(((RequestPersistTask) task).getParams());
- //job.setLogListener(entranceContext.getOrCreateLogManager());
- //job.setProgressListener(entranceContext.getOrCreatePersistenceManager());
- //job.setJobListener(entranceContext.getOrCreatePersistenceManager());
+ //TODO 放置source到RequestTask的properties中，后续会进行优化
+ Map properties = TaskUtils.getRuntimeMap(job.getParams());
+ properties.put(GovernanceConstant.TASK_SOURCE_MAP_KEY(), ((RequestPersistTask) task).getSource());
job.setEntranceListenerBus(entranceContext.getOrCreateEventListenerBus());
job.setEntranceContext(entranceContext);
job.setListenerEventBus(null);
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/ParserUtils.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/ParserUtils.java
similarity index 95%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/ParserUtils.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/ParserUtils.java
index bc3aff72ab..b1816e298f 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/ParserUtils.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/ParserUtils.java
@@ -17,7 +17,7 @@
package com.webank.wedatasphere.linkis.entrance.parser;
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration$;
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask;
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask;
import com.webank.wedatasphere.linkis.protocol.task.Task;
import org.apache.commons.lang.StringUtils;
@@ -25,10 +25,7 @@
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
-/**
- * created by enjoyyin on 2018/10/17
- * Description:
- */
+
public final class ParserUtils {
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/AbstractPersistenceEngine.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/AbstractPersistenceEngine.java
similarity index 100%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/AbstractPersistenceEngine.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/AbstractPersistenceEngine.java
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
similarity index 95%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
index 67daab244a..80ff979281 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
@@ -23,13 +23,12 @@
package com.webank.wedatasphere.linkis.entrance.persistence;
import com.google.gson.Gson;
-import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration;
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration$;
import com.webank.wedatasphere.linkis.entrance.exception.EntranceIllegalParamException;
import com.webank.wedatasphere.linkis.entrance.exception.EntranceRPCException;
import com.webank.wedatasphere.linkis.entrance.exception.QueryFailedException;
+import com.webank.wedatasphere.linkis.governance.common.entity.task.*;
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant;
-import com.webank.wedatasphere.linkis.protocol.query.*;
import com.webank.wedatasphere.linkis.protocol.task.Task;
import com.webank.wedatasphere.linkis.rpc.Sender;
import org.slf4j.Logger;
@@ -49,7 +48,7 @@ public class QueryPersistenceEngine extends AbstractPersistenceEngine{
public QueryPersistenceEngine(){
/*
- Get the corresponding sender through datawork-cloud-publicservice(通过datawork-cloud-publicservice 拿到对应的sender)
+ Get the corresponding sender through datawork-linkis-publicservice(通过datawork-linkis-publicservice 拿到对应的sender)
*/
sender = Sender.getSender(EntranceConfiguration$.MODULE$.QUERY_PERSISTENCE_SPRING_APPLICATION_NAME().getValue());
}
@@ -84,7 +83,7 @@ public void persist(Task task) throws QueryFailedException, EntranceIllegalParam
throw new QueryFailedException(20011, "insert task failed, reason: " + message);
}
String taskStr = object.toString();
- Long taskID = Long.parseLong(taskStr.substring(0,taskStr.indexOf(".")));
+ Long taskID = Long.parseLong(taskStr);
((RequestPersistTask) task).setTaskID(taskID);
}
}else{
@@ -116,7 +115,7 @@ public Task retrieve(Long taskID)throws EntranceIllegalParamException, QueryFail
logger.error("By taskID: {} request the corresponding task return status code is not 0, the query fails(通过taskID: {} 请求相应的task返回状态码不为0，查询失败)", taskID);
throw new QueryFailedException(20010, "retrieve task failed, reason: " + message);
}
- java.util.Map data = responsePersist.getData();
+ Map data = responsePersist.getData();
if (data != null){
Object object = data.get(TaskConstant.TASK);
if (object instanceof List){
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
similarity index 92%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
index a73cd8d8e8..ebfed2bff1 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,8 +24,8 @@
import com.webank.wedatasphere.linkis.entrance.EntranceContext;
import com.webank.wedatasphere.linkis.entrance.cs.CSEntranceHelper;
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob;
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask;
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo;
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask;
import com.webank.wedatasphere.linkis.protocol.task.Task;
import com.webank.wedatasphere.linkis.scheduler.executer.OutputExecuteResponse;
import com.webank.wedatasphere.linkis.scheduler.queue.Job;
@@ -84,7 +81,7 @@ public void onResultSetCreated(Job job, OutputExecuteResponse response) {
path = createResultSetEngine().persistResultSet(job, response);
} catch (Throwable e) {
job.onFailure("persist resultSet failed!", e);
- if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersisted();
+ if (isEntranceJob) ((EntranceJob) job).incrementResultSetPersisted();
return;
}
if(StringUtils.isNotBlank(path)) {
@@ -99,7 +96,7 @@ public void onResultSetCreated(Job job, OutputExecuteResponse response) {
logger.error("job {} onLogUpdate error, reason:", job.getId(), e1);
} //ignore it
if(isEntranceJob) {
- ((EntranceJob)job).incrementResultSetPersisted();
+ ((EntranceJob) job).incrementResultSetPersisted();
}
return;
}
@@ -108,7 +105,7 @@ public void onResultSetCreated(Job job, OutputExecuteResponse response) {
if(StringUtils.isEmpty(requestPersistTask.getResultLocation())) synchronized (task) {
if(StringUtils.isNotEmpty(requestPersistTask.getResultLocation())) {
if(isEntranceJob) {
- ((EntranceJob)job).incrementResultSetPersisted();
+ ((EntranceJob) job).incrementResultSetPersisted();
}
return;
}
@@ -122,13 +119,13 @@ public void onResultSetCreated(Job job, OutputExecuteResponse response) {
}
}
if(isEntranceJob) {
- ((EntranceJob)job).incrementResultSetPersisted();
+ ((EntranceJob) job).incrementResultSetPersisted();
}
}
@Override
public void onResultSizeCreated(Job job, int resultSize) {
- if(job instanceof EntranceJob) {
+ if (job instanceof EntranceJob) {
((EntranceJob) job).setResultSize(resultSize);
}
}
@@ -136,9 +133,6 @@ public void onResultSizeCreated(Job job, int resultSize) {
@Override
public void onProgressUpdate(Job job, float progress, JobProgressInfo[] progressInfo) {
job.setProgress(progress);
- if (job instanceof EntranceJob){
- ((EntranceJob) job).setProgressInfo(progressInfo);
- }
updateJobStatus(job);
}
@@ -162,7 +156,7 @@ public void onJobWaitForRetry(Job job) {
@Override
public void onJobCompleted(Job job) {
- //update by peaceWong(2020/05/10) to set jobID to CS
+ //update by peaceWong to set jobID to CS
try {
if (job.isSucceed()) {
CSEntranceHelper.registerCSRSData(job);
@@ -170,7 +164,6 @@ public void onJobCompleted(Job job) {
} catch (Throwable e) {
logger.error("Failed to register cs rs data ", e);
}
- //end update
updateJobStatus(job);
}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java
new file mode 100644
index 0000000000..93b2c3264a
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java
@@ -0,0 +1,405 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.restful;
+
+import com.google.gson.Gson;
+import com.google.gson.internal.LinkedTreeMap;
+import com.webank.wedatasphere.linkis.common.log.LogUtils;
+import com.webank.wedatasphere.linkis.entrance.EntranceServer;
+import com.webank.wedatasphere.linkis.entrance.annotation.EntranceServerBeanAnnotation;
+import com.webank.wedatasphere.linkis.entrance.background.BackGroundService;
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration;
+import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob;
+import com.webank.wedatasphere.linkis.entrance.log.LogReader;
+import com.webank.wedatasphere.linkis.entrance.utils.JobHistoryHelper;
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask;
+import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant;
+import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo;
+import com.webank.wedatasphere.linkis.protocol.task.Task;
+import com.webank.wedatasphere.linkis.protocol.utils.ZuulEntranceUtils;
+import com.webank.wedatasphere.linkis.rpc.Sender;
+import com.webank.wedatasphere.linkis.scheduler.queue.Job;
+import com.webank.wedatasphere.linkis.server.Message;
+import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
+import com.webank.wedatasphere.linkis.server.socket.controller.ServerEvent;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+import scala.Option;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.*;
+
+/**
+ * Description: an implementation class of EntranceRestfulRemote
+ */
+@Path("/entrance")
+@Component
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class EntranceRestfulApi implements EntranceRestfulRemote {
+
+ private EntranceServer entranceServer;
+
+ private static final Logger logger = LoggerFactory.getLogger(EntranceRestfulApi.class);
+
+ @EntranceServerBeanAnnotation.EntranceServerAutowiredAnnotation
+ public void setEntranceServer(EntranceServer entranceServer) {
+ this.entranceServer = entranceServer;
+ }
+
+ /**
+ * The execute function handles the request submitted by the user to execute the task, and the execution ID is returned to the user.
+ * execute函数处理的是用户提交执行任务的请求，返回给用户的是执行ID
+ * json Incoming key-value pair(传入的键值对)
+ * Repsonse
+ */
+ @Override
+ @POST
+ @Path("/execute")
+ public Response execute(@Context HttpServletRequest req, Map json) {
+ Message message = null;
+// try{
+ logger.info("Begin to get an execID");
+ json.put(TaskConstant.UMUSER, SecurityFilter.getLoginUsername(req));
+ String execID = entranceServer.execute(json);
+ Job job = entranceServer.getJob(execID).get();
+ Task task = ((EntranceJob) job).getTask();
+ Long taskID = ((RequestPersistTask) task).getTaskID();
+ pushLog(LogUtils.generateInfo("You have submitted a new job, script code (after variable substitution) is"), job);
+ pushLog("************************************SCRIPT CODE************************************", job);
+ pushLog(((RequestPersistTask) task).getCode(), job);
+ pushLog("************************************SCRIPT CODE************************************", job);
+ pushLog(LogUtils.generateInfo("Your job is accepted, jobID is " + execID + " and taskID is " + taskID + ". Please wait it to be scheduled"), job);
+ execID = ZuulEntranceUtils.generateExecID(execID, Sender.getThisServiceInstance().getApplicationName(), new String[]{Sender.getThisInstance()});
+ message = Message.ok();
+ message.setMethod("/api/entrance/execute");
+ message.data("execID", execID);
+ message.data("taskID", taskID);
+ logger.info("End to get an an execID: {}, taskID: {}", execID, taskID);
+// }catch(ErrorException e){
+// message = Message.error(e.getDesc());
+// message.setStatus(1);
+// message.setMethod("/api/entrance/execute");
+// }
+ return Message.messageToResponse(message);
+
+ }
+
+ @Override
+ @POST
+ @Path("/submit")
+ public Response submit(@Context HttpServletRequest req, Map json) {
+ Message message = null;
+ logger.info("Begin to get an execID");
+ json.put(TaskConstant.SUBMIT_USER, SecurityFilter.getLoginUsername(req));
+ String execID = entranceServer.execute(json);
+ Job job = entranceServer.getJob(execID).get();
+ Task task = ((EntranceJob) job).getTask();
+ Long taskID = ((RequestPersistTask) task).getTaskID();
+ pushLog(LogUtils.generateInfo("You have submitted a new job, script code (after variable substitution) is"), job);
+ pushLog("************************************SCRIPT CODE************************************", job);
+ pushLog(((RequestPersistTask) task).getCode(), job);
+ pushLog("************************************SCRIPT CODE************************************", job);
+ pushLog(LogUtils.generateInfo("Your job is accepted, jobID is " + execID + " and taskID is " + taskID + ". Please wait it to be scheduled"), job);
+ execID = ZuulEntranceUtils.generateExecID(execID, Sender.getThisServiceInstance().getApplicationName(), new String[]{Sender.getThisInstance()});
+ message = Message.ok();
+ message.setMethod("/api/entrance/submit");
+ message.data("execID", execID);
+ message.data("taskID", taskID);
+ logger.info("End to get an an execID: {}, taskID: {}", execID, taskID);
+ return Message.messageToResponse(message);
+ }
+
+ private void pushLog(String log, Job job) {
+ entranceServer.getEntranceContext().getOrCreateLogManager().onLogUpdate(job, log);
+ }
+
+ @Override
+ @GET
+ @Path("/{id}/status")
+ public Response status(@PathParam("id") String id, @QueryParam("taskID") String taskID) {
+ Message message = null;
+ String realId = ZuulEntranceUtils.parseExecID(id)[3];
+ Option job = Option.apply(null);
+ try {
+ job = entranceServer.getJob(realId);
+ } catch (Exception e) {
+ logger.warn("获取任务 {} 状态时出现错误", realId, e);
+ //如果获取错误了,证明在内存中已经没有了,去jobhistory找寻一下taskID代表的任务的状态，然后返回
+ long realTaskID = Long.parseLong(taskID);
+ String status = JobHistoryHelper.getStatusByTaskID(realTaskID);
+ message = Message.ok();
+ message.setMethod("/api/entrance/" + id + "/status");
+ message.data("status", status).data("execID", id);
+ return Message.messageToResponse(message);
+ }
+ if (job.isDefined()) {
+ message = Message.ok();
+ message.setMethod("/api/entrance/" + id + "/status");
+ message.data("status", job.get().getState().toString()).data("execID", id);
+ } else {
+ message = Message.error("ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空，不能获取相应的任务状态)");
+ }
+ return Message.messageToResponse(message);
+ }
+
+
+
+
+ @Override
+ @GET
+ @Path("/{id}/progress")
+ public Response progress(@PathParam("id") String id) {
+ Message message = null;
+ String realId = ZuulEntranceUtils.parseExecID(id)[3];
+ Option job = entranceServer.getJob(realId);
+ if (job.isDefined()) {
+ JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo();
+ if (jobProgressInfos == null) {
+ message = Message.error("Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)");
+ message.setMethod("/api/entrance/" + id + "/progress");
+ } else {
+ List> list = new ArrayList<>();
+ for (JobProgressInfo jobProgressInfo : jobProgressInfos) {
+ if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue())) {
+ Map map = new HashMap<>();
+ map.put("id", jobProgressInfo.id());
+ map.put("succeedTasks", jobProgressInfo.succeedTasks());
+ map.put("failedTasks", jobProgressInfo.failedTasks());
+ map.put("runningTasks", jobProgressInfo.runningTasks());
+ map.put("totalTasks", jobProgressInfo.totalTasks());
+ list.add(map);
+ } else if (jobProgressInfo.failedTasks() > 0 || jobProgressInfo.runningTasks() > 0) {
+ Map map = new HashMap<>();
+ map.put("id", jobProgressInfo.id());
+ map.put("succeedTasks", jobProgressInfo.succeedTasks());
+ map.put("failedTasks", jobProgressInfo.failedTasks());
+ map.put("runningTasks", jobProgressInfo.runningTasks());
+ map.put("totalTasks", jobProgressInfo.totalTasks());
+ list.add(map);
+ }
+ }
+ message = Message.ok();
+ message.setMethod("/api/entrance/" + id + "/progress");
+ message.data("progress", job.get().getProgress()).data("execID", id).data("progressInfo", list);
+ }
+ } else {
+ message = Message.error("The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)");
+ }
+ return Message.messageToResponse(message);
+ }
+
+ @Override
+ @GET
+ @Path("/{id}/log")
+ public Response log(@Context HttpServletRequest req, @PathParam("id") String id) {
+ String realId = ZuulEntranceUtils.parseExecID(id)[3];
+ Option job = Option.apply(null);
+ Message message = null;
+ try {
+ job = entranceServer.getJob(realId);
+ } catch (final Throwable t) {
+ message = Message.error("The job you just executed has ended. This interface no longer provides a query. It is recommended that you download the log file for viewing.(您刚刚执行的job已经结束，本接口不再提供查询，建议您下载日志文件进行查看)");
+ message.setMethod("/api/entrance/" + id + "/log");
+ return Message.messageToResponse(message);
+ }
+ if (job.isDefined()) {
+ logger.debug("开始获取 {} 的日志", job.get().getId());
+ LogReader logReader = entranceServer.getEntranceContext().getOrCreateLogManager().getLogReader(realId);
+ int fromLine = 0;
+ int size = 100;
+ boolean distinctLevel = true;
+ if (req != null) {
+ String fromLineStr = req.getParameter("fromLine");
+ String sizeStr = req.getParameter("size");
+ if (StringUtils.isNotBlank(fromLineStr)) {
+ fromLine = Math.max(Integer.parseInt(fromLineStr), 0);
+ }
+ if (StringUtils.isNotBlank(sizeStr)) {
+ size = Integer.parseInt(sizeStr) >= 0 ? Integer.parseInt(sizeStr) : 10000;
+ }
+ String distinctLevelStr = req.getParameter("distinctLevel");
+ if ("false".equals(distinctLevelStr)) {
+ distinctLevel = false;
+ }
+ }
+
+ Object retLog = null;
+ int retFromLine = 0;
+ try {
+ if (distinctLevel) {
+ String[] logs = new String[4];
+ retFromLine = logReader.readArray(logs, fromLine, size);
+ retLog = new ArrayList(Arrays.asList(logs));
+ } else {
+ StringBuilder sb = new StringBuilder();
+ retFromLine = logReader.read(sb, fromLine, size);
+ retLog = sb.toString();
+ }
+ } catch (IllegalStateException e) {
+ logger.error("为 {} 获取日志失败 原因：{}", job.get().getId(), e.getMessage());
+ message = Message.ok();
+ message.setMethod("/api/entrance/" + id + "/log");
+ message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine);
+ } catch (final IllegalArgumentException e) {
+ logger.error("为 {} 获取日志失败", job.get().getId());
+ message = Message.ok();
+ message.setMethod("/api/entrance/" + id + "/log");
+ message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine);
+ return Message.messageToResponse(message);
+ } catch (final Exception e1) {
+ logger.error("为 {} 获取日志失败", job.get().getId(), e1);
+ message = Message.error("Failed to get log information(获取日志信息失败)");
+ message.setMethod("/api/entrance/" + id + "/log");
+ message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine);
+ return Message.messageToResponse(message);
+ }
+ message = Message.ok();
+ message.setMethod("/api/entrance/" + id + "/log");
+ message.data("log", retLog).data("execID", id).data("fromLine", retFromLine + fromLine);
+ logger.debug("获取 {} 日志成功", job.get().getId());
+ } else {
+ message = Message.error("Can't find execID(不能找到execID): " + id + "Corresponding job, can not get the corresponding log(对应的job，不能获得对应的日志)");
+ message.setMethod("/api/entrance/" + id + "/log");
+ }
+ return Message.messageToResponse(message);
+ }
+
+
+ @Override
+ @GET
+ @Path("/{id}/kill")
+ public Response kill(@PathParam("id") String id, @QueryParam("taskID") long taskID) {
+ String realId = ZuulEntranceUtils.parseExecID(id)[3];
+ //通过jobid获取job,可能会由于job找不到而导致有looparray的报错,一旦报错的话，就可以将该任务直接置为Cancenlled
+ Option job = Option.apply(null);
+ try {
+ job = entranceServer.getJob(realId);
+ } catch (Exception e) {
+ logger.warn("can not find a job in entranceServer, will force to kill it", e);
+ //如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的
+ JobHistoryHelper.forceKill(taskID);
+ Message message = Message.ok("强制杀死任务");
+ message.setMethod("/api/entrance/" + id + "/kill");
+ message.setStatus(0);
+ return Message.messageToResponse(message);
+ }
+ Message message = null;
+ if (job.isEmpty()) {
+ message = Message.error("Can't find execID(不能找到execID): " + id + "Corresponding job, can't kill(对应的job，不能进行kill)");
+ message.setMethod("/api/entrance/" + id + "/kill");
+ message.setStatus(1);
+ } else {
+ try {
+ logger.info("begin to kill job {} ", job.get().getId());
+ job.get().kill();
+ message = Message.ok("Successfully killed the job(成功kill了job)");
+ message.setMethod("/api/entrance/" + id + "/kill");
+ message.setStatus(0);
+ message.data("execID", id);
+ //ensure the job's state is cancelled in database
+ if (job.get() instanceof EntranceJob) {
+ EntranceJob entranceJob = (EntranceJob) job.get();
+ Task task = entranceJob.getTask();
+ ((RequestPersistTask) task).setStatus("Cancelled");
+ this.entranceServer.getEntranceContext().getOrCreatePersistenceManager().createPersistenceEngine().updateIfNeeded(task);
+ }
+ logger.info("end to kill job {} ", job.get().getId());
+ } catch (Throwable t) {
+ logger.error("kill job {} failed ", job.get().getId(), t);
+ message = Message.error("An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)");
+ message.setMethod("/api/entrance/" + id + "/kill");
+ message.setStatus(1);
+ }
+ }
+ return Message.messageToResponse(message);
+ }
+
+ @Override
+ @GET
+ @Path("/{id}/pause")
+ public Response pause(@PathParam("id") String id) {
+ String realId = ZuulEntranceUtils.parseExecID(id)[3];
+ Option job = entranceServer.getJob(realId);
+ Message message = null;
+ if (job.isEmpty()) {
+ message = Message.error("不能找到execID: " + id + "对应的job，不能进行pause");
+ message.setMethod("/api/entrance/" + id + "/pause");
+ message.setStatus(1);
+ } else {
+ try {
+ //todo job pause 接口还未实现和给出
+ //job.pause();
+ logger.info("begin to pause job {} ", job.get().getId());
+ message = Message.ok("成功pause了job");
+ message.setStatus(0);
+ message.data("execID", id);
+ message.setMethod("/api/entrance/" + id + "/pause");
+ logger.info("end to pause job {} ", job.get().getId());
+ } catch (Throwable t) {
+ logger.info("pause job {} failed ", job.get().getId());
+ message = Message.error("Abnormal when pausing job, pause failed(pause job的时候出现了异常，pause失败)");
+ message.setMethod("/api/entrance/" + id + "/pause");
+ message.setStatus(1);
+ }
+ }
+ return Message.messageToResponse(message);
+ }
+
+ @Override
+ @POST
+ @Path("/backgroundservice")
+ public Response backgroundservice(@Context HttpServletRequest req, Map json) {
+ Message message = null;
+ logger.info("Begin to get an execID");
+ String backgroundType = (String) json.get("background");
+ BackGroundService[] bgServices = entranceServer.getEntranceContext().getOrCreateBackGroundService();
+ BackGroundService bgService = null;
+ for (BackGroundService backGroundService : bgServices) {
+ if (backgroundType.equals(backGroundService.serviceType())) {
+ bgService = backGroundService;
+ break;
+ }
+ }
+ Gson gson = new Gson();
+ Map executionCode = (Map) json.get("executionCode");
+ executionCode = gson.fromJson(gson.toJson(executionCode), LinkedTreeMap.class);
+ json.put("executionCode", executionCode);
+ json.put(TaskConstant.UMUSER, SecurityFilter.getLoginUsername(req));
+ ServerEvent serverEvent = new ServerEvent();
+ serverEvent.setData(json);
+ serverEvent.setUser(SecurityFilter.getLoginUsername(req));
+ ServerEvent operation = bgService.operation(serverEvent);
+ String execID = entranceServer.execute(operation.getData());
+ Task task = ((EntranceJob) entranceServer.getJob(execID).get()).getTask();
+ Long taskID = ((RequestPersistTask) task).getTaskID();
+ execID = ZuulEntranceUtils.generateExecID(execID, Sender.getThisServiceInstance().getApplicationName(), new String[]{Sender.getThisInstance()});
+ message = Message.ok();
+ message.setMethod("/api/entrance/backgroundservice");
+ message.data("execID", execID);
+ message.data("taskID", taskID);
+ logger.info("End to get an an execID: {}, taskID: {}", execID, taskID);
+ return Message.messageToResponse(message);
+
+ }
+}
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/server/DefaultEntranceServer.java
similarity index 94%
rename from ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/server/DefaultEntranceServer.java
rename to linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/server/DefaultEntranceServer.java
index 7813eccf89..3bfd3f3f0c 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/server/DefaultEntranceServer.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/server/DefaultEntranceServer.java
@@ -25,10 +25,7 @@
import javax.annotation.PostConstruct;
-/**
- * created by enjoyyin on 2018/10/12
- * Description:
- */
+
@EntranceServerBeanAnnotation
public class DefaultEntranceServer extends EntranceServer {
@@ -46,6 +43,7 @@ public DefaultEntranceServer(EntranceContext entranceContext) {
@PostConstruct
public void init() {
getEntranceWebSocketService();
+ addRunningJobEngineStatusMonitor();
}
@Override
@@ -63,4 +61,7 @@ public LogReader logReader(String execId) {
return getEntranceContext().getOrCreateLogManager().getLogReader(execId);
}
+ private void addRunningJobEngineStatusMonitor() {
+
+ }
}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/resources/application.yml b/linkis-computation-governance/linkis-entrance/src/main/resources/application.yml
new file mode 100644
index 0000000000..62b82b3283
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/resources/application.yml
@@ -0,0 +1,27 @@
+server:
+ port: 9104
+spring:
+ application:
+ name: linkis-cg-entrance
+
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://127.0.0.1:20303/eureka/
+ instance:
+ metadata-map:
+ test:
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
+# register-with-eureka: false
+# fetch-registry: false
+ribbon:
+ ReadTimeout: 10000
+ ConnectTimeout: 10000
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance/src/main/resources/linkis-server.properties b/linkis-computation-governance/linkis-entrance/src/main/resources/linkis-server.properties
new file mode 100644
index 0000000000..c34c581003
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/resources/linkis-server.properties
@@ -0,0 +1,20 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+##restful
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.entrance.restful
+wds.linkis.server.socket.mode=false
+#wds.linkis.entrance.config.log.path=hdfs:///tmp/linkis/
+#wds.linkis.resultSet.store.path=hdfs:///tmp/linkis
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance/src/main/resources/log4j2.xml b/linkis-computation-governance/linkis-entrance/src/main/resources/log4j2.xml
new file mode 100644
index 0000000000..0f19fc3c17
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/resources/log4j2.xml
@@ -0,0 +1,33 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceContext.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceContext.scala
similarity index 97%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceContext.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceContext.scala
index 881f544158..3f216504f3 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceContext.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceContext.scala
@@ -23,9 +23,7 @@ import com.webank.wedatasphere.linkis.entrance.log.LogManager
import com.webank.wedatasphere.linkis.entrance.persistence.PersistenceManager
import com.webank.wedatasphere.linkis.scheduler.Scheduler
-/**
- * Created by enjoyyin on 2018/9/4.
- */
+
abstract class EntranceContext {
def getOrCreateScheduler(): Scheduler
@@ -40,6 +38,7 @@ abstract class EntranceContext {
/**
* Please note: it can be empty(请注意：可以为空)
+ *
* @return
*/
def getOrCreateEventListenerBus: EntranceEventListenerBus[EntranceEventListener, EntranceEvent]
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceParser.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceParser.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceParser.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceParser.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
similarity index 97%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
index da2188f763..d12199fb8b 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,15 +18,13 @@ import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException}
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
import com.webank.wedatasphere.linkis.entrance.log.LogReader
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.scheduler.queue.{Job, SchedulerEventState}
import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.exception.ExceptionUtils
-/**
- * Created by enjoyyin on 2018/9/4.
- */
+
abstract class EntranceServer extends Logging {
private var entranceWebSocketService: Option[EntranceWebSocketService] = None
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
similarity index 92%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
index 89ae6100fd..ec0e0ef792 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
@@ -30,9 +30,9 @@ import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob
import com.webank.wedatasphere.linkis.entrance.log.LogReader
import com.webank.wedatasphere.linkis.entrance.restful.EntranceRestfulApi
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.utils.ZuulEntranceUtils
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.queue.{Job, SchedulerEventState}
@@ -41,9 +41,7 @@ import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import com.webank.wedatasphere.linkis.server.socket.controller.{ServerEvent, ServerEventService, SocketServerEvent}
import org.apache.commons.lang.StringUtils
-/**
- * Created by enjoyyin on 2018/9/14.
- */
+
class EntranceWebSocketService extends ServerEventService with EntranceEventListener {
private val jobIdToEventId = new util.HashMap[String, Integer]
@@ -59,7 +57,6 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList
private val killUrlPattern = (restfulURI + """entrance/(.+)/kill""").r
private val pauseUrlPattern = (restfulURI + """entrance/(.+)/pause""").r
private val backgroundUrlPattern = restfulURI + """entrance/backgroundservice"""
- private val runtimeTuningUrlPattern = (restfulURI + """entrance/(.+)/runtimeTuning""").r
def setEntranceServer(entranceServer: EntranceServer):Unit = this.entranceServer = entranceServer
def setEntranceRestfulApi(entranceRestfulApi: EntranceRestfulApi):Unit = this.entranceRestfulApi = entranceRestfulApi
@@ -140,13 +137,9 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList
LogUtils.generateInfo(s"Your job's execution code is (after variable substitution and code check) "))
entranceServer.getEntranceContext.getOrCreateLogManager().onLogUpdate(job,
"************************************SCRIPT CODE************************************")
-// entranceServer.getEntranceContext.getOrCreateLogManager().onLogUpdate(job,task.asInstanceOf[RequestPersistTask].getCode)
val code = task.asInstanceOf[RequestPersistTask].getCode
val codeLength = code.length()
- val pushLog = if (codeLength > 1000){
- code.substring(0, 1000) + " ... "
- } else code
- entranceServer.getEntranceContext.getOrCreateLogManager().onLogUpdate(job,pushLog)
+ entranceServer.getEntranceContext.getOrCreateLogManager().onLogUpdate(job,code)
entranceServer.getEntranceContext.getOrCreateLogManager().onLogUpdate(job,
"************************************SCRIPT CODE************************************")
entranceServer.getEntranceContext.getOrCreateLogManager().onLogUpdate(job,
@@ -167,11 +160,11 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList
var retMessage:Message = null
val realID = ZuulEntranceUtils.parseExecID(id)(3)
entranceServer.getJob(realID) foreach {
- case entranceExecutionJob:EntranceExecutionJob => {
+ case entranceExecutionJob: EntranceExecutionJob => {
info(s"begin to get job $realID log via websocket")
- val logsArr:Array[String] = new Array[String](4)
+ val logsArr: Array[String] = new Array[String](4)
entranceExecutionJob.getWebSocketLogReader.foreach(logReader => logReader.readArray(logsArr, 0, 100))
- val logList:util.List[String] = new util.ArrayList[String]()
+ val logList: util.List[String] = new util.ArrayList[String]()
logsArr foreach logList.add
retMessage = Message.ok("Successfully obtained log information(成功获取到日志信息)")
retMessage.data("execID", id).data("log", logList).data("websocketTag", websocketTagJobID.get(realID))
@@ -196,11 +189,11 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList
//val realID:String = if (entranceServer.getJob(id).isDefined) id else ZuulEntranceUtils.parseExecID(id)(2)
val realID:String = if (!id.contains(":")) id else ZuulEntranceUtils.parseExecID(id)(3)
entranceServer.getJob(realID) foreach {
- case entranceExecutionJob:EntranceExecutionJob =>
+ case entranceExecutionJob: EntranceExecutionJob =>
val longExecID = ZuulEntranceUtils.generateExecID(realID,
entranceExecutionJob.getTask.asInstanceOf[RequestPersistTask].getExecuteApplicationName, Sender.getThisInstance,
entranceExecutionJob.getTask.asInstanceOf[RequestPersistTask].getRequestApplicationName)
- if(!jobIdToEventId.containsKey(realID) && event != null) jobIdToEventId synchronized jobIdToEventId.put(realID, event.getId)
+ if (!jobIdToEventId.containsKey(realID) && event != null) jobIdToEventId synchronized jobIdToEventId.put(realID, event.getId)
val status = entranceExecutionJob.getState
retMessage = Message.ok("Get the status of the task successfully(获取任务状态成功)")
@@ -226,7 +219,7 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList
var retMessage:Message = null
val realID = ZuulEntranceUtils.parseExecID(id)(3)
entranceServer.getJob(realID) foreach {
- case entranceExecutionJob:EntranceExecutionJob => {
+ case entranceExecutionJob: EntranceExecutionJob => {
val progress = entranceExecutionJob.getProgress
retMessage = Message.ok("Get the task progress successfully(获取任务进度成功)")
val taskID = entranceExecutionJob.getTask.asInstanceOf[RequestPersistTask].getTaskID
@@ -254,16 +247,16 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList
var retMessage:Message = null
val realID = ZuulEntranceUtils.parseExecID(id)(3)
entranceServer.getJob(realID) foreach {
- case entranceExecutionJob:EntranceExecutionJob =>
- try{
+ case entranceExecutionJob: EntranceExecutionJob =>
+ try {
entranceExecutionJob.kill()
retMessage = Message.ok("Kill task succeeded(kill任务成功)")
retMessage.setMethod(restfulURI + "entrance/" + id + "/kill")
retMessage.setStatus(0)
return retMessage
- }catch{
- case e:Exception => retMessage = Message.error("Kill task failed(kill任务失败)", e)
- case t:Throwable => retMessage = Message.error("Kill task failed(kill任务失败)", t)
+ } catch {
+ case e: Exception => retMessage = Message.error("Kill task failed(kill任务失败)", e)
+ case t: Throwable => retMessage = Message.error("Kill task failed(kill任务失败)", t)
}
case _ =>
}
@@ -380,27 +373,33 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList
}
}
})
- val logList:util.List[String] = new util.ArrayList[String]()
+ if (StringUtils.isBlank(info.toString()) &&
+ StringUtils.isBlank(warn.toString()) &&
+ StringUtils.isBlank(error.toString()) &&
+ StringUtils.isBlank(all.toString())) {
+ return
+ }
+ val logList: util.List[String] = new util.ArrayList[String]()
logList.add(error.toString())
logList.add(warn.toString())
logList.add(info.toString())
logList.add(all.toString())
message = Message.ok("Return log information(返回日志信息)")
val executeApplicationName = job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getExecuteApplicationName
- val creator:String = job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getRequestApplicationName
- val execID:String = ZuulEntranceUtils.generateExecID(job.getId, executeApplicationName, Sender.getThisInstance, creator)
+ val creator: String = job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getRequestApplicationName
+ val execID: String = ZuulEntranceUtils.generateExecID(job.getId, executeApplicationName, Sender.getThisInstance, creator)
message.setMethod(restfulURI + "entrance/" + execID + "/log")
val taskID = job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getTaskID
- message.data("execID",execID).data("log", logList).data("websocketTag", websocketTagJobID.get(job.getId)).data("taskID", taskID)
+ message.data("execID", execID).data("log", logList).data("websocketTag", websocketTagJobID.get(job.getId)).data("taskID", taskID)
sendMsg(job, message)
}
def pushProgressToFrontend(job: Job, progress: Float, progressInfo: Array[JobProgressInfo]): Unit = {
val progressInfoMap = progressInfo.map(info => toJavaMap(Map("id" -> info.id, "succeedTasks" -> info.succeedTasks,
- "failedTasks" -> info.failedTasks , "runningTasks" -> info.runningTasks, "totalTasks" -> info.totalTasks)))
+ "failedTasks" -> info.failedTasks, "runningTasks" -> info.runningTasks, "totalTasks" -> info.totalTasks)))
val executeApplicationName = job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getExecuteApplicationName
val creator = job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getRequestApplicationName
job.asInstanceOf[EntranceJob].setProgressInfo(progressInfo)
- val execID:String = ZuulEntranceUtils.generateExecID(job.getId, executeApplicationName, Sender.getThisInstance, creator)
+ val execID: String = ZuulEntranceUtils.generateExecID(job.getId, executeApplicationName, Sender.getThisInstance, creator)
val message = Message.ok("返回进度信息!")
message.setMethod(restfulURI + "entrance/" + execID + "/progress")
val taskID = job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getTaskID
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/AbstractBackGroundService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/AbstractBackGroundService.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/AbstractBackGroundService.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/AbstractBackGroundService.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/BackGroundService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/BackGroundService.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/BackGroundService.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/background/BackGroundService.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala
similarity index 85%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala
index f2258e1563..f5532539d1 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala
@@ -15,18 +15,17 @@
*/
package com.webank.wedatasphere.linkis.entrance.cache
+
import java.util
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.protocol.conf.{RequestQueryGlobalConfig, ResponseQueryConfig}
import com.webank.wedatasphere.linkis.protocol.CacheableProtocol
-import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryGlobalConfig, ResponseQueryConfig}
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.rpc.RPCMapCache
-/**
- * Created by enjoyyin on 2018/11/4.
- */
+
object GlobalConfigurationKeyValueCache extends
RPCMapCache[RequestPersistTask, String, String](EntranceConfiguration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue) {
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala
new file mode 100644
index 0000000000..ff2ff61dd5
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.conf
+
+import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType}
+
+
+object EntranceConfiguration {
+
+ val ENTRANCE_SCHEDULER_MAX_PARALLELISM_USERS = CommonVars("wds.linkis.entrance.scheduler.maxParallelismUsers", new Integer(1000))
+ val ENTRANCE_LISTENER_BUS_EVENT_QUEUE_CAPACITY = CommonVars("wds.linkis.entrance.listenerBus.queue.capacity", new Integer(5000))
+
+ val JOB_MAX_PERSIST_WAIT_TIME = CommonVars("wds.linkis.entrance.job.persist.wait.max", new TimeType("5m"))
+
+ val MULTI_ENTRANCE_CONDITION = CommonVars("wds.linkis.entrance.multi.entrance.flag", true)
+
+ val RESULT_SET_STORE_PATH = CommonVars("wds.linkis.resultSet.store.path", CommonVars[String]("wds.linkis.filesystem.hdfs.root.path").getValue)
+
+ /**
+ * QUERY_PERSISTENCE_SPRING_APPLICATION_NAME is the name of the application that represents the query module in springcloud
+ * QUERY_PERSISTENCE_SPRING_APPLICATION_NAME 是表示query模块在springcloud中的应用名称
+ */
+ val QUERY_PERSISTENCE_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.query.application.name", "linkis-ps-publicservice")
+ /**
+ * DEFAULT_LOGPATH_PREFIX is the prefix that represents the default log storage path
+ * DEFAULT_LOGPATH_PREFIX 是表示默认的日志存储路径的前缀
+ */
+ val DEFAULT_LOGPATH_PREFIX = CommonVars[String]("wds.linkis.entrance.config.log.path", CommonVars[String]("wds.linkis.filesystem.hdfs.root.path").getValue)
+ /**
+ * Default_Cache_Max is used to specify the size of the LoopArray of the CacheLogWriter
+ * Default_Cache_Max 是用来指定CacheLogWriter的LoopArray的大小
+ */
+ val DEFAULT_CACHE_MAX = CommonVars("wds.linkis.entrance.log.cacheMax", 500)
+ /**
+ * Default_Log_CharSet is used to specify the encoding mode of the log storage.
+ * Default_Log_CharSet 是用来指定日志存储的编码方式
+ */
+ val DEFAULT_LOG_CHARSET = CommonVars("wds.linkis.entrance.log.defaultCharSet", "utf-8")
+ /**
+ * The application name of the console module in spring-cloud
+ * console 模块在spring-cloud中的应用名称
+ */
+ val CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.console.configuration.application.name", "linkis-ps-publicservice")
+ val CLOUD_CONSOLE_VARIABLE_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.console.variable.application.name", "linkis-ps-publicservice")
+ /**
+ * The logPath in the console module returns the key in the map.
+ * console 模块中logPath在返回map中的key
+ */
+ val CLOUD_CONSOLE_LOGPATH_KEY = CommonVars("wds.linkis.console.config.logPath", "wds.linkis.config.logPath")
+ /**
+ * requestApplicationName(Creator) The default service name, the default is IDE
+ * requestApplicationName(Creator) 默认的服务名，默认为IDE
+ */
+ val DEFAULT_REQUEST_APPLICATION_NAME = CommonVars("wds.linkis.default.requestApplication.name", "IDE")
+ /**
+ * runType
+ */
+ val DEFAULT_RUN_TYPE = CommonVars("wds.linkis.default.runType", "sql")
+
+ val DEFAULT_CREATE_SERVICE = CommonVars("wds.linkis.default.create.service", "dss")
+
+ val LOG_WARN_EXCLUDE = CommonVars("wds.linkis.warn.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,com.webank.wedatasphere")
+
+ val CLEAR_LOG = CommonVars("wds.linkis.log.clear", false)
+
+ /**
+ * LOG_EXCLUDE is used to remove the log of the framework log, such as hive spark spring, so that it is not pushed to the front end through websocket.
+ * LOG_EXCLUDE 是用来进行把框架日志，比如hive spark spring等日志进行剔除，不让其通过websocket进行推送到前端
+ */
+ val LOG_EXCLUDE = CommonVars("wds.linkis.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,com.webank.wedatasphere,com.webank")
+
+ /**
+ * wds.linkis.dwc.instance is a parameter used to control the number of engines each user starts.
+ *wds.linkis.instance 是用来进行控制每个用户启动engine数量的参数
+ */
+ val WDS_LINKIS_INSTANCE = CommonVars("wds.linkis.instance", 3)
+
+ val LOG_EXCLUDE_ALL = CommonVars("wds.linkis.log.exclude.all", "com.netflix")
+
+ val MAX_ASK_EXECUTOR_TIME = CommonVars("wds.linkis.max.ask.executor.time", new TimeType("5m"))
+
+ val ERROR_CODE_FILE_DIR = CommonVars("wds.linkis.errorcode.file.dir", "")
+
+ val ENTRANCE_USER = CommonVars("wds.linkis.entrance.user", "")
+
+ val ERROR_CODE_FILE = CommonVars("wds.linkis.errorcode.file", "")
+
+ //todo enjoyyin Simple processing first, this log is not filtered, and subsequent optimization is performed.(先简单处理 不过滤这个日志，后续进行优化)
+ val HIVE_SPECIAL_LOG_INCLUDE = CommonVars("wds.linkis.hive.special.log.include", "org.apache.hadoop.hive.ql.exec.Task")
+
+ val SHARE_FILE_PRE = CommonVars("wds.linkis.share.file.prefix", "")
+
+ val HIVE_THREAD_NAME = CommonVars("wds.linkis.hive.thread.name", "[Thread")
+
+ val HIVE_STAGE_NAME = CommonVars("wds.linkis.hive.stage.name", "Stage-")
+
+ val SPARK_SPECIAL_LOG_INCLUDE = CommonVars("wds.linkis.spark.special.log.include", "com.webank.wedatasphere.linkis.engine.spark.utils.JobProgressUtil")
+
+
+ val SPARK_PROGRESS_NAME = CommonVars("wds.linkis.spark.progress.name", "com.webank.wedatasphere.linkis.engine.spark.utils.JobProgressUtil$")
+
+ val END_FLAG = CommonVars("bdp.dataworkcloud.entrance.end.flag", "info -")
+
+ val HIVE_CREATE_TABLE_LOG = CommonVars("wds.linkis.hive.create.table.log", "numFiles")
+
+ val HIVE_PRINT_INFO_LOG = CommonVars("wds.linkis.hive.printinfo.log", "printInfo -")
+
+ val DATAMAP_APPID = CommonVars("wds.linkis.entrance.datawmap.appid", "75116e6f690e6d222fb55f2dd6f9cb5a")
+
+ val DATAMAP_APPTOKEN = CommonVars("wds.linkis.entrance.datamap.apptoken",
+ "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1aWQiOiJjb29wZXJ5YW5nIiwiYXBwa2V5IjoiNzUxMTZlNmY2OTBlNmQyMjJmYjU1ZjJkZDZmOWNiNWEiLCJpYXQiOjE1NzE4MTQxOTYzNDZ9.JfjvsiNMWSc1RnQh1k7vsNjwp4YRquT_XXbYmlEgBgc")
+
+
+ val DATA_MAP_IP = CommonVars("wds.linkis.entrance.datamap.ip", "")
+
+ //val DATA_MAP_PORT = CommonVars("wds.linkis.entrance.datamap.port", "8122")
+ val DATA_MAP_PORT = CommonVars("wds.linkis.entrance.datamap.port", "9001")
+
+ val IS_BDP_ENV = CommonVars("wds.linkis.entrance.bdp.env", "true")
+
+
+ val SHELL_DANGER_CHECK_SWITCH = CommonVars("wds.linkis.entrance.shell.danger.check.enabled", false)
+ val SHELL_DANGER_USAGE = CommonVars("wds.linkis.shell.danger.usage", "rm,sh,find,kill,python,for,source,hdfs,hadoop,spark-sql,spark-submit,pyspark,spark-shell,hive,yarn")
+ val SHELL_WHITE_USAGE = CommonVars("wds.linkis.shell.white.usage", "cd,ls")
+
+ val FLOW_EXECUTION_CREATOR = CommonVars("wds.linkis.entrance.flow.creator", "nodeexecution")
+
+ val SCHEDULER_CREATOR = CommonVars("wds.linkis.entrance.scheduler.creator", "scheduler")
+
+
+ val IS_QML = CommonVars("wds.linkis.entrance.is.qml", false)
+
+ val PROGRESS_PUSH = CommonVars[String]("wds.linkis.entrance.push.progress", "false")
+
+ val CONCURRENT_FACTORY_MAX_CAPACITY = CommonVars("wds.linkis.concurrent.group.factory.capacity", 1000)
+
+ val CONCURRENT_MAX_RUNNING_JOBS = CommonVars("wds.linkis.concurrent.group.factory.running.jobs", 30)
+
+ val CONCURRENT_EXECUTOR_TIME = CommonVars("wds.linkis.concurrent.group.factory.executor.time", 5 * 60 * 1000)
+
+ val ENGINE_MANAGER_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.enginemanager.application.name", "linkis-cg-engineconnmanager")
+
+ val ENTRANCE_ENGINE_LASTUPDATE_TIMEOUT = CommonVars("wds.linkis.entrance.engine.lastupdate.timeout", new TimeType("5s"))
+ val ENTRANCE_ENGINE_ACTIVITY_TIMEOUT = CommonVars("wds.linkis.entrance.engine.timeout", new TimeType("10s"))
+ val ENTRANCE_ENGINE_ACTIVITY_MONITOR_INTERVAL = CommonVars("wds.linkis.entrance.engine.activity_monitor.interval", new TimeType("3s"))
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConstant.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConstant.scala
new file mode 100644
index 0000000000..06f49e5655
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConstant.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.conf
+
+import com.google.gson.Gson
+
+
+object EntranceConstant {
+
+ val GSON = new Gson()
+
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala
similarity index 95%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala
index 1caa1325a4..44139ff5ea 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala
@@ -25,8 +25,8 @@ import com.webank.wedatasphere.linkis.cs.common.entity.source.{CommonContextKey,
import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
import com.webank.wedatasphere.linkis.scheduler.queue.Job
import org.apache.commons.lang.StringUtils
@@ -34,10 +34,7 @@ import org.apache.commons.lang.StringUtils
import scala.collection.JavaConversions._
import scala.collection.mutable
-/**
- * @author peacewong
- * @date 2020/3/5 15:33
- */
+
object CSEntranceHelper extends Logging {
@@ -150,7 +147,7 @@ object CSEntranceHelper extends Logging {
* @return
*/
def addCSVariable(requestPersistTask: RequestPersistTask): Unit = {
- val variableMap = new mutable.HashMap[String, String]()
+ val variableMap = new util.HashMap[String, Any]()
val (contextIDValueStr, nodeNameStr) = getContextInfo(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]])
if (StringUtils.isNotBlank(contextIDValueStr)) {
@@ -161,7 +158,9 @@ object CSEntranceHelper extends Logging {
variableMap.put(linkisVariable.getKey, linkisVariable.getValue)
}
}
- TaskUtils.addVariableMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]], variableMap)
+ if(variableMap.nonEmpty){
+ TaskUtils.addVariableMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]], variableMap)
+ }
info(s"parse variable end nodeName:$nodeNameStr")
}
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEvent.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEvent.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEvent.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEvent.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListener.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListener.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListener.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListener.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListenerBus.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListenerBus.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListenerBus.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/event/EntranceEventListenerBus.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceCastFailException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceCastFailException.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceCastFailException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceCastFailException.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorException.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/EntranceErrorException.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala
new file mode 100644
index 0000000000..aae992b4ba
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.exception
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+
+
+case class JobHistoryFailedException(errorMsg:String) extends ErrorException(50081, errorMsg)
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/LogReadFailedException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/LogReadFailedException.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/LogReadFailedException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/LogReadFailedException.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/SensitiveTablesCheckException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/SensitiveTablesCheckException.scala
new file mode 100644
index 0000000000..7bd885c11e
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/SensitiveTablesCheckException.scala
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.exception
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+
+
+case class SensitiveTablesCheckException(errorMsg:String) extends ErrorException(50079, errorMsg)
+
+case class DangerousGramsCheckException(errorMsg:String) extends ErrorException(50081, errorMsg)
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/SubmitFailedException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/SubmitFailedException.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/SubmitFailedException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/SubmitFailedException.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/DefaultEntranceExecutor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/DefaultEntranceExecutor.scala
new file mode 100644
index 0000000000..ad72245900
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/DefaultEntranceExecutor.scala
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.execute
+
+import com.webank.wedatasphere.linkis.common.utils.Utils
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.{RequestTask, ResponseTaskStatus}
+import com.webank.wedatasphere.linkis.orchestrator.ecm.entity.Mark
+import com.webank.wedatasphere.linkis.scheduler.executer.{ConcurrentTaskOperateSupport, ErrorExecuteResponse, ExecutorState, SubmitResponse}
+
+
+
+class DefaultEntranceExecutor(id: Long, mark: Mark) extends EntranceExecutor(id, mark) with ConcurrentTaskOperateSupport {
+
+
+ /* private def doMethod[T](exec: String => T): T = if (engineReturns.isEmpty)
+ throw new EntranceErrorException(20001, s"Engine${id} could not find a job in RUNNING state(Engine${id}找不到处于RUNNING状态的Job)")
+ else exec(engineReturns(0).execId)*/
+
+
+ override protected def callExecute(request: RequestTask): EngineExecuteAsynReturn = {
+ val response = getEngineConnExecutor().execute(request)
+ response match {
+ case SubmitResponse(execId) => new EngineExecuteAsynReturn(request, getEngineConnExecutor().getServiceInstance.getInstance, execId, engineReturn => {
+ info("remove execId-" + execId + " with instance " + getEngineConnExecutor().getServiceInstance)
+ engineReturns -= engineReturn
+ })
+ case ErrorExecuteResponse(message, t) =>
+ info(s"failed to submit task to engineConn,reason: $message")
+ throw t
+ }
+ }
+
+ override def close(): Unit = {
+ if (engineReturns.nonEmpty) engineReturns.foreach { e =>
+ e.notifyError(s"$toString has already been completed with state $state.")
+ e.notifyStatus(ResponseTaskStatus(e.execId, ExecutionNodeStatus.Failed))
+ }
+ }
+
+ override def kill(jobId: String): Boolean = {
+ info(s"start to kill job $jobId")
+ engineReturns.find(_.getJobId.contains(jobId)).exists(e => killExecId(e.execId))
+ }
+
+ override def killAll(): Boolean = {
+ engineReturns.foreach(f => Utils.tryQuietly(killExecId(f.execId)))
+ true
+ }
+
+ override def pause(jobId: String): Boolean = {
+ //TODO
+ true
+ }
+
+
+ override def pauseAll(): Boolean = {
+ //TODO
+ true
+ }
+
+ override def resume(jobId: String): Boolean = {
+ //TODO
+ true
+ }
+
+ override def resumeAll(): Boolean = {
+ //TODO
+ true
+ }
+
+
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EngineLockListener.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EngineLockListener.scala
similarity index 80%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EngineLockListener.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EngineLockListener.scala
index 65a1df54dc..9f9a390705 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EngineLockListener.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EngineLockListener.scala
@@ -16,10 +16,9 @@
package com.webank.wedatasphere.linkis.entrance.execute
-/**
- * Created by enjoyyin on 2018/9/15.
- */
+
trait EngineLockListener {
- def onEngineLocked(engine: EntranceEngine, lock: String): Unit
- def onEngineLockUsed(engine: EntranceEngine): Unit
+ def onEngineLocked(engine: EntranceExecutor, lock: String): Unit
+
+ def onEngineLockUsed(engine: EntranceExecutor): Unit
}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutionService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutionService.scala
new file mode 100644
index 0000000000..994b8d1a4a
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutionService.scala
@@ -0,0 +1,258 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.execute
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.EntranceContext
+import com.webank.wedatasphere.linkis.entrance.annotation.EntranceContextBeanAnnotation
+import com.webank.wedatasphere.linkis.entrance.conf.{EntranceConfiguration, EntranceConstant}
+import com.webank.wedatasphere.linkis.entrance.event.EntranceProgressEvent
+import com.webank.wedatasphere.linkis.governance.common.conf.GovernanceCommonConf
+import com.webank.wedatasphere.linkis.governance.common.entity.{ExecutionNodeStatus, NodeExistStatus}
+import com.webank.wedatasphere.linkis.governance.common.protocol.engineconn.{RequestEngineStatusBatch, ResponseEngineStatusBatch}
+import com.webank.wedatasphere.linkis.governance.common.protocol.task._
+import com.webank.wedatasphere.linkis.governance.common.utils.GovernanceConstant
+import com.webank.wedatasphere.linkis.manager.common.entity.enumeration.NodeStatus
+import com.webank.wedatasphere.linkis.manager.common.protocol.node.{RequestNodeStatus, ResponseNodeStatus}
+import com.webank.wedatasphere.linkis.message.annotation.Receiver
+import com.webank.wedatasphere.linkis.message.builder.ServiceMethodContext
+import com.webank.wedatasphere.linkis.orchestrator.ecm.service.{EngineConnExecutor, TaskExecutionReceiver}
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.rpc.exception.DWCRPCRetryException
+import com.webank.wedatasphere.linkis.rpc.utils.RPCUtils
+import com.webank.wedatasphere.linkis.scheduler.executer.AliasOutputExecuteResponse
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+import org.springframework.stereotype.Service
+
+import javax.annotation.PostConstruct
+import scala.collection.JavaConverters._
+import java.util
+import java.util.concurrent.TimeUnit
+
+
+@Service
+class EntranceExecutionService extends TaskExecutionReceiver with Logging {
+
+ @EntranceContextBeanAnnotation.EntranceContextAutowiredAnnotation
+ private var entranceContext: EntranceContext = _
+
+ @PostConstruct
+ private def init(): Unit = {
+ addEngineExecutorStatusMonitor()
+ }
+
+ private[execute] def getEntranceExecutorManager = entranceContext.getOrCreateScheduler().getSchedulerContext
+ .getOrCreateExecutorManager.asInstanceOf[EntranceExecutorManager]
+
+ private def findEngineExecuteAsyncReturn(execId: String, sender: Sender, message: String): Option[EngineExecuteAsynReturn] = {
+ val serviceInstance = RPCUtils.getServiceInstanceFromSender(sender)
+ if (serviceInstance == null || serviceInstance.getInstance == null) {
+ warn(s"because of not support sender $sender, cannot update the message $message for execId $execId.")
+ return None
+ }
+ val engine = getEntranceExecutorManager.getEntranceExecutorByInstance(serviceInstance.getInstance)
+ if (engine.isEmpty) {
+ warn(s"${getEntranceExecutorManager.getClass.getSimpleName} cannot find a entranceEngine to update the message $message for execId $execId from $sender.")
+ None
+ } else {
+ val jobReturn = engine.flatMap(_.getEngineReturns.find(_.execId == execId))
+ if (jobReturn.isEmpty) warn(s"${engine.get} cannot find a job to update the message $message for execId $execId from $sender, EngineReturns List is ${engine.map(_.getEngineReturns.map(_.execId).toList).get}.")
+ jobReturn
+ }
+ }
+
+ def onOperate(execId: String, sender: Sender, op: Job => Unit, message: String): Unit = findEngineExecuteAsyncReturn(execId, sender, message).foreach { er =>
+ er.notifyHeartbeat()
+ er.getJobId.flatMap(entranceContext.getOrCreateScheduler().get).foreach {
+ case job: Job => op(job)
+ case _ =>
+ }
+ }
+
+ private def askRetryWhenExecuteAsyncReturnIsEmpty(execId: String, sender: Sender, message: String) =
+ findEngineExecuteAsyncReturn(execId, sender, message).getOrElse {
+ Utils.tryQuietly(Thread.sleep(50))
+ findEngineExecuteAsyncReturn(execId, sender, toString)
+ .getOrElse(throw new DWCRPCRetryException(s"please retry, since $execId cannot be consumed by entrance!"))
+ }
+
+ @Receiver
+ override def taskLogReceiver(taskLog: ResponseTaskLog, smc: ServiceMethodContext): Unit = {
+ val sender = smc.getSender
+ onOperate(taskLog.execId, sender, entranceContext.getOrCreateLogManager().onLogUpdate(_, taskLog.log), "ResponseTaskLog")
+ updateExecutorActivityTime(RPCUtils.getServiceInstanceFromSender(sender))
+ }
+
+ private def updateExecutorActivityTime(serviceInstance: ServiceInstance) = {
+ if (null != serviceInstance) {
+ val engineConnExecutor = getEntranceExecutorManager.engineConnManager.getEngineConnExecutorCache().getOrDefault(serviceInstance, null)
+ if (null != engineConnExecutor) synchronized {
+ engineConnExecutor.updateLastUpdateTime()
+ } else {
+ warn(s"EngineConnExecutor ${serviceInstance.toString} cannot be found in engineConnExecutorCache")
+ }
+ }
+ }
+
+ @Receiver
+ override def taskProgressReceiver(taskProgress: ResponseTaskProgress, smc: ServiceMethodContext): Unit = {
+ val sender = smc.getSender
+ Utils.tryAndWarn {
+ onOperate(taskProgress.execId, sender, job => {
+ entranceContext.getOrCreateEventListenerBus.post(EntranceProgressEvent(job, taskProgress.progress, taskProgress.progressInfo))
+ entranceContext.getOrCreatePersistenceManager().onProgressUpdate(job, taskProgress.progress, taskProgress.progressInfo)
+ job.asInstanceOf[EntranceJob].setProgressInfo(taskProgress.progressInfo)
+ }, "ResponseTaskProgress")
+ }
+ updateExecutorActivityTime(RPCUtils.getServiceInstanceFromSender(sender))
+ }
+
+ @Receiver
+ override def taskStatusReceiver(taskStatus: ResponseTaskStatus, smc: ServiceMethodContext): Unit = {
+ val sender = smc.getSender
+ if (ExecutionNodeStatus.isCompleted(taskStatus.status))
+ info(s"The execId ${taskStatus.execId} from engine $sender is completed with state ${taskStatus.status}.")
+ askRetryWhenExecuteAsyncReturnIsEmpty(taskStatus.execId, sender, s"ResponseTaskStatus(${taskStatus.execId}, ${taskStatus.status})")
+ .notifyStatus(taskStatus)
+ updateExecutorActivityTime(RPCUtils.getServiceInstanceFromSender(sender))
+ }
+
+ @Receiver
+ override def taskResultSizeReceiver(taskResultSize: ResponseTaskResultSize, smc: ServiceMethodContext): Unit = {
+ val sender = smc.getSender
+ askRetryWhenExecuteAsyncReturnIsEmpty(taskResultSize.execId, sender, "ResponseTaskResultSize")
+ .getJobId.flatMap(entranceContext.getOrCreateScheduler().get).foreach {
+ case j: Job => entranceContext.getOrCreatePersistenceManager().onResultSizeCreated(j, taskResultSize.resultSize)
+ case _ =>
+ }
+ }
+
+ @Receiver
+ override def taskResultSetReceiver(taskResultSet: ResponseTaskResultSet, smc: ServiceMethodContext): Unit = {
+ val sender = smc.getSender
+ onOperate(taskResultSet.execId, sender
+ , entranceContext.getOrCreatePersistenceManager()
+ .onResultSetCreated(_, AliasOutputExecuteResponse(taskResultSet.alias, taskResultSet.output))
+ , "ResponseTaskResultSet")
+ }
+
+ @Receiver
+ override def taskErrorReceiver(taskTaskError: ResponseTaskError, smc: ServiceMethodContext): Unit = {
+ val sender = smc.getSender
+// askRetryWhenExecuteAsyncReturnIsEmpty(taskTaskError.execId, sender, s"ResponseTaskError(${taskTaskError.execId}, ${taskTaskError.errorMsg})")
+// .notifyError(taskTaskError.errorMsg)
+ onOperate(taskTaskError.execId, sender, entranceContext.getOrCreateLogManager().onLogUpdate(_, taskTaskError.errorMsg), "ResponseTaskErrorLog")
+ }
+
+ private def addEngineExecutorStatusMonitor(): Unit = {
+ val task = new Runnable {
+ override def run(): Unit = {
+ val startTime = System.currentTimeMillis()
+ val engineExecutorCache = getEntranceExecutorManager.engineConnManager.getEngineConnExecutorCache()
+ if (engineExecutorCache.size() > 0) {
+ info(s"Entrance Executor cache num : ${engineExecutorCache.size()}")
+ }
+ val unActivityEngines = engineExecutorCache.asScala.filter(startTime - _._2.getLastUpdateTime() > EntranceConfiguration.ENTRANCE_ENGINE_LASTUPDATE_TIMEOUT.getValue.toLong).keySet
+ if (null != unActivityEngines && !unActivityEngines.isEmpty) {
+ info(s"There are ${unActivityEngines.size} unActivity engines.")
+ val engineList = new util.ArrayList[ServiceInstance]()
+ unActivityEngines.foreach(engine => {
+ engineList.add(engine)
+ if (engineList.size() >= GovernanceConstant.REQUEST_ENGINE_STATUS_BATCH_LIMIT) {
+ queryEngineStatusAndHandle(engineList, engineExecutorCache)
+ engineList.clear()
+ }
+ })
+ if (!engineList.isEmpty) {
+ queryEngineStatusAndHandle(engineList, engineExecutorCache)
+ engineList.clear()
+ }
+ }
+ val endTime = System.currentTimeMillis()
+ if (endTime - startTime >= EntranceConfiguration.ENTRANCE_ENGINE_ACTIVITY_MONITOR_INTERVAL.getValue.toLong) {
+ error("Query engines status costs longer time than query task interval, you should increase interval.")
+ }
+ }
+ }
+ Utils.defaultScheduler.scheduleWithFixedDelay(task, 10000, EntranceConfiguration.ENTRANCE_ENGINE_ACTIVITY_MONITOR_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS)
+ info("Entrance engineStatusMonitor inited.")
+ }
+
+ private def queryEngineStatusAndHandle(engineList: util.List[ServiceInstance], engineExecutorCache: util.Map[ServiceInstance, EngineConnExecutor]): Unit = {
+ val requestEngineStatus = RequestEngineStatusBatch(engineList)
+ Utils.tryAndError {
+ Sender.getSender(GovernanceCommonConf.MANAGER_SPRING_NAME.getValue).ask(requestEngineStatus) match {
+ case response: ResponseEngineStatusBatch =>
+ if (null != response.msg) {
+ info(s"ResponseEngineStatusBatch msg : ${response.msg}")
+ }
+ if (response.engineStatus.size() != requestEngineStatus.engineList.size()) {
+ error("ResponseEngineStatusBatch engines size is not euqal requet.")
+ }
+ response.engineStatus.asScala.foreach(status => {
+ status._2 match {
+ case NodeExistStatus.UnExist =>
+ warn(s"Engine ${status._1.toString} is Failed, now go to clear its task.")
+ endJobByEngineInstance(status._1, NodeStatus.Failed)
+ case NodeExistStatus.Exist | NodeExistStatus.Unknown =>
+ val engineConnExecutor = engineExecutorCache.getOrDefault(status._1, null)
+ if (null != engineConnExecutor) {
+ Utils.tryCatch {
+ // todo check - only for engine with accessible executor
+ val requestNodeStatus = new RequestNodeStatus
+ Sender.getSender(status._1).ask(requestNodeStatus) match {
+ case rs: ResponseNodeStatus =>
+ if (NodeStatus.isCompleted(rs.getNodeStatus)) {
+ endJobByEngineInstance(status._1, rs.getNodeStatus)
+ } else {
+ warn("Will update engineConnExecutor lastupdated time")
+ updateExecutorActivityTime(status._1)
+ }
+ case o: Any =>
+ Utils.tryAndWarn(warn(s"Unknown response : ${EntranceConstant.GSON.toJson(o)} for request : ${EntranceConstant.GSON.toJson(requestNodeStatus)}"))
+ }
+ } {
+ case t: Throwable =>
+ error(s"Failed to get status of engineConn : ${status._1.toString}, now end the job. ", t)
+ endJobByEngineInstance(status._1, NodeStatus.Failed)
+ }
+ }
+ case o: Any =>
+ error(s"Status of engine ${status._1.toString} is ${status._2}")
+ }
+ })
+ case _ =>
+ error(s"Invalid response. request : ${EntranceConstant.GSON.toJson(requestEngineStatus)}")
+ }
+ }
+ }
+
+ private def endJobByEngineInstance(engineServiceInstance: ServiceInstance, engineNodeStatus: NodeStatus) = {
+ Utils.tryQuietly {
+ val entranceExecutor = getEntranceExecutorManager.getEntranceExecutorByInstance(engineServiceInstance.getInstance).getOrElse(null)
+ if (null != entranceExecutor) {
+ entranceExecutor.getEngineReturns.foreach(er => {
+ er.notifyError(s"Engine has exited unexpectedly with status : ${engineNodeStatus}")
+ er.notifyStatus(ResponseTaskStatus(er.execId, ExecutionNodeStatus.Failed))
+ })
+ } else {
+ warn(s"Cannot find entranceExecutor for ${engineServiceInstance.toString}")
+ }
+ }
+ }
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutor.scala
new file mode 100644
index 0000000000..7c5193537d
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutor.scala
@@ -0,0 +1,189 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.execute
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.governance.common.entity.ExecutionNodeStatus._
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.{RequestTask, ResponseTaskStatus}
+import com.webank.wedatasphere.linkis.orchestrator.ecm.entity.Mark
+import com.webank.wedatasphere.linkis.orchestrator.ecm.service.EngineConnExecutor
+import com.webank.wedatasphere.linkis.protocol.UserWithCreator
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState.ExecutorState
+import com.webank.wedatasphere.linkis.scheduler.executer._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+abstract class EntranceExecutor(val id: Long, val mark: Mark) extends Executor with Logging {
+
+
+ private implicit var userWithCreator: UserWithCreator = _
+
+ private var engineConnExecutor: EngineConnExecutor = _
+
+ protected val engineReturns = ArrayBuffer[EngineExecuteAsynReturn]()
+
+ protected var interceptors: Array[ExecuteRequestInterceptor] = Array(LabelExecuteRequestInterceptor, JobExecuteRequestInterceptor)
+
+
+ def setInterceptors(interceptors: Array[ExecuteRequestInterceptor]) = if (interceptors != null && interceptors.nonEmpty) {
+ this.interceptors = interceptors
+ }
+
+
+ def setUser(user: String): Unit = userWithCreator = if (userWithCreator != null) UserWithCreator(user, userWithCreator.creator)
+ else UserWithCreator(user, null)
+
+ def getUser = if (userWithCreator != null) userWithCreator.user else null
+
+ def setCreator(creator: String): Unit = userWithCreator = if (userWithCreator != null) UserWithCreator(userWithCreator.user, creator)
+ else UserWithCreator(null, creator)
+
+ def getCreator = if (userWithCreator != null) userWithCreator.creator else null
+
+
+ def getEngineConnExecutor(): EngineConnExecutor = this.engineConnExecutor
+
+ def setEngineConnExecutor(engineConnExecutor: EngineConnExecutor): Unit = {
+ this.engineConnExecutor = engineConnExecutor
+ }
+
+ def getInstance: ServiceInstance = getEngineConnExecutor().getServiceInstance
+
+ private[execute] def getEngineReturns = engineReturns.toArray
+
+ override def execute(executeRequest: ExecuteRequest): ExecuteResponse = {
+ var request: RequestTask = null
+ interceptors.foreach(in => request = in.apply(request, executeRequest))
+ if (request.getProperties != null &&
+ request.getProperties.containsKey(ReconnectExecuteRequestInterceptor.PROPERTY_EXEC_ID)) {
+ val execId = ReconnectExecuteRequestInterceptor.PROPERTY_EXEC_ID.toString
+ Utils.tryCatch {
+ getEngineConnExecutor().status(execId)
+ val engineReturn = new EngineExecuteAsynReturn(request, getInstance.getInstance, execId, _ => callback())
+ engineReturns synchronized engineReturns += engineReturn
+ return engineReturn
+ } { t: Throwable =>
+ error(s"Failed to get execId $execId status", t)
+ }
+ }
+ val engineReturn = callExecute(request)
+ engineReturns synchronized engineReturns += engineReturn
+ engineReturn
+ }
+
+ protected def callback(): Unit = {}
+
+ protected def callExecute(request: RequestTask): EngineExecuteAsynReturn
+
+ override def toString: String = s"${getInstance.getApplicationName}Engine($getId, $getUser, $getCreator, ${getInstance.getInstance})"
+
+ protected def killExecId(execId: String): Boolean = {
+ info(s"begin to send killExecId, execID: $execId")
+ Utils.tryAndError(getEngineConnExecutor().killTask(execId))
+ true
+ }
+
+ override def getId: Long = this.id
+
+ override def state: ExecutorState = ExecutorState.Idle
+
+ override def getExecutorInfo: ExecutorInfo = {
+ null
+ }
+
+ def canEqual(other: Any): Boolean = other.isInstanceOf[EntranceExecutor]
+
+ override def equals(other: Any): Boolean = other match {
+ case that: EntranceExecutor =>
+ (that canEqual this) &&
+ getEngineConnExecutor().equals(that.getEngineConnExecutor())
+ case _ => false
+ }
+
+ def getExecId(jobId: String): String = {
+ val erOption = engineReturns.find(_.getJobId.contains(jobId))
+ if ( erOption.isDefined ) {
+ erOption.get.execId
+ } else {
+ null
+ }
+ }
+
+ override def hashCode(): Int = {
+ getEngineConnExecutor().hashCode()
+ }
+}
+
+class EngineExecuteAsynReturn(request: RequestTask, val instance: String,
+ val execId: String, callback: EngineExecuteAsynReturn => Unit) extends AsynReturnExecuteResponse with Logging {
+ getJobId.foreach(id => info("Job " + id + " received a execId " + execId + " from engine " + instance))
+
+ private var notifyJob: ExecuteResponse => Unit = _
+
+ private var error: Throwable = _
+
+ private var errorMsg: String = _
+
+ private var lastNotifyTime = System.currentTimeMillis
+
+ def getLastNotifyTime = lastNotifyTime
+
+ private[execute] def notifyStatus(responseEngineStatus: ResponseTaskStatus): Unit = {
+ lastNotifyTime = System.currentTimeMillis()
+ val response = responseEngineStatus.status match {
+ case Succeed => Some(SuccessExecuteResponse())
+ case Failed | Cancelled | Timeout => Some(ErrorExecuteResponse(errorMsg, error))
+ case _ => None
+ }
+ response.foreach { r =>
+ getJobId.foreach(id => info("Job " + id + " with execId-" + execId + " from engine " + instance + " completed with state " + r))
+ callback(this)
+ if (notifyJob == null) this synchronized (while (notifyJob == null) this.wait(1000))
+ notifyJob(r)
+ }
+ }
+
+ private[execute] def notifyHeartbeat(): Unit = {
+ lastNotifyTime = System.currentTimeMillis()
+ }
+
+ private[execute] def notifyError(errorMsg: String): Unit = {
+ lastNotifyTime = System.currentTimeMillis()
+ this.errorMsg = errorMsg
+ }
+
+ private[execute] def notifyError(errorMsg: String, t: Throwable): Unit = {
+ lastNotifyTime = System.currentTimeMillis()
+ this.errorMsg = errorMsg
+ this.error = t
+ }
+
+ private[execute] def getJobId: Option[String] = {
+ val jobId = request.getProperties.get(JobExecuteRequestInterceptor.PROPERTY_JOB_ID)
+ jobId match {
+ case j: String => Option(j)
+ case _ => None
+ }
+ }
+
+ override def notify(rs: ExecuteResponse => Unit): Unit = {
+ notifyJob = rs
+ this synchronized notify()
+ }
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala
new file mode 100644
index 0000000000..7c0871461b
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.execute
+
+import java.util
+import java.util.Date
+import java.util.concurrent.atomic.AtomicLong
+
+import com.webank.wedatasphere.linkis.common.exception.WarnException
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
+import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.manager.label.utils.LabelUtils
+import com.webank.wedatasphere.linkis.orchestrator.ecm.EngineConnManager
+import com.webank.wedatasphere.linkis.orchestrator.ecm.entity.{DefaultMarkReq, MarkReq, Policy}
+import com.webank.wedatasphere.linkis.orchestrator.ecm.service.impl.ComputationConcurrentEngineConnExecutor
+import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
+import com.webank.wedatasphere.linkis.scheduler.executer.{Executor, ExecutorManager}
+import com.webank.wedatasphere.linkis.scheduler.queue.{GroupFactory, Job, SchedulerEvent}
+import com.webank.wedatasphere.linkis.server.JMap
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions._
+import scala.concurrent.duration.Duration
+
+
+abstract class EntranceExecutorManager(groupFactory: GroupFactory, val engineConnManager: EngineConnManager) extends ExecutorManager with Logging {
+
+ private val idGenerator = new AtomicLong(0)
+
+ private val idToEngines = new util.HashMap[Long, EntranceExecutor]
+
+ private val instanceToEngines = new util.HashMap[String, EntranceExecutor]
+
+ def getOrCreateInterceptors(): Array[ExecuteRequestInterceptor]
+
+ override def delete(executor: Executor): Unit = {
+ if (null != executor) {
+ executor.close()
+ val entranceExecutor = idToEngines.remove(executor.getId)
+ instanceToEngines.remove(entranceExecutor.getInstance.getInstance)
+ entranceExecutor.getEngineConnExecutor() match {
+ case current: ComputationConcurrentEngineConnExecutor =>
+ case _ => engineConnManager.releaseEngineConnExecutor(entranceExecutor.getEngineConnExecutor(), entranceExecutor.mark)
+ }
+ }
+ }
+
+ protected def createMarkReq(requestPersistTask: RequestPersistTask): MarkReq = {
+ val markReq = new DefaultMarkReq
+ markReq.setPolicyObj(Policy.Task)
+ markReq.setCreateService(requestPersistTask.getCreateService)
+ markReq.setDescription(requestPersistTask.getDescription)
+ markReq.setEngineConnCount(3)
+ val properties = if (requestPersistTask.getParams == null) new util.HashMap[String, String]
+ else {
+ val startupMap = TaskUtils.getStartupMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]])
+ val properties = new JMap[String, String]
+ startupMap.foreach { case (k, v) => if (v != null && StringUtils.isNotEmpty(v.toString)) properties.put(k, v.toString) }
+ properties
+ }
+ // todo get default config from db
+ markReq.setProperties(properties)
+ markReq.setUser(requestPersistTask.getUmUser)
+ markReq.setLabels(LabelUtils.labelsToMap(requestPersistTask.getLabels))
+ markReq
+ }
+
+
+ override def askExecutor(schedulerEvent: SchedulerEvent): Option[Executor] = schedulerEvent match {
+ case job: Job =>
+ val executor = createExecutor(job)
+ if (executor != null) {
+ job match {
+ case entranceExecutionJob: EntranceExecutionJob => val task = entranceExecutionJob.getTask
+ task.asInstanceOf[RequestPersistTask].setEngineStartTime(new Date())
+ case _ =>
+ }
+ Some(executor)
+ } else None
+
+ }
+
+ override def askExecutor(schedulerEvent: SchedulerEvent, wait: Duration): Option[Executor] = schedulerEvent match {
+ case job: Job =>
+ val startTime = System.currentTimeMillis()
+ var warnException: WarnException = null
+ var executor: Option[Executor] = None
+ while (System.currentTimeMillis - startTime < wait.toMillis && executor.isEmpty)
+ Utils.tryCatch(askExecutor(job)) {
+ case warn: WarnException =>
+ this.warn("request engine failed!", warn)
+ warnException = warn
+ None
+ case t: Throwable => throw t
+ } match {
+ case Some(e) => executor = Option(e)
+ case _ =>
+ if (System.currentTimeMillis - startTime < wait.toMillis) {
+ val interval = math.min(3000, wait.toMillis - System.currentTimeMillis + startTime)
+ //getOrCreateEngineManager().waitForIdle(interval)
+ }
+ }
+ if (warnException != null && executor.isEmpty) throw warnException
+ executor
+ }
+
+ override def getById(id: Long): Option[Executor] = {
+ Option(idToEngines.get(id))
+ }
+
+ override def getByGroup(groupName: String): Array[Executor] = {
+ //TODO by peaceWong
+ null
+ }
+
+ override protected def createExecutor(schedulerEvent: SchedulerEvent): EntranceExecutor = schedulerEvent match {
+ case job: EntranceJob =>
+ job.getTask match {
+ case requestPersistTask: RequestPersistTask =>
+ // CreateMarkReq
+ val markReq = createMarkReq(requestPersistTask)
+ // getMark
+ val mark = engineConnManager.applyMark(markReq)
+ // getEngineConn Executor
+ job.getLogListener.foreach(_.onLogUpdate(job, "Background is starting a new engine for you, it may take several seconds, please wait"))
+ val engineConnExecutor = engineConnManager.getAvailableEngineConnExecutor(mark)
+ //TODO 修改Executor创建为builder模式
+ val entranceEntranceExecutor = new DefaultEntranceExecutor(idGenerator.incrementAndGet(), mark)
+ idToEngines.put(entranceEntranceExecutor.getId, entranceEntranceExecutor)
+ instanceToEngines.put(engineConnExecutor.getServiceInstance.getInstance, entranceEntranceExecutor)
+ entranceEntranceExecutor.setEngineConnExecutor(engineConnExecutor)
+ entranceEntranceExecutor.setInterceptors(getOrCreateInterceptors())
+ job.getLogListener.foreach(_.onLogUpdate(job, s" Congratulations! Your new engine has started successfully，Engine are ${engineConnExecutor.getServiceInstance}"))
+ entranceEntranceExecutor
+ case _ =>
+ throw new EntranceErrorException(20001, "Task is not requestPersistTask, cannot to create Executor")
+ }
+ case _ =>
+ throw new EntranceErrorException(20001, "Task is not EntranceJob, cannot to create Executor")
+ }
+
+ override def shutdown(): Unit = {}
+
+ def getEntranceExecutorByInstance(instance: String): Option[EntranceExecutor] = {
+ Option(instanceToEngines.get(instance))
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala
new file mode 100644
index 0000000000..9c2c7a4a04
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala
@@ -0,0 +1,196 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.execute
+
+import java.util
+import java.util.concurrent.atomic.AtomicInteger
+
+import com.webank.wedatasphere.linkis.common.log.LogUtils
+import com.webank.wedatasphere.linkis.common.utils.Utils
+import com.webank.wedatasphere.linkis.entrance.EntranceContext
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.entrance.event._
+import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
+import com.webank.wedatasphere.linkis.entrance.persistence.HaPersistenceTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.orchestrator.ecm.service.EngineConnExecutor
+import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
+import com.webank.wedatasphere.linkis.protocol.task.Task
+import com.webank.wedatasphere.linkis.rpc.utils.RPCUtils
+import com.webank.wedatasphere.linkis.scheduler.executer.{CompletedExecuteResponse, ErrorExecuteResponse}
+import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState._
+import com.webank.wedatasphere.linkis.scheduler.queue.{Job, SchedulerEventState}
+
+
+abstract class EntranceJob extends Job {
+
+ private var creator: String = _
+ private var user: String = _
+ private var params: util.Map[String, Any] = new util.HashMap[String, Any](1)
+ private var task: Task = _
+
+ private var entranceListenerBus: Option[EntranceEventListenerBus[EntranceEventListener, EntranceEvent]] = None
+ private var progressInfo: Array[JobProgressInfo] = Array.empty
+ private val persistedResultSets = new AtomicInteger(0)
+ private var resultSize = -1
+ private var entranceContext: EntranceContext = _
+
+ def getTask: Task = task
+
+ def setTask(task: Task): Unit = this.task = task
+
+ def setCreator(creator: String): Unit = this.creator = creator
+
+ def getCreator: String = creator
+
+ def setUser(user: String): Unit = this.user = user
+
+ def getUser: String = user
+
+ def setParams(params: util.Map[String, Any]): Unit = this.params = params
+
+ def getParams: util.Map[String, Any] = params
+
+ def setEntranceListenerBus(entranceListenerBus: EntranceEventListenerBus[EntranceEventListener, EntranceEvent]): Unit =
+ this.entranceListenerBus = Option(entranceListenerBus)
+
+ def getEntranceListenerBus = this.entranceListenerBus
+
+ def setProgressInfo(progressInfo: Array[JobProgressInfo]): Unit = this.progressInfo = progressInfo
+
+ def getProgressInfo: Array[JobProgressInfo] = this.progressInfo
+
+ def setEntranceContext(entranceContext: EntranceContext): Unit = this.entranceContext = entranceContext
+
+ def getEntranceContext: EntranceContext = this.entranceContext
+
+
+ def setResultSize(resultSize: Int): Unit = {
+ this.resultSize = resultSize
+ persistedResultSets synchronized persistedResultSets.notify()
+ }
+
+ def incrementResultSetPersisted(): Unit = {
+ persistedResultSets.incrementAndGet()
+ persistedResultSets synchronized persistedResultSets.notify()
+ }
+
+ protected def isWaitForPersistedTimeout(startWaitForPersistedTime: Long): Boolean =
+ System.currentTimeMillis - startWaitForPersistedTime >= EntranceConfiguration.JOB_MAX_PERSIST_WAIT_TIME.getValue.toLong
+
+
+ override def beforeStateChanged(fromState: SchedulerEventState, toState: SchedulerEventState): Unit = {
+ if (SchedulerEventState.isCompleted(toState) && (resultSize < 0 || persistedResultSets.get() < resultSize)) {
+ val startWaitForPersistedTime = System.currentTimeMillis
+ persistedResultSets synchronized {
+ while ((resultSize < 0 || persistedResultSets.get() < resultSize) && getErrorResponse == null && !isWaitForPersistedTimeout(startWaitForPersistedTime))
+ persistedResultSets.wait(3000)
+ }
+ if (isWaitForPersistedTimeout(startWaitForPersistedTime)) onFailure("persist resultSets timeout!", new EntranceErrorException(20305, "persist resultSets timeout!"))
+ if (isSucceed && getErrorResponse != null) {
+ val _toState = if (getErrorResponse.t == null) Cancelled else Failed
+ transition(_toState)
+ return
+ }
+ }
+ super.beforeStateChanged(fromState, toState)
+ }
+
+ override def afterStateChanged(fromState: SchedulerEventState, toState: SchedulerEventState): Unit = {
+ if (SchedulerEventState.isRunning(toState)) {
+ def setEngineInstance(task: Task): Unit = task match {
+ case requestTask: RequestPersistTask => getExecutor match {
+ case engine: EntranceExecutor => requestTask.setEngineInstance(engine.getInstance.getInstance)
+ case _ =>
+ }
+ case haTask: HaPersistenceTask => setEngineInstance(haTask.task)
+ case _ =>
+ }
+
+ setEngineInstance(task)
+ }
+ super.afterStateChanged(fromState, toState)
+ toState match {
+ case Scheduled =>
+ getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateInfo("Your job is Scheduled. Please wait it to run.")))
+ case WaitForRetry =>
+ getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateInfo("Your job is turn to retry. Please wait it to schedule.")))
+ case Running =>
+ getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateInfo("Your job is Running now. Please wait it to complete.")))
+ //TODO job start event
+ case _ if SchedulerEventState.isCompleted(toState) =>
+ endTime = System.currentTimeMillis()
+ if (getJobInfo != null) getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateInfo(getJobInfo.getMetric)))
+ if (isSucceed)
+ getLogListener.foreach(_.onLogUpdate(this,
+ LogUtils.generateInfo("Congratulations. Your job completed with status Success.")))
+ else getLogListener.foreach(_.onLogUpdate(this,
+ LogUtils.generateInfo(s"Sorry. Your job completed with a status $toState. You can view logs for the reason.")))
+ this.setProgress(1.0f)
+ entranceListenerBus.foreach(_.post(EntranceProgressEvent(this, 1.0f, this.getProgressInfo)))
+ this.getProgressListener.foreach(listener => listener.onProgressUpdate(this, 1.0f, Array[JobProgressInfo]()))
+ case _ =>
+ }
+ entranceListenerBus.foreach(_.post(EntranceJobEvent(this.getId)))
+ }
+
+ override def onFailure(errorMsg: String, t: Throwable): Unit = {
+ this.entranceListenerBus.foreach(_.post(
+ EntranceLogEvent(this, LogUtils.generateERROR(s"Sorry, your job executed failed with reason: $errorMsg"))))
+ super.onFailure(errorMsg, t)
+ }
+
+ override protected def transitionCompleted(executeCompleted: CompletedExecuteResponse): Unit = {
+ executeCompleted match {
+ case error: ErrorExecuteResponse if RPCUtils.isReceiverNotExists(error.t) =>
+ entranceListenerBus.foreach(_.post(MissingEngineNotifyEvent(this, error.t, getExecutor)))
+ case _ =>
+ }
+ Utils.tryAndErrorMsg(clearInstanceInfo())("Failed to clear executor")
+ super.transitionCompleted(executeCompleted)
+ }
+
+ private def clearInstanceInfo(): Unit = {
+ val executorManager = entranceContext.getOrCreateScheduler().getSchedulerContext.getOrCreateExecutorManager
+ executorManager.delete(getExecutor)
+ }
+
+ def transitionCompleted(executeCompleted: CompletedExecuteResponse, reason: String): Unit = {
+ info("Job directly completed with reason: " + reason)
+ transitionCompleted(executeCompleted)
+ }
+
+ override protected def isJobShouldRetry(errorExecuteResponse: ErrorExecuteResponse): Boolean = isJobSupportRetry && errorExecuteResponse != null &&
+ (if (RPCUtils.isReceiverNotExists(errorExecuteResponse.t)) {
+ getExecutor match {
+ case e: EntranceExecutor =>
+ val instance = e.getInstance.getInstance
+ getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateSystemWarn(s"Since the submitted engine rejects the connection, the system will automatically retry and exclude the engine $instance.(由于提交的引擎拒绝连接，系统将自动进行重试，并排除引擎 $instance.)")))
+ case _ =>
+ }
+ true
+ } else super.isJobShouldRetry(errorExecuteResponse))
+
+ def operation[T](operate: EntranceExecutor => T ): T = {
+ this.getExecutor match {
+ case entranceExecutor: EntranceExecutor =>
+ operate(entranceExecutor)
+ case _ => throw new EntranceErrorException(10000, "不支持的操作")
+ }
+
+ }
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/ExecuteRequestInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/ExecuteRequestInterceptor.scala
similarity index 81%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/ExecuteRequestInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/ExecuteRequestInterceptor.scala
index aa79e922e3..188680df82 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/ExecuteRequestInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/ExecuteRequestInterceptor.scala
@@ -16,28 +16,26 @@
package com.webank.wedatasphere.linkis.entrance.execute
-import com.webank.wedatasphere.linkis.protocol.engine.{RequestTask, RequestTaskExecute}
+import com.webank.wedatasphere.linkis.governance.common.protocol.task.{RequestTask, RequestTaskExecute}
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, JobExecuteRequest}
import scala.collection.JavaConversions
-/**
- * Created by enjoyyin on 2018/9/17.
- */
+
trait ExecuteRequestInterceptor {
def apply(requestTask: RequestTask, executeRequest: ExecuteRequest): RequestTask
}
-object LockExecuteRequestInterceptor extends ExecuteRequestInterceptor {
+object LabelExecuteRequestInterceptor extends ExecuteRequestInterceptor {
override def apply(requestTask: RequestTask, executeRequest: ExecuteRequest): RequestTask = executeRequest match {
- case lock: LockExecuteRequest =>
- val rq = if(requestTask == null) {
+ case labelExecuteRequest: LabelExecuteRequest =>
+ val rq = if (requestTask == null) {
val requestTask = new RequestTaskExecute
requestTask.setCode(executeRequest.code)
requestTask
} else requestTask
- rq.setLock(lock.lock)
+ rq.setLabels(labelExecuteRequest.labels)
rq
case _ => requestTask
}
@@ -87,15 +85,3 @@ object RuntimePropertiesExecuteRequestInterceptor extends ExecuteRequestIntercep
case _ => requestTask
}
}
-
-//object RunTypeExecuteRequestInterceptor extends ExecuteRequestInterceptor{
-// private val runType:String = "runType"
-// override def apply(requestTask: RequestTask, executeRequest: ExecuteRequest): RequestTask = {
-// executeRequest match {
-// case r:RunTypeExecuteRequest =>
-// requestTask.data(runType, r.runType)
-// case _ =>
-// }
-// requestTask
-// }
-//}
\ No newline at end of file
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/JobLockExecuteRequest.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/JobLockExecuteRequest.scala
similarity index 86%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/JobLockExecuteRequest.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/JobLockExecuteRequest.scala
index dbd09ce10a..7dbe9c4351 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/JobLockExecuteRequest.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/JobLockExecuteRequest.scala
@@ -16,15 +16,15 @@
package com.webank.wedatasphere.linkis.entrance.execute
-/**
- * Created by enjoyyin on 2018/9/15.
- */
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+
trait ReconnectExecuteRequest {
val execId: String
}
-trait LockExecuteRequest {
- val lock: String
+trait LabelExecuteRequest {
+ val labels: java.util.List[Label[_]]
}
trait StorePathExecuteRequest {
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/impl/EntranceExecutorManagerImpl.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/impl/EntranceExecutorManagerImpl.scala
new file mode 100644
index 0000000000..a86227ddf9
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/impl/EntranceExecutorManagerImpl.scala
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.execute.impl
+
+import com.webank.wedatasphere.linkis.entrance.execute._
+import com.webank.wedatasphere.linkis.orchestrator.ecm.EngineConnManager
+import com.webank.wedatasphere.linkis.scheduler.listener.ExecutorListener
+import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory
+
+/**
+ * Created by enjoyyin on 2018/9/26.
+ */
+class EntranceExecutorManagerImpl(groupFactory: GroupFactory,
+ engineConnManager: EngineConnManager) extends EntranceExecutorManager(groupFactory, engineConnManager) {
+
+
+ override def getOrCreateInterceptors(): Array[ExecuteRequestInterceptor] = Array(JobExecuteRequestInterceptor,
+ LabelExecuteRequestInterceptor, ReconnectExecuteRequestInterceptor, StorePathExecuteRequestInterceptor, RuntimePropertiesExecuteRequestInterceptor)
+
+ override def setExecutorListener(engineListener: ExecutorListener): Unit = {}
+}
\ No newline at end of file
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
similarity index 88%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
index 0d88e5afdb..b966dc6d42 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,16 +16,16 @@ package com.webank.wedatasphere.linkis.entrance.interceptor
import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.protocol.task.Task
-/**
- * Created by enjoyyin on 2018/9/4.
- */
+
+
trait EntranceInterceptor {
/**
* The apply function is to supplement the information of the incoming parameter task, making the content of this task more complete.
- * Additional information includes: database information supplement, custom variable substitution, code check, limit limit, etc.
+ * * Additional information includes: database information supplement, custom variable substitution, code check, limit limit, etc.
* apply函数是对传入参数task进行信息的补充，使得这个task的内容更加完整。
* 补充的信息包括: 数据库信息补充、自定义变量替换、代码检查、limit限制等
+ *
* @param task
* @param logAppender Used to cache the necessary reminder logs and pass them to the upper layer(用于缓存必要的提醒日志，传给上层)
* @return
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/RuntypeInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/RuntypeInterceptor.scala
similarity index 89%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/RuntypeInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/RuntypeInterceptor.scala
index 3b14958a6c..4a3101d983 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/RuntypeInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/RuntypeInterceptor.scala
@@ -15,18 +15,18 @@
*/
package com.webank.wedatasphere.linkis.entrance.interceptor
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
/**
- * created by enjoyyin on 2018/12/5
* Description: this interceptor is used to complete code with run type for
* further use in engine
*/
class RuntypeInterceptor extends EntranceInterceptor {
- override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match{
- case requestPersistTask:RequestPersistTask =>
+ override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match {
+ case requestPersistTask: RequestPersistTask =>
requestPersistTask.getRunType.toLowerCase() match {
case "python" | "py" | "pyspark" => val code = requestPersistTask.getExecutionCode
requestPersistTask.setExecutionCode("%python\n" + code)
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/CodeCheckException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/CodeCheckException.scala
similarity index 79%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/CodeCheckException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/CodeCheckException.scala
index 2a8469c5db..2f648b7a9a 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/CodeCheckException.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/CodeCheckException.scala
@@ -18,11 +18,8 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.exception
import com.webank.wedatasphere.linkis.common.exception.ErrorException
-/**
- * created by enjoyyin on 2018/10/20
- * Description:
- */
-case class CodeCheckException(errCode:Int,
- errDesc:String) extends ErrorException(errCode, errDesc){
+
+case class CodeCheckException(errCode: Int,
+ errDesc: String) extends ErrorException(errCode, errDesc) {
}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LabelCheckException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LabelCheckException.scala
new file mode 100644
index 0000000000..47bb4a12df
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LabelCheckException.scala
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.interceptor.exception
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+
+
+case class LabelCheckException(errCode: Int,
+ errDesc: String) extends ErrorException(errCode, errDesc) {
+
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LimitCheckException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LimitCheckException.scala
similarity index 78%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LimitCheckException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LimitCheckException.scala
index 708b2a8dcf..3d4b2e43b1 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LimitCheckException.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LimitCheckException.scala
@@ -18,11 +18,8 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.exception
import com.webank.wedatasphere.linkis.common.exception.ErrorException
-/**
- * created by enjoyyin on 2018/10/20
- * Description:
- */
-case class LimitCheckException(errCode:Int,
- errDesc:String) extends ErrorException(errCode, errDesc) {
+
+case class LimitCheckException(errCode: Int,
+ errDesc: String) extends ErrorException(errCode, errDesc) {
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LogPathCreateException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LogPathCreateException.scala
similarity index 78%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LogPathCreateException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LogPathCreateException.scala
index 7ab4a857b6..8db8a6a3e4 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LogPathCreateException.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/LogPathCreateException.scala
@@ -18,12 +18,9 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.exception
import com.webank.wedatasphere.linkis.common.exception.ErrorException
-/**
- * created by enjoyyin on 2018/10/23
- * Description:
- */
-case class LogPathCreateException(errCode:Int,
- errDesc:String)
- extends ErrorException(errCode,errDesc){
+
+case class LogPathCreateException(errCode: Int,
+ errDesc: String)
+ extends ErrorException(errCode, errDesc) {
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/PythonCodeCheckException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/PythonCodeCheckException.scala
similarity index 80%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/PythonCodeCheckException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/PythonCodeCheckException.scala
index 273e2a78a9..f91767bf53 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/PythonCodeCheckException.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/PythonCodeCheckException.scala
@@ -18,12 +18,9 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.exception
import com.webank.wedatasphere.linkis.common.exception.ErrorException
-/**
- * created by enjoyyin on 2018/10/22
- * Description:
- */
-case class PythonCodeCheckException(errCode:Int,
- errDesc:String)extends ErrorException(errCode, errDesc) {
+
+case class PythonCodeCheckException(errCode: Int,
+ errDesc: String) extends ErrorException(errCode, errDesc) {
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
similarity index 78%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
index b371fdacbe..2b965e291f 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
@@ -18,10 +18,7 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.exception
import com.webank.wedatasphere.linkis.common.exception.ErrorException
-/**
- * created by enjoyyin on 2018/10/19
- * Description:
- */
-case class VarSubstitutionException(errCode:Int,
- errDesc:String) extends ErrorException(errCode,errDesc){
+
+case class VarSubstitutionException(errCode: Int,
+ errDesc: String) extends ErrorException(errCode, errDesc) {
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala
similarity index 92%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala
index 26629f847a..943b1b298f 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala
@@ -18,13 +18,10 @@ import java.lang
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.cs.CSEntranceHelper
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
-/**
- * @author peacewong
- * @date 2020/3/24 18:28
- */
+
class CSEntranceInterceptor extends EntranceInterceptor with Logging {
override def apply(task: Task, logAppender: lang.StringBuilder): Task = {
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala
new file mode 100644
index 0000000000..a9e1911657
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.interceptor.impl
+
+import java.lang
+import java.util.regex.Pattern
+
+import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.task.Task
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.util.matching.Regex
+
+
+class CommentInterceptor extends EntranceInterceptor {
+ /**
+ * The apply function is to supplement the information of the incoming parameter task, making the content of this task more complete.
+ * * Additional information includes: database information supplement, custom variable substitution, code check, limit limit, etc.
+ * apply函数是对传入参数task进行信息的补充，使得这个task的内容更加完整。
+ * 补充的信息包括: 数据库信息补充、自定义变量替换、代码检查、limit限制等
+ *
+ * @param task
+ * @param logAppender Used to cache the necessary reminder logs and pass them to the upper layer(用于缓存必要的提醒日志，传给上层)
+ * @return
+ */
+ override def apply(task: Task, logAppender: lang.StringBuilder): Task = task match {
+ case requestPersistTask: RequestPersistTask => requestPersistTask.getRunType match {
+ case "sql" | "hql" => requestPersistTask.setExecutionCode(SQLCommentHelper.dealComment(requestPersistTask.getExecutionCode))
+ case "python" | "py" => requestPersistTask.setExecutionCode(PythonCommentHelper.dealComment(requestPersistTask.getExecutionCode))
+ case "scala" | "java" => requestPersistTask.setExecutionCode(ScalaCommentHelper.dealComment(requestPersistTask.getExecutionCode))
+ case "sh" | "shell" =>
+ case _ =>
+ }
+ requestPersistTask
+ case _ => task
+ }
+}
+
+trait CommentHelper {
+ val commentPattern: Regex
+
+ def dealComment(code: String): String
+}
+
+object SQLCommentHelper extends CommentHelper {
+ override val commentPattern: Regex = """\s*--.+\s*""".r.unanchored
+ private val comment = "(?ms)('(?:''|[^'])*')|--.*?$|/\\*.*?\\*/|#.*?$|"
+ private val logger: Logger = LoggerFactory.getLogger(getClass)
+
+ override def dealComment(code: String): String = {
+ try {
+ val p = Pattern.compile(comment)
+ val sql = p.matcher(code).replaceAll("$1")
+ sql
+ } catch {
+ case e: Exception => logger.warn("sql comment failed")
+ code
+ case t: Throwable => logger.warn("sql comment failed")
+ code
+ }
+ }
+}
+
+object PythonCommentHelper extends CommentHelper {
+ override val commentPattern: Regex = """^\s*#.+\s*""".r.unanchored
+ val pythonCommentPattern: String = "(?ms)([\"'](?:|[^'])*['\"])|#.*?$|/\\*.*?\\*/"
+
+ override def dealComment(code: String): String = {
+ code
+ }
+}
+
+
+object ScalaCommentHelper extends CommentHelper {
+ override val commentPattern: Regex = """^\s*//.+\s*""".r.unanchored
+ private val scalaCommentPattern: String = "(?ms)([\"'](?:|[^'])*['\"])|//.*?$|/\\*.*?\\*/"
+
+ override def dealComment(code: String): String = code
+}
+
+
+object CommentMain {
+ def main(args: Array[String]): Unit = {
+ val sqlCode = "select * from default.user;--你好;show tables"
+ val sqlCode1 = "select * from default.user--你好;show tables"
+ println(SQLCommentHelper.dealComment(sqlCode))
+ }
+}
+
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
similarity index 81%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
index 334960e7ac..3ac8e8057e 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,7 +13,6 @@
package com.webank.wedatasphere.linkis.entrance.interceptor.impl
-
import java.text.SimpleDateFormat
import java.util
import java.util.{Calendar, Date}
@@ -24,7 +20,7 @@ import java.util.{Calendar, Date}
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.VarSubstitutionException
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
import com.webank.wedatasphere.linkis.protocol.variable.{RequestQueryAppVariable, ResponseQueryVariable}
@@ -37,9 +33,7 @@ import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.util.control.Exception._
-/**
- * Created by enjoyyin on 11/14/17.
- */
+
object CustomVariableUtils extends Logging {
//hql sql jdbc to sql python to py
val SQL_TYPE = "sql"
@@ -49,36 +43,37 @@ object CustomVariableUtils extends Logging {
val R_TYPE:String = "r"
val RUN_DATE = "run_date"
val TEAM:String = "team"
- val SHELL_TYPE:String="shell"
/**
* date Format
*/
val dateFormat = new SimpleDateFormat("yyyyMMdd")
val dateFormat_std = new SimpleDateFormat("yyyy-MM-dd")
+
/**
* replace custom variable
* 1. Get the user-defined variable from the code and replace it
- * 2. If 1 is not done, then get the user-defined variable from args and replace it.
- * 3. If 2 is not done, get the user-defined variable from the console and replace it.
-
+ * * 2. If 1 is not done, then get the user-defined variable from args and replace it.
+ * * 3. If 2 is not done, get the user-defined variable from the console and replace it.
+ *
* 1. 从代码中得到用户定义的变量，进行替换
* 2. 如果1没有做，那么从args中得到用户定义的变量，进行替换
* 3. 如果2没有做，从CS中得到用户定义的变量，进行替换
*3. 如果3没有做，从控制台中得到用户定义的变量，进行替换
+ *
* @param task : requestPersistTask
* @return
*/
def replaceCustomVar(task: Task, runType: String): (Boolean, String) = {
- val code:String = task.asInstanceOf[RequestPersistTask].getExecutionCode
- val umUser:String = task.asInstanceOf[RequestPersistTask].getUmUser
+ val code: String = task.asInstanceOf[RequestPersistTask].getExecutionCode
+ val umUser: String = task.asInstanceOf[RequestPersistTask].getUmUser
var codeType = SQL_TYPE
runType match {
- case "hql" | "sql" | "jdbc" | "hive" | "psql" | "essql" => codeType = SQL_TYPE
- case "python" => codeType = PY_TYPE
+ case "hql" | "sql" | "jdbc" | "hive" => codeType = SQL_TYPE
+ case "python" | "py" => codeType = PY_TYPE
case "java" => codeType = JAVA_TYPE
- case "scala" | "esjson" => codeType = SCALA_TYPE
- case "shell" | "sh" => codeType = SHELL_TYPE
+ case "scala" => codeType = SCALA_TYPE
+ case "sh" | "shell" => codeType = SQL_TYPE
case _ => return (false, code)
}
@@ -101,7 +96,7 @@ object CustomVariableUtils extends Logging {
} else {
nameAndType(key) = StringType(value)
}
- }
+ }
}
}
@@ -110,26 +105,26 @@ object CustomVariableUtils extends Logging {
putNameAndType(nameAndValue)
task match {
- case requestPersistTask:RequestPersistTask =>
+ case requestPersistTask: RequestPersistTask =>
/* Perform the second step to replace the parameters passed in args*/
/* 进行第二步，对args传进的参数进行替换*/
val variableMap = TaskUtils.getVariableMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]])
- .map{case (k, v) => k -> v.asInstanceOf[String]}
+ .map { case (k, v) => k -> v.asInstanceOf[String] }
putNameAndType(variableMap)
case _ =>
}
- /* Go to the four step and take the user's parameters to the cloud-publicservice module.*/
- /*进行第四步，向cloud-publicservice模块去拿用户的参数*/
+ /* Go to the four step and take the user's parameters to the linkis-ps-publicservice module.*/
+ /*进行第四步，向linkis-ps-publicservice模块去拿用户的参数*/
val sender = Sender.getSender(EntranceConfiguration.CLOUD_CONSOLE_VARIABLE_SPRING_APPLICATION_NAME.getValue)
task match {
- case requestPersistTask:RequestPersistTask =>
- val umUser:String = requestPersistTask.getUmUser
- val creator:String = requestPersistTask.getRequestApplicationName
- val runType:String = requestPersistTask.getRunType
- val requestQueryAppVariable:RequestQueryAppVariable = RequestQueryAppVariable(umUser, creator, runType)
- val response:ResponseQueryVariable = sender.ask(requestQueryAppVariable).asInstanceOf[ResponseQueryVariable]
+ case requestPersistTask: RequestPersistTask =>
+ val umUser: String = requestPersistTask.getUmUser
+ val creator: String = requestPersistTask.getRequestApplicationName
+ val runType: String = requestPersistTask.getRunType
+ val requestQueryAppVariable: RequestQueryAppVariable = RequestQueryAppVariable(umUser, creator, runType)
+ val response: ResponseQueryVariable = sender.ask(requestQueryAppVariable).asInstanceOf[ResponseQueryVariable]
val keyAndValue = response.getKeyAndValue
- val keyAndValueScala:mutable.Map[String, String] = keyAndValue
+ val keyAndValueScala: mutable.Map[String, String] = keyAndValue
putNameAndType(keyAndValueScala)
case _ =>
}
@@ -163,59 +158,55 @@ object CustomVariableUtils extends Logging {
*/
def parserVar(code: String, nameAndType: mutable.Map[String, VariableType]): String = {
- val codeReg = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r
- val calReg = "(\\s*[A-Za-z][A-Za-z0-9_]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r
+ val codeReg = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r
+ val calReg = "(\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r
+ val parseCode = new StringBuilder
+ val codes = codeReg.split(code)
val expressionCache = mutable.HashSet[String]()
- var targetCode = code
- var noChange = false
- while (!noChange && codeReg.findFirstIn(targetCode).isDefined) {
- var i = 0
- val parseCode = new StringBuilder
- val codes = codeReg.split(targetCode)
- codeReg.findAllIn(targetCode).foreach(str => {
- calReg.findFirstMatchIn(str).foreach(ma => {
- i = i + 1
- val name = ma.group(1)
- val signal = ma.group(2)
- val bValue = ma.group(3)
-
- if (name == null || name.trim.isEmpty) {
- throw VarSubstitutionException(20041,s"[$str] replaced var is null")
+
+ var i = 0
+
+ codeReg.findAllIn(code).foreach(str => {
+ calReg.findFirstMatchIn(str).foreach(ma => {
+ i = i + 1
+ val name = ma.group(1)
+ val signal = ma.group(2)
+ val bValue = ma.group(3)
+
+ if (name == null || name.trim.isEmpty) {
+ throw VarSubstitutionException(20041,s"[$str] replaced var is null")
+ } else {
+ var expression = name.trim
+ val varType = nameAndType.get(name.trim).orNull
+ if (varType == null) {
+ warn(s"Use undefined variables or use the set method: [$str](使用了未定义的变量或者使用了set方式:[$str])")
+ parseCode ++= codes(i - 1) ++ str
} else {
- var expression = name.trim
- val varType = nameAndType.get(name.trim).orNull
- if (varType == null) {
- warn(s"Use undefined variables or use the set method: [$str](使用了未定义的变量或者使用了set方式:[$str])")
- parseCode ++= codes(i - 1) ++ str
- } else {
- var res: String = varType.getValue
- if (signal != null && !signal.trim.isEmpty) {
- if (bValue == null || bValue.trim.isEmpty) {
- throw VarSubstitutionException(20042, s"[$str] expression is not right, please check")
- } else {
- expression = expression + "_" + signal.trim + "_" + bValue.trim
- res = varType.calculator(signal.trim, bValue.trim)
- }
+ var res: String = varType.getValue
+ if (signal != null && !signal.trim.isEmpty) {
+ if (bValue == null || bValue.trim.isEmpty) {
+ throw VarSubstitutionException(20042, s"[$str] expression is not right, please check")
+ } else {
+ expression = expression + "_" + signal.trim + "_" + bValue.trim
+ res = varType.calculator(signal.trim, bValue.trim)
}
- if (!expressionCache.contains(expression)) {
- info(s"Variable expression [$str] = $res(变量表达式[$str] = $res)")
- //println(s"变量表达式[$str] = $res")
- expressionCache += expression
- }
- //println(s"变量表达式序号:$i\t[$str] = $res")
- parseCode ++= codes(i - 1) ++ res
}
+ if (!expressionCache.contains(expression)) {
+ info(s"Variable expression [$str] = $res(变量表达式[$str] = $res)")
+ //println(s"变量表达式[$str] = $res")
+ expressionCache += expression
+ }
+ //println(s"变量表达式序号:$i\t[$str] = $res")
+ parseCode ++= codes(i - 1) ++ res
}
- })
+ }
})
- if (i == codes.length - 1) {
- parseCode ++= codes(i)
- }
- val parsedCode = org.apache.commons.lang.StringUtils.strip(deleteUselessSemicolon(parseCode))
- if (targetCode.equals(parsedCode)) noChange = true
- targetCode = parsedCode
+ })
+ if (i == codes.length - 1) {
+ parseCode ++= codes(i)
}
- targetCode
+ val parsedCode = deleteUselessSemicolon(parseCode)
+ org.apache.commons.lang.StringUtils.strip(parsedCode)
// Utils.trimBlank()
}
@@ -310,8 +301,6 @@ object CustomVariableUtils extends Logging {
case SCALA_TYPE => varString = """\s*//@set\s*.+\s*"""
errString = """\s*//@.+"""
case JAVA_TYPE => varString = """\s*!!@set\s*.+\s*"""
- case SHELL_TYPE => varString = """\s*#@set\s*.+\s*"""
- errString = """\s*#@"""
}
val customRegex = varString.r.unanchored
@@ -387,7 +376,7 @@ object CustomVariableUtils extends Logging {
def main(args: Array[String]): Unit = {
val code = "--@set a=1\n--@set b=2\nselect ${a +2},${a + 1},${a},${a },${b},${b}"
val task = new RequestPersistTask
- val args:java.util.Map[String, Object] = new util.HashMap[String, Object]()
+ val args: java.util.Map[String, Object] = new util.HashMap[String, Object]()
args.put(RUN_DATE, "20181030")
task.setExecutionCode(code)
task.setParams(args)
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/DBInfoCompleteInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/DBInfoCompleteInterceptor.scala
similarity index 83%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/DBInfoCompleteInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/DBInfoCompleteInterceptor.scala
index f3a65260ee..890963680b 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/DBInfoCompleteInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/DBInfoCompleteInterceptor.scala
@@ -19,11 +19,8 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.impl
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
import com.webank.wedatasphere.linkis.protocol.task.Task
-/**
- * created by enjoyyin on 2018/10/19
- * Description: For database information completion(用于数据库信息补全)
- */
-class DBInfoCompleteInterceptor extends EntranceInterceptor{
+
+class DBInfoCompleteInterceptor extends EntranceInterceptor {
override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = {
//todo Database information completion(数据库信息补全)
task
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
similarity index 84%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
index e6c54a95f4..6b4eb7a131 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
@@ -22,17 +22,15 @@ import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.common.log.LogUtils
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.{PythonCodeCheckException, ScalaCodeCheckException}
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import org.apache.commons.lang.StringUtils
import org.slf4j.{Logger, LoggerFactory}
import scala.collection.mutable.ArrayBuffer
-/**
- * created by enjoyyin on 2018/10/19
- * Description:
- */
+
abstract class Explain extends Logging {
/**
* 用于检查code是否符合规范
@@ -50,18 +48,18 @@ object SparkExplain extends Explain {
private val sy = Pattern.compile("sys\\.")
private val scCancelAllJobs = Pattern.compile("sc\\.cancelAllJobs(\\s*)")
private val runtime = Pattern.compile("Runtime\\.getRuntime")
+ private val LINE_BREAK = "\n"
private val LOG:Logger = LoggerFactory.getLogger(getClass)
override def authPass(code: String, error: StringBuilder): Boolean = {
-
+ if (EntranceConfiguration.IS_QML.getValue) {
+ return true
+ }
if (scStop.matcher(code).find()) {
error.append("Authentication error: sc.stop() is not allowed in IDE.")
false
} else if (systemExit.matcher(code).find()) {
error.append("Authentication error: System.exit(exitCode) is not allowed in IDE.")
false
- } else if (sy.matcher(code).find()) {
- error.append("Authentication error: the package `sys` cannot be used in IDE.")
- false
} else if (scCancelAllJobs.matcher(code).find()) {
error.append("Authentication error: sc.cancelAllJobs() is not allowed in IDE.")
false
@@ -82,6 +80,7 @@ object SQLExplain extends Explain {
val SQL_APPEND_LIMIT:String = " limit " + SQL_DEFAULT_LIMIT.getValue
val DROP_TABLE_SQL = "\\s*drop\\s+table\\s+\\w+\\s*"
val CREATE_DATABASE_SQL = "\\s*create\\s+database\\s+\\w+\\s*"
+ private val IDE_ALLOW_NO_LIMIT_REGEX = "--set\\s*ide.engine.no.limit.allow\\s*=\\s*true".r.unanchored
private val LINE_BREAK = "\n"
private val COMMENT_FLAG = "--"
val SET_OWN_USER = "set owner user"
@@ -98,20 +97,31 @@ object SQLExplain extends Explain {
/**
* to deal with sql limit
- * @param executionCode sql code
+ *
+ * @param executionCode sql code
* @param requestPersistTask use to store inited logs
*/
- def dealSQLLimit(executionCode:String, requestPersistTask: RequestPersistTask, logAppender: java.lang.StringBuilder):Unit = {
- val fixedCode:ArrayBuffer[String] = new ArrayBuffer[String]()
- executionCode.split(";") foreach { singleCode =>
+ def dealSQLLimit(executionCode: String, requestPersistTask: RequestPersistTask, logAppender: java.lang.StringBuilder): Unit = {
+ val fixedCode: ArrayBuffer[String] = new ArrayBuffer[String]()
+ val tempCode = SQLCommentHelper.dealComment(executionCode)
+ val isNoLimitAllowed = Utils.tryCatch {
+ IDE_ALLOW_NO_LIMIT_REGEX.findFirstIn(executionCode).isDefined
+ } {
+ case e: Exception => logger.warn("sql limit check error happens")
+ executionCode.contains(IDE_ALLOW_NO_LIMIT)
+ }
+ if (isNoLimitAllowed) logAppender.append(LogUtils.generateWarn("请注意,SQL全量导出模式打开\n"))
+ tempCode.split(";") foreach { singleCode =>
if (isSelectCmd(singleCode)){
val trimCode = singleCode.trim
- if (isSelectCmdNoLimit(trimCode) && !executionCode.contains(IDE_ALLOW_NO_LIMIT)){
- logAppender.append(LogUtils.generateWarn(s"You submitted a sql $trimCode without limit, program will add limit 5000 to your sql") + "\n")
- fixedCode += (trimCode + SQL_APPEND_LIMIT)
- }else if (isSelectOverLimit(singleCode) && !executionCode.contains(IDE_ALLOW_NO_LIMIT)){
+ if (isSelectCmdNoLimit(trimCode) && !isNoLimitAllowed){
+ logAppender.append(LogUtils.generateWarn(s"You submitted a sql without limit, DSS will add limit 5000 to your sql") + "\n")
+ //将注释先干掉,然后再进行添加limit
+ val realCode = cleanComment(trimCode)
+ fixedCode += (realCode + SQL_APPEND_LIMIT)
+ }else if (isSelectOverLimit(singleCode) && !isNoLimitAllowed){
val trimCode = singleCode.trim
- logAppender.append(LogUtils.generateWarn(s"You submitted a sql $trimCode with limit exceeding 5000, it is not allowed. Program will change your limit to 5000") + "\n")
+ logAppender.append(LogUtils.generateWarn(s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000") + "\n")
fixedCode += repairSelectOverLimit(trimCode)
}else{
fixedCode += singleCode.trim
@@ -236,6 +246,9 @@ object PythonExplain extends Explain{
*/
private val SC_STOP = """sc\.stop""".r.unanchored
override def authPass(code: String, error: StringBuilder): Boolean = {
+ if (EntranceConfiguration.IS_QML.getValue) {
+ return true
+ }
code.split(System.lineSeparator()) foreach {code =>
if (IMPORT_SYS_MOUDLE.findAllIn(code).nonEmpty || FROM_SYS_IMPORT.findAllIn(code).nonEmpty)
throw PythonCodeCheckException(20070, "can not use sys module")
@@ -257,6 +270,9 @@ object ScalaExplain extends Explain{
private val runtime = """Runtime.getRunTime""".r.unanchored
private val LOG:Logger = LoggerFactory.getLogger(getClass)
override def authPass(code: String, error: StringBuilder): Boolean = {
+ if (EntranceConfiguration.IS_QML.getValue) {
+ return true
+ }
code match {
case systemExit() => LOG.error("scala code can not use System.exit")
throw ScalaCodeCheckException(20074, "scala code can not use System.exit")
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/LabelCheckInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/LabelCheckInterceptor.scala
new file mode 100644
index 0000000000..826940ecd1
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/LabelCheckInterceptor.scala
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.interceptor.impl
+
+import java.lang
+
+import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
+import com.webank.wedatasphere.linkis.entrance.interceptor.exception.LabelCheckException
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel}
+import com.webank.wedatasphere.linkis.protocol.task.Task
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions._
+
+
+class LabelCheckInterceptor extends EntranceInterceptor {
+
+ /**
+ * The apply function is to supplement the information of the incoming parameter task, making the content of this task more complete.
+ * * Additional information includes: database information supplement, custom variable substitution, code check, limit limit, etc.
+ * apply函数是对传入参数task进行信息的补充，使得这个task的内容更加完整。
+ * 补充的信息包括: 数据库信息补充、自定义变量替换、代码检查、limit限制等
+ *
+ * @param task
+ * @param logAppender Used to cache the necessary reminder logs and pass them to the upper layer(用于缓存必要的提醒日志，传给上层)
+ * @return
+ */
+ override def apply(task: Task, logAppender: lang.StringBuilder): Task = {
+ task match {
+ case requestPersistTask: RequestPersistTask =>
+ val labels = requestPersistTask.getLabels
+ checkEngineTypeLabel(labels)
+ checkUserCreatorLabel(labels)
+ task
+ case _ => task
+ }
+ }
+
+ private def checkEngineTypeLabel(labels: java.util.List[Label[_]]): Unit = {
+ val engineTypeLabelOption = labels.find(_.isInstanceOf[EngineTypeLabel])
+ if (engineTypeLabelOption.isDefined) {
+ val engineLabel = engineTypeLabelOption.get.asInstanceOf[EngineTypeLabel]
+ if (StringUtils.isNotBlank(engineLabel.getEngineType)) {
+ return
+ }
+ }
+ throw LabelCheckException(50079, "engineTypeLabel must be need")
+ }
+
+
+ private def checkUserCreatorLabel(labels: java.util.List[Label[_]]): Unit = {
+ val userCreatorLabelOption = labels.find(_.isInstanceOf[UserCreatorLabel])
+ if (userCreatorLabelOption.isDefined) {
+ val userCreator = userCreatorLabelOption.get.asInstanceOf[UserCreatorLabel]
+ if (StringUtils.isNotBlank(userCreator.getUser)) {
+ return
+ }
+ }
+ throw LabelCheckException(50079, "UserCreatorLabel must be need")
+ }
+
+
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/LogPathCreateInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/LogPathCreateInterceptor.scala
new file mode 100644
index 0000000000..9b806e6a1b
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/LogPathCreateInterceptor.scala
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.interceptor.impl
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.cache.GlobalConfigurationKeyValueCache
+import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
+import com.webank.wedatasphere.linkis.entrance.interceptor.exception.LogPathCreateException
+import com.webank.wedatasphere.linkis.entrance.parser.ParserUtils
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.task.Task
+
+/**
+ * Description:Log path generation interceptor, used to set the path log of the task(日志路径生成拦截器, 用于设置task的路径日志)
+ */
+class LogPathCreateInterceptor extends EntranceInterceptor with Logging {
+
+ @throws[ErrorException]
+ override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = {
+ task match {
+ case requestPersistTask: RequestPersistTask => Utils.tryThrow {
+ ParserUtils.generateLogPath(requestPersistTask, Utils.tryAndWarn(GlobalConfigurationKeyValueCache.getCacheMap(requestPersistTask)))
+ requestPersistTask
+ } {
+ case e: ErrorException =>
+ val exception: LogPathCreateException = LogPathCreateException(20075, "Failed to get logPath(获取logPath失败)，reason: " + e.getMessage)
+ exception.initCause(e)
+ exception
+ case t: Throwable =>
+ val exception: LogPathCreateException = LogPathCreateException(20075, "Failed to get logPath(获取logPath失败), reason: " + t.getCause)
+ exception.initCause(t)
+ exception
+ }
+ case _ => task
+ }
+ }
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
similarity index 83%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
index 5c4c95b76d..ceabc45885 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
@@ -19,22 +19,21 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.impl
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.PythonCodeCheckException
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
import org.apache.commons.lang.exception.ExceptionUtils
/**
- * created by enjoyyin on 2018/10/22
* Description: Check for python code, prohibiting the use of sys, os, and creating processes(用于python代码的检查，禁止使用sys、os以及创建进程等行为)
*/
-class PythonCodeCheckInterceptor extends EntranceInterceptor{
- override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match{
- case requestPersistTask:RequestPersistTask =>
+class PythonCodeCheckInterceptor extends EntranceInterceptor {
+ override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match {
+ case requestPersistTask: RequestPersistTask =>
val error = new StringBuilder
requestPersistTask.getRunType match {
- case "python" | "pyspark" =>
- Utils.tryThrow(PythonExplain.authPass(requestPersistTask.getExecutionCode, error)){
- case PythonCodeCheckException(errCode,errDesc) =>
+ case "python" | "pyspark" | "py" =>
+ Utils.tryThrow(PythonExplain.authPass(requestPersistTask.getExecutionCode, error)) {
+ case PythonCodeCheckException(errCode, errDesc) =>
requestPersistTask.setErrCode(errCode)
requestPersistTask.setErrDesc(errDesc)
PythonCodeCheckException(errCode, errDesc)
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLCodeCheckInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLCodeCheckInterceptor.scala
new file mode 100644
index 0000000000..5bad733844
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLCodeCheckInterceptor.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.interceptor.impl
+
+import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
+import com.webank.wedatasphere.linkis.entrance.interceptor.exception.CodeCheckException
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.task.Task
+
+/**
+ * Description:
+ */
+class SQLCodeCheckInterceptor extends EntranceInterceptor {
+
+ override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match {
+ case requestPersistTask: RequestPersistTask =>
+ requestPersistTask.getEngineType.toLowerCase() match {
+ case "hql" | "sql" | "jdbc" | "hive" =>
+ val sb: StringBuilder = new StringBuilder
+ val isAuth: Boolean = SQLExplain.authPass(requestPersistTask.getExecutionCode, sb)
+ if (!isAuth) {
+ throw CodeCheckException(20051, "sql code check failed, reason is " + sb.toString())
+ }
+ case _ =>
+ }
+ requestPersistTask
+ }
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
similarity index 77%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
index 05081e3d08..83943defb1 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
@@ -17,22 +17,21 @@
package com.webank.wedatasphere.linkis.entrance.interceptor.impl
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
/**
- * created by enjoyyin on 2018/10/19
* Description: LimitEntranceInterceptor Interceptors are used to set certain restrictions(拦截器用于对某些限制的设定)
*/
class SQLLimitEntranceInterceptor extends EntranceInterceptor {
override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = {
task match {
- case requestPersistTask:RequestPersistTask => requestPersistTask.getEngineType.toLowerCase() match {
- case "hql" | "sql" | "jdbc"|"hive" | "psql" | "essql" => val executionCode = requestPersistTask.getExecutionCode
+ case requestPersistTask: RequestPersistTask => requestPersistTask.getEngineType.toLowerCase() match {
+ case "hql" | "sql" | "jdbc" | "hive" => val executionCode = requestPersistTask.getExecutionCode
SQLExplain.dealSQLLimit(executionCode, requestPersistTask, logAppender)
case _ =>
}
- requestPersistTask
+ requestPersistTask
case _ => task
}
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
similarity index 79%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
index 557504f46e..cafa99aa63 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
@@ -21,25 +21,22 @@ import java.lang
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.ScalaCodeCheckException
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
-/**
- * created by enjoyyin on 2019/2/25
- * Description:
- */
+
class ScalaCodeInterceptor extends EntranceInterceptor {
private val SCALA_TYPE = "scala"
- override def apply(task: Task, logAppender: lang.StringBuilder): Task = task match{
- case requestPersistTask:RequestPersistTask => val error = new StringBuilder
+ override def apply(task: Task, logAppender: lang.StringBuilder): Task = task match {
+ case requestPersistTask: RequestPersistTask => val error = new StringBuilder
requestPersistTask.getRunType match {
- case SCALA_TYPE => Utils.tryThrow(ScalaExplain.authPass(requestPersistTask.getExecutionCode, error)){
+ case SCALA_TYPE => Utils.tryThrow(ScalaExplain.authPass(requestPersistTask.getExecutionCode, error)) {
case ScalaCodeCheckException(errorCode, errDesc) => requestPersistTask.setErrCode(errorCode)
requestPersistTask.setErrDesc(errDesc)
ScalaCodeCheckException(errorCode, errDesc)
- case t:Throwable => val exception = ScalaCodeCheckException(20074, "Scala code check failed(scala代码检查失败)")
+ case t: Throwable => val exception = ScalaCodeCheckException(20074, "Scala code check failed(scala代码检查失败)")
exception.initCause(t)
exception
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
similarity index 84%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
index 3a6beb5391..1b3d13edb4 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
@@ -18,25 +18,24 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.impl
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.CodeCheckException
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
/**
- * created by enjoyyin on 2018/10/22
* Description:
* Yòng yú jiǎnchá spark dàimǎ
* 11/5000
* Used to check the spark code(用于检查spark代码)
*/
-class SparkCodeCheckInterceptor extends EntranceInterceptor{
+class SparkCodeCheckInterceptor extends EntranceInterceptor {
override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = {
task match {
case requestPersistTask: RequestPersistTask =>
requestPersistTask.getRunType.toLowerCase() match {
- case "scala" => val stringBuilder:StringBuilder = new StringBuilder()
+ case "scala" => val stringBuilder: StringBuilder = new StringBuilder()
val isAuth = SparkExplain.authPass(requestPersistTask.getExecutionCode, stringBuilder)
- if (!isAuth){
+ if (!isAuth) {
throw CodeCheckException(20050, "spark code check failed")
}
case _ =>
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/StorePathEntranceInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/StorePathEntranceInterceptor.scala
similarity index 78%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/StorePathEntranceInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/StorePathEntranceInterceptor.scala
index 1216592ac1..11d8ee3ace 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/StorePathEntranceInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/StorePathEntranceInterceptor.scala
@@ -16,20 +16,19 @@
package com.webank.wedatasphere.linkis.entrance.interceptor.impl
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.cache.GlobalConfigurationKeyValueCache
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
import org.apache.commons.lang.time.DateFormatUtils
-/**
- * Created by enjoyyin on 2018/11/3.
- */
-class StorePathEntranceInterceptor extends EntranceInterceptor {
+
+class StorePathEntranceInterceptor extends EntranceInterceptor with Logging {
/**
* The apply function is to supplement the information of the incoming parameter task, making the content of this task more complete.
- * Additional information includes: database information supplement, custom variable substitution, code check, limit limit, etc.
+ * * Additional information includes: database information supplement, custom variable substitution, code check, limit limit, etc.
* apply函数是对传入参数task进行信息的补充，使得这个task的内容更加完整。
* 补充的信息包括: 数据库信息补充、自定义变量替换、代码检查、limit限制等
*
@@ -38,9 +37,9 @@ class StorePathEntranceInterceptor extends EntranceInterceptor {
*/
override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match {
case persistTask: RequestPersistTask =>
- val globalConfig = GlobalConfigurationKeyValueCache.getCacheMap(persistTask)
+ val globalConfig = Utils.tryAndWarn(GlobalConfigurationKeyValueCache.getCacheMap(persistTask))
var parentPath: String = null
- if (globalConfig.containsKey(EntranceConfiguration.RESULT_SET_STORE_PATH.key))
+ if (null != globalConfig && globalConfig.containsKey(EntranceConfiguration.RESULT_SET_STORE_PATH.key))
parentPath = EntranceConfiguration.RESULT_SET_STORE_PATH.getValue(globalConfig)
else {
parentPath = EntranceConfiguration.RESULT_SET_STORE_PATH.getValue
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
similarity index 84%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
index bc1646dac9..5cadab5c92 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
@@ -21,28 +21,27 @@ import com.webank.wedatasphere.linkis.common.log.LogUtils
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.VarSubstitutionException
-import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
import org.apache.commons.lang.exception.ExceptionUtils
/**
- * created by enjoyyin on 2018/10/19
* Description: For variable substitution(用于变量替换)
*/
-class VarSubstitutionInterceptor extends EntranceInterceptor{
+class VarSubstitutionInterceptor extends EntranceInterceptor {
@throws[ErrorException]
override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = {
task match {
- case requestPersistTask:RequestPersistTask =>
+ case requestPersistTask: RequestPersistTask =>
Utils.tryThrow {
logAppender.append(LogUtils.generateInfo("Program is substituting variables for you") + "\n")
val (result, code) = CustomVariableUtils.replaceCustomVar(requestPersistTask, requestPersistTask.getEngineType)
if (result) requestPersistTask.setExecutionCode(code)
- logAppender.append(LogUtils.generateInfo( "Variables substitution ended successfully") + "\n")
+ logAppender.append(LogUtils.generateInfo("Variables substitution ended successfully") + "\n")
requestPersistTask
- }{
- case e:VarSubstitutionException =>
+ } {
+ case e: VarSubstitutionException =>
val exception = VarSubstitutionException(20050, "Variable replacement failed!(变量替换失败！)" + ExceptionUtils.getRootCauseMessage(e))
exception.initCause(e)
exception
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/Cache.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/Cache.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/Cache.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/Cache.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogManager.scala
new file mode 100644
index 0000000000..6dca503a3b
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogManager.scala
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.webank.wedatasphere.linkis.entrance.log
+
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.entrance.exception.{CacheNotReadyException, EntranceErrorCode}
+import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+/**
+ * LogManager implementation, using a singleton class
+ * LogManager 的实现, 采用单例类进行
+ */
+class CacheLogManager extends LogManager {
+
+ override def getLogReader(execId: String): LogReader = {
+ var retLogReader:LogReader = null
+ this.entranceContext.getOrCreateScheduler().get(execId).foreach {
+ case entranceExecutionJob: EntranceExecutionJob =>
+ retLogReader = entranceExecutionJob.getLogReader.getOrElse({
+ this.synchronized {
+ val logWriter: CacheLogWriter =
+ entranceExecutionJob.getLogWriter.getOrElse(createLogWriter(entranceExecutionJob)).asInstanceOf[CacheLogWriter]
+ val sharedCache: Cache = logWriter.getCache.
+ getOrElse(throw CacheNotReadyException(EntranceErrorCode.CACHE_NOT_READY.getErrCode, EntranceErrorCode.CACHE_NOT_READY.getDesc))
+ val logPath: String = entranceExecutionJob.getTask match {
+ case requestPersistTask: RequestPersistTask => requestPersistTask.getLogPath
+ case _ => null
+ }
+ new CacheLogReader(logPath, EntranceConfiguration.DEFAULT_LOG_CHARSET.getValue, sharedCache, entranceExecutionJob.getUser)
+ }
+ })
+ entranceExecutionJob.setLogReader(retLogReader)
+ case _ => null
+ }
+ retLogReader
+ }
+
+
+ override def createLogWriter(job: Job): LogWriter = {
+
+ job match {
+ case entranceExecutionJob: EntranceExecutionJob => {
+ val cache: Cache = Cache(EntranceConfiguration.DEFAULT_CACHE_MAX.getValue)
+ val logPath: String = entranceExecutionJob.getTask.asInstanceOf[RequestPersistTask].getLogPath
+ val cacheLogWriter: CacheLogWriter =
+ new CacheLogWriter(logPath, EntranceConfiguration.DEFAULT_LOG_CHARSET.getValue, cache, entranceExecutionJob.getUser)
+ entranceExecutionJob.setLogWriter(cacheLogWriter)
+ val webSocketCacheLogReader: WebSocketCacheLogReader =
+ new WebSocketCacheLogReader(logPath, EntranceConfiguration.DEFAULT_LOG_CHARSET.getValue, cache, entranceExecutionJob.getUser)
+ entranceExecutionJob.setWebSocketLogReader(webSocketCacheLogReader)
+ val webSocketLogWriter: WebSocketLogWriter = new WebSocketLogWriter(entranceExecutionJob, entranceContext.getOrCreateEventListenerBus)
+ entranceExecutionJob.setWebSocketLogWriter(webSocketLogWriter)
+ cacheLogWriter
+ }
+ case _ => null
+ }
+ }
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogReader.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogReader.scala
similarity index 95%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogReader.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogReader.scala
index 832520c7e6..9412f643bd 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogReader.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogReader.scala
@@ -14,12 +14,7 @@
* limitations under the License.
*/
-/**
- * author: enjoyyin
- * date: 2018/9/5
- * time: 11:14
- * Description:
- */
+
package com.webank.wedatasphere.linkis.entrance.log
import java.io.{IOException, InputStream}
@@ -46,7 +41,7 @@ class CacheLogReader(logPath:String ,
private def createInputStream: InputStream = {
if (fileSystem == null) this synchronized {
if (fileSystem == null){
- fileSystem = FSFactory.getFs(new FsPath(logPath))
+ fileSystem = FSFactory.getFsByProxyUser(new FsPath(logPath), user)
fileSystem.init(new util.HashMap[String, String]())
}
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCode.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCode.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCode.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCode.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeListener.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeListener.scala
new file mode 100644
index 0000000000..2e67ae94b6
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeListener.scala
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.log
+
+import com.webank.wedatasphere.linkis.entrance.EntranceParser
+import com.webank.wedatasphere.linkis.entrance.persistence.PersistenceManager
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.task.Task
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+import org.apache.commons.lang.StringUtils
+import org.slf4j.{Logger, LoggerFactory}
+
+
+trait ErrorCodeListener {
+
+ def onErrorCodeCreated(job: Job, errorCode: String, detailErrorMsg: String)
+}
+
+class PersistenceErrorCodeListener extends ErrorCodeListener{
+
+ private var persistenceManager: PersistenceManager = _
+ private var entranceParser: EntranceParser = _
+
+ private val logger:Logger = LoggerFactory.getLogger(classOf[PersistenceErrorCodeListener])
+
+ def setPersistenceManager(persistenceManager: PersistenceManager): Unit = this.persistenceManager = persistenceManager
+ def getPersistenceManager = persistenceManager
+ def setEntranceParser(entranceParser: EntranceParser): Unit = this.entranceParser = entranceParser
+ def getEntranceParser = entranceParser
+
+ /**
+ * onErrorCodeCreated: When a job is running, it terminates unexpectedly or generates an error, and the error code and error information need to be persisted to the database.
+ * The subsequent front end will get this error code when querying a specific task as a key error message.
+ * onErrorCodeCreated: 当一个job在运行的时候，意外终止或者产生错误，需要将错误码和错误信息持久化到数据库中
+ * 后续前端在查询一个具体任务的时候，会得到这个错误码，作为关键错误信息
+ * @param job Wrong job(出错的job)
+ * @param errorCode error code(错误码)
+ * @param detailErrorMsg wrong description（错误描述）
+ */
+ override def onErrorCodeCreated(job: Job, errorCode: String, detailErrorMsg: String): Unit = {
+ val task:Task = this.entranceParser.parseToTask(job)
+ task match{
+ case requestPersistTask:RequestPersistTask => if (StringUtils.isEmpty(requestPersistTask.getErrDesc) || "50032".equals(errorCode)){
+ requestPersistTask.setErrCode(Integer.parseInt(errorCode))
+ val realErrorMsg = if (detailErrorMsg.length <= 255) detailErrorMsg else detailErrorMsg.substring(0, 255)
+ requestPersistTask.setErrDesc(realErrorMsg)
+ persistenceManager.createPersistenceEngine().updateIfNeeded(task)
+ }
+ case _ => logger.warn("task {} is not a instance of RequestPersistTask", task.getExecId)
+ }
+ }
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala
new file mode 100644
index 0000000000..7900d0f048
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala
@@ -0,0 +1,234 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.log
+
+import java.io.{BufferedReader, InputStreamReader}
+import java.util
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.storage.FSFactory
+import javax.annotation.PostConstruct
+
+import scala.collection.mutable.ArrayBuffer
+
+
+abstract class ErrorCodeManager {
+
+ def getErrorCodes: Array[ErrorCode]
+
+ def errorMatch(log: String): Option[(String, String)] = {
+ getErrorCodes.foreach(e => if(e.regex.findFirstIn(log).isDefined) {
+ val matched = e.regex.unapplySeq(log)
+ if(matched.nonEmpty)
+ return Some(e.code -> e.message.format(matched.get:_*))
+ else Some(e.code -> e.message)
+ })
+ None
+ }
+}
+
+abstract class FileErrorCodeManager extends ErrorCodeManager with Logging {
+ override def getErrorCodes: Array[ErrorCode] = (new ArrayBuffer[ErrorCode]() ++= getCommonErrorCodes ++= getErrorCodesDynamic).toArray
+
+ val errorCodeFileDir: String = {
+ val specifiedDir = EntranceConfiguration.ERROR_CODE_FILE_DIR.getValue
+ if (specifiedDir.endsWith("/")) specifiedDir else specifiedDir + "/"
+ }
+
+ val errorCodeFile: String
+ private val user = EntranceConfiguration.ENTRANCE_USER.getValue
+ private val fileSystem = FSFactory.getFs(new FsPath(errorCodeFileDir))
+
+ @PostConstruct
+ def init(): Unit = {
+ Utils.tryAndWarn(fileSystem.init(new util.HashMap[String, String]()))
+ }
+
+ val dynamicErrorCodes: ArrayBuffer[ErrorCode] = new ArrayBuffer[ErrorCode]()
+
+ Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
+ override def run(): Unit = {
+ logger.info("start to get error code properties from {}", errorCodeFile)
+ dynamicErrorCodes.clear()
+ val bufferedReader = new BufferedReader(new InputStreamReader(fileSystem.read(new FsPath(errorCodeFile)), "utf-8"))
+ var line: String = null
+ while ( {
+ line = bufferedReader.readLine(); line != null
+ }) {
+ val arr = line.split(",")
+ if (arr.length < 3){
+ logger.warn("errorcode line {} format is not correct", line)
+ }else{
+ dynamicErrorCodes += ErrorCode(arr(0).r.unanchored, arr(1), arr(2))
+ }
+ }
+ }
+ }, 0, 1,TimeUnit.HOURS)
+
+
+ def getErrorCodesDynamic:Array[ErrorCode] = dynamicErrorCodes.toArray
+
+ def getCommonErrorCodes:Array[ErrorCode] = {
+ Array(ErrorCode("queue (\\S+) is not exists in YARN".r.unanchored, "10001", "会话创建失败，%s队列不存在，请检查队列设置是否正确"),
+ ErrorCode("User (\\S+) cannot submit applications to queue (\\S+)".r.unanchored, "10001", "会话创建失败，用户%s不能提交应用到队列：%s，请检查队列设置是否正确"),
+ ErrorCode("您本次向任务队列（[a-zA-Z_0-9\\.]+）请求资源（(.+)），任务队列最大可用资源（.+），任务队列剩余可用资源（(.+)）您已占用任务队列资源（.+）".r.unanchored, "20001", "Session创建失败，当前申请资源%s，队列可用资源%s,请检查资源配置是否合理"),
+ ErrorCode("远程服务器没有足够资源实例化[a-zA-Z]+ Session，通常是由于您设置【驱动内存】或【客户端内存】过高导致的，建议kill脚本，调低参数后重新提交！等待下次调度...".r.unanchored, "20002", "Session创建失败，服务器资源不足，请稍后再试"),
+ ErrorCode("request resources from ResourceManager has reached 560++ tries, give up and mark it as FAILED.".r.unanchored, "20003", "Session创建失败，队列资源不足，请稍后再试"),
+ ErrorCode("Caused by:\\s*java.io.FileNotFoundException".r.unanchored, "20003", "文件%s不存在"),
+ ErrorCode("OutOfMemoryError".r.unanchored, "20083", "Java进程内存溢出"),
+ ErrorCode("Permission denied:\\s*user=[a-zA-Z0-9_]+,\\s*access=[A-Z]+\\s*,\\s*inode=\"([a-zA-Z0-9/_\\.]+)\"".r.unanchored, "30001", "%s无权限访问，请申请开通数据表权限"),
+ ErrorCode("Database '([a-zA-Z_0-9]+)' not found".r.unanchored, "40001", "数据库%s不存在，请检查引用的数据库是否有误"),
+ ErrorCode("Database does not exist: ([a-zA-Z_0-9]+)".r.unanchored, "40001", "数据库%s不存在，请检查引用的数据库是否有误"),
+ ErrorCode("Table or view not found: ([`\\.a-zA-Z_0-9]+)".r.unanchored, "40002", "表%s不存在，请检查引用的表是否有误"),
+ ErrorCode("Table not found '([a-zA-Z_0-9]+)'".r.unanchored, "40002", "表%s不存在，请检查引用的表是否有误"),
+ ErrorCode("cannot resolve '`(.+)`' given input columns".r.unanchored, "40003", "字段%s不存在，请检查引用的字段是否有误"),
+ ErrorCode(" Invalid table alias or column reference '(.+)':".r.unanchored, "40003", "字段%s不存在，请检查引用的字段是否有误"),
+ ErrorCode("([a-zA-Z_0-9]+) is not a valid partition column in table ([`\\.a-zA-Z_0-9]+)".r.unanchored, "40004", "分区字段%s不存在，请检查引用的表%s是否为分区表或分区字段有误"),
+ ErrorCode("Partition spec \\{(\\S+)\\} contains non-partition columns".r.unanchored, "40004", "分区字段%s不存在，请检查引用的表是否为分区表或分区字段有误"),
+ ErrorCode("table is not partitioned but partition spec exists:\\{(.+)\\}".r.unanchored, "40004", "分区字段%s不存在，请检查引用的表是否为分区表或分区字段有误"),
+ ErrorCode("extraneous input '\\)'".r.unanchored, "50001", "括号不匹配，请检查代码中括号是否前后匹配"),
+ ErrorCode("missing EOF at '\\)'".r.unanchored, "50001", "括号不匹配，请检查代码中括号是否前后匹配"),
+ ErrorCode("expression '(\\S+)' is neither present in the group by".r.unanchored, "50002", "非聚合函数%s必须写在group by中，请检查代码的group by语法"),
+ ErrorCode("grouping expressions sequence is empty,\\s?and '(\\S+)' is not an aggregate function".r.unanchored, "50002", "非聚合函数%s必须写在group by中，请检查代码的group by语法"),
+ ErrorCode("Expression not in GROUP BY key '(\\S+)'".r.unanchored, "50002", "非聚合函数%s必须写在group by中，请检查代码的group by语法"),
+ ErrorCode("Undefined function: '(\\S+)'".r.unanchored, "50003", "未知函数%s，请检查代码中引用的函数是否有误"),
+ ErrorCode("Invalid function '(\\S+)'".r.unanchored, "50003", "未知函数%s，请检查代码中引用的函数是否有误"),
+ ErrorCode("Reference '(\\S+)' is ambiguous".r.unanchored, "50004", "字段%s存在名字冲突，请检查子查询内是否有同名字段"),
+ ErrorCode("Ambiguous column Reference '(\\S+)' in subquery".r.unanchored, "50004", "字段%s存在名字冲突，请检查子查询内是否有同名字段"),
+ ErrorCode("Column '(\\S+)' Found in more than One Tables/Subqueries".r.unanchored, "50005", "字段%s必须指定表或者子查询别名，请检查该字段来源"),
+ ErrorCode("Table or view '(\\S+)' already exists in database '(\\S+)'".r.unanchored, "50006", "表%s在数据库%s中已经存在，请删除相应表后重试"),
+ ErrorCode("Table (\\S+) already exists".r.unanchored, "50006", "表%s在数据库中已经存在，请删除相应表后重试"),
+ ErrorCode("Table already exists".r.unanchored, "50006", "表%s在数据库中已经存在，请删除相应表后重试"),
+ ErrorCode("""AnalysisException: (\S+) already exists""".r.unanchored, "50006", "表%s在数据库中已经存在，请删除相应表后重试"),
+ ErrorCode("java.io.FileNotFoundException: (\\S+) \\(No such file or directory\\)".r.unanchored,"64001","找不到导入文件地址：%s"),
+ ErrorCode("java.io.IOException: Permission denied(.+)at org.apache.poi.xssf.streaming.SXSSFWorkbook.createAndRegisterSXSSFSheet".r.unanchored,"64002","导出为excel时临时文件目录权限异常"),
+ ErrorCode("java.io.IOException: Mkdirs failed to create (\\S+) (.+)".r.unanchored,"64003","导出文件时无法创建目录：%s"),
+ ErrorCode("""ImportError: No module named (\S+)""".r.unanchored, "50008", "导入模块错误，系统没有%s模块，请联系运维人员安装"),
+ ErrorCode("""requires that the data to be inserted have the same number of columns as the target table""".r.unanchored, "50009",
+ "插入目标表字段数量不匹配,请检查代码！"),
+ ErrorCode("""missing \) at '(\S+)' near ''""".r.unanchored, "50010", "%s处括号不匹配，请检查代码！"),
+ ErrorCode("""due to data type mismatch: differing types in""".r.unanchored,
+ "50011", "数据类型不匹配，请检查代码！"),
+ ErrorCode("""Invalid column reference (\S+)""".r.unanchored, "50012", "字段%s引用有误，请检查字段是否存在！"),
+ ErrorCode("""Can't extract value from (\S+): need struct type but got string""".r.unanchored, "50013", "字段%s提取数据失败"),
+ ErrorCode("""mismatched input '(\S+)' expecting""".r.unanchored, "50014", "括号或者关键字不匹配，请检查代码！"),
+ ErrorCode("""GROUP BY position (\S+) is not in select list""".r.unanchored, "50015", "group by 位置2不在select列表中，请检查代码！"),
+ ErrorCode("""'NoneType' object""".r.unanchored, "50016", "代码中存在NoneType空类型变量，请检查代码"),
+ ErrorCode("""IndexError:List index out of range""".r.unanchored, "50017", "数组越界"),
+ ErrorCode("""Can't extract value from (\S+): need struct type but got string""".r.unanchored, "50013", "字段提取数据失败请检查字段类型"),
+ ErrorCode("""Cannot insert into target table because column number/types are different '(\S+)'""".r.unanchored, "50018", "插入数据未指定目标表字段%s，请检查代码！"),
+ ErrorCode("""Invalid table alias '(\S+)'""".r.unanchored, "50019", "表别名%s错误，请检查代码！"),
+ ErrorCode("""UDFArgumentException Argument expected""".r.unanchored, "50020", "UDF函数未指定参数，请检查代码！"),
+ ErrorCode("""aggregate functions are not allowed in GROUP BY""".r.unanchored, "50021", "聚合函数%s不能写在group by 中，请检查代码！"),
+ ErrorCode("SyntaxError".r.unanchored, "50007", "您的代码有语法错误，请您修改代码之后执行"),
+ ErrorCode("""Table not found""".r.unanchored, "40002", "表不存在，请检查引用的表是否有误"),
+ ErrorCode("""No matching method""".r.unanchored, "40003", "函数使用错误，请检查您使用的函数方式"),
+ ErrorCode("""is killed by user""".r.unanchored, "50032", "用户主动kill任务"),
+ ErrorCode("""name '(\S+)' is not defined""".r.unanchored, "60001", "python代码变量%s未定义"),
+ ErrorCode("""Undefined function:\s+'(\S+)'""".r.unanchored, "60002", "python udf %s 未定义"),
+ ErrorCode("FAILED: ParseException".r.unanchored, "50007", "您的sql代码可能有语法错误，请检查sql代码"),
+ ErrorCode("org.apache.spark.sql.catalyst.parser.ParseException".r.unanchored,"50007", "您的sql代码可能有语法错误，请检查sql代码"),
+ ErrorCode("""ParseException:""".r.unanchored, "60003", "脚本语法有误"),
+ ErrorCode("""Permission denied""".r.unanchored, "60010", "您可能没有相关权限"),
+ ErrorCode("""cannot concatenate '(\S+)' and '(\S+)'""".r.unanchored, "61027", "python执行不能将%s和%s两种类型进行连接"),
+ ErrorCode("""Py4JJavaError: An error occurred""".r.unanchored, "60020", "pyspark执行失败，可能是语法错误或stage失败"),
+ ErrorCode("""unexpected indent""".r.unanchored, "61028", "python代码缩进对齐有误"),
+ ErrorCode("""is exceeded""".r.unanchored, "60078", "个人库超过限制"),
+ ErrorCode("""unexpected indent""".r.unanchored, "69582", "python代码缩进有误"),
+ ErrorCode("""unexpected character after line""".r.unanchored, "69583", "python代码反斜杠后面必须换行"),
+ ErrorCode("""Invalid row number""".r.unanchored, "60091", "导出Excel表超过最大限制1048575"),
+ ErrorCode("""parquet.io.ParquetDecodingException""".r.unanchored, "60092", "python save as table未指定格式，默认用parquet保存，hive查询报错"),
+ ErrorCode("""errCode: 11011""".r.unanchored, "11011", "远程服务器内存资源不足"),
+ ErrorCode("""errCode: 11012""".r.unanchored, "11012", "远程服务器CPU资源不足"),
+ ErrorCode("""errCode: 11013""".r.unanchored, "11013", "远程服务器实例资源不足"),
+ ErrorCode("""errCode: 11014""".r.unanchored, "11014", "队列CPU资源不足"),
+ ErrorCode("""errCode: 11015""".r.unanchored, "11015", "队列内存资源不足"),
+ ErrorCode("""errCode: 11016""".r.unanchored, "11016", "队列实例数超过限制"),
+ ErrorCode("""errCode: 11017""".r.unanchored, "11017", "超出全局计算引擎实例限制"),
+ ErrorCode("""资源不足""".r.unanchored, "60035", "资源不足，启动引擎失败"),
+ ErrorCode("""获取Yarn队列信息异常""".r.unanchored, "60075", "获取Yarn队列信息异常,可能是您设置的yarn队列不存在")
+ )
+ }
+}
+
+/**
+ * errorCodeManager的单例对象,主要是用来生成固定的错误码
+ */
+object FixedErrorCodeManager extends FileErrorCodeManager {
+
+ override val errorCodeFile: String = ""
+
+ override def getErrorCodes: Array[ErrorCode] = getCommonErrorCodes
+}
+
+
+object Main{
+ def main(args: Array[String]): Unit = {
+ }
+}
+
+
+/**
+ * RefreshableErrorCodeManager corresponds to FixedErrorCodeManager, and refresheyeErrorCodeManager can update its own errorCodes through the query module.
+ * The purpose is to enable users to update the error code at any time by modifying the database.
+ * RefreshableErrorCodeManager 与 FixedErrorCodeManager 是对应的，refreshaleErrorCodeManager可以通过query模块进行更新自身的errorCodes
+ * 目的是为了能够让用户通过修改数据库随时更新错误码
+ */
+//
+//object RefreshableErrorCodeManager extends ErrorCodeManager{
+// private val sender:Sender =
+// Sender.getSender(EntranceConfiguration.QUERY_PERSISTENCE_SPRING_APPLICATION_NAME.getValue, 1000 * 60 * 60, 100)
+//
+// private val logger:Logger = LoggerFactory.getLogger(getClass)
+// private var errorCodes:Array[ErrorCode] = _
+//
+// class FetchErrorCodeThread extends Runnable{
+// override def run(): Unit = {
+// val requestErrorCode = new RequestErrorCode
+// val responseErrorCode = sender.send(requestErrorCode).asInstanceOf[ResponseErrorCode]
+// Utils.tryAndWarnMsg{
+// val responseErrorCode = sender.send(requestErrorCode).asInstanceOf[ResponseErrorCode]
+// val status = responseErrorCode.getStatus
+// val message = responseErrorCode.getMessage
+// if (status != 0){
+// logger.warn(s"Error encounters when retrieve errorCodes from query module, reason: $message")
+// }else{
+// val errorCodeList = responseErrorCode.getResult
+// val arrayBuffer = new ArrayBuffer[ErrorCode]()
+// import scala.collection.JavaConversions._
+// errorCodeList foreach { errorCode =>
+// val regex = errorCode.getErrorRegex.r.unanchored
+// val errorCode_ = errorCode.getErrorCode
+// val errorDesc = errorCode.getErrorDesc
+// arrayBuffer += ErrorCode(regex, errorCode_, errorDesc)
+// }
+// errorCodes = arrayBuffer.toArray
+// }
+// }("Query ErrorCodes failed. You may check the cause or just ignore it ")
+// }
+// }
+//
+// override def getErrorCodes: Array[ErrorCode] = errorCodes
+//}
+
+
+
+
+
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogFilter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogFilter.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogFilter.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogFilter.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogHandler.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogHandler.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogHandler.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogHandler.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogManager.scala
new file mode 100644
index 0000000000..9db056eba1
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogManager.scala
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.log
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.EntranceContext
+import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob
+import com.webank.wedatasphere.linkis.scheduler.listener.LogListener
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+
+
+abstract class LogManager extends LogListener with Logging{
+
+ protected var errorCodeListener: Option[ErrorCodeListener] = None
+ protected var errorCodeManager: Option[ErrorCodeManager] = None
+ protected var entranceContext: EntranceContext = _
+
+ def setEntranceContext(entranceContext: EntranceContext): Unit = this.entranceContext = entranceContext
+ def setErrorCodeListener(errorCodeListener: ErrorCodeListener): Unit = this.errorCodeListener = Option(errorCodeListener)
+ def setErrorCodeManager(errorCodeManager: ErrorCodeManager): Unit = this.errorCodeManager = Option(errorCodeManager)
+
+ def getLogReader(execId: String): LogReader
+
+ def createLogWriter(job: Job): LogWriter
+
+ override def onLogUpdate(job: Job, log: String): Unit = {
+ Utils.tryCatch{
+ job match{
+ case entranceExecutionJob: EntranceExecutionJob =>
+ if (entranceExecutionJob.getLogWriter.isEmpty) entranceExecutionJob synchronized {
+ if (entranceExecutionJob.getLogWriter.isEmpty) createLogWriter(entranceExecutionJob)
+ }
+ entranceExecutionJob.getLogWriter.foreach(logWriter => logWriter.write(log))
+ entranceExecutionJob.getWebSocketLogWriter.foreach(writer => writer.write(log))
+ errorCodeManager.foreach(_.errorMatch(log).foreach { case (code, errorMsg) =>
+ errorCodeListener.foreach(_.onErrorCodeCreated(job, code, errorMsg))
+ })
+ case _ =>
+ }
+ }{
+ case e: Exception => logger.warn(s"write log for job ${job.getId} failed", e)
+ case t: Throwable => logger.warn(s"write log for job ${job.getId} failed", t)
+ }
+ }
+}
\ No newline at end of file
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogReader.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogReader.scala
similarity index 82%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogReader.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogReader.scala
index 6a85906d1c..2afa4733d7 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogReader.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogReader.scala
@@ -26,9 +26,7 @@ import org.apache.commons.lang.StringUtils
import scala.util.matching.Regex
-/**
- * Created by enjoyyin on 2018/9/4.
- */
+
abstract class LogReader(charset: String) extends Closeable with Logging{
import LogReader._
@@ -70,6 +68,18 @@ abstract class LogReader(charset: String) extends Closeable with Logging{
if (!flag) concatLog(length, singleLog, warning, all)
case INFO_HEADER1() | INFO_HEADER2() =>
val hiveLogSpecial:String = EntranceConfiguration.HIVE_SPECIAL_LOG_INCLUDE.getValue
+ val sparkLogSpecial:String = EntranceConfiguration.SPARK_SPECIAL_LOG_INCLUDE.getValue
+ val hiveCreateTableLog:String = EntranceConfiguration.HIVE_CREATE_TABLE_LOG.getValue
+ if (singleLog.contains(hiveLogSpecial) && singleLog.contains(hiveCreateTableLog)){
+ val threadName = EntranceConfiguration.HIVE_THREAD_NAME.getValue
+ val printInfo = EntranceConfiguration.HIVE_PRINT_INFO_LOG.getValue
+ val start = singleLog.indexOf(threadName)
+ val end = singleLog.indexOf(printInfo) + printInfo.length
+ if(start > 0 && end > 0) {
+ val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length)
+ concatLog(length, realLog, info, all)
+ }
+ }
if (singleLog.contains(hiveLogSpecial) && singleLog.contains("map") && singleLog.contains("reduce")){
val start = singleLog.indexOf(EntranceConfiguration.HIVE_THREAD_NAME.getValue)
val end = singleLog.indexOf(EntranceConfiguration.HIVE_STAGE_NAME.getValue)
@@ -77,6 +87,15 @@ abstract class LogReader(charset: String) extends Closeable with Logging{
val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length)
concatLog(length, realLog, info, all)
}
+ }else if (singleLog.contains(sparkLogSpecial)){
+ val className = EntranceConfiguration.SPARK_PROGRESS_NAME.getValue
+ val endFlag = EntranceConfiguration.END_FLAG.getValue
+ val start = singleLog.indexOf(className)
+ val end = singleLog.indexOf(endFlag) + endFlag.length
+ if(start > 0 && end > 0) {
+ val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length)
+ concatLog(length, realLog, info, all)
+ }
}else{
val arr = EntranceConfiguration.LOG_EXCLUDE.getValue.split(",").map (word => word.trim)
var flag = false
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogWriter.scala
similarity index 78%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogWriter.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogWriter.scala
index 0d9e0cfece..6a56bdb858 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogWriter.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LogWriter.scala
@@ -24,12 +24,10 @@ import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.utils.FileSystemUtils
-import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream
+
-/**
- * Created by enjoyyin on 2018/9/4.
- */
abstract class LogWriter(charset: String) extends Closeable with Flushable with Logging {
private var firstWrite = true
@@ -52,15 +50,25 @@ abstract class LogWriter(charset: String) extends Closeable with Flushable with
- def flush(): Unit = Utils.tryQuietly(outputStream.flush(), t => {
- warn("Error encounters when flush log,", t)
- })
-
+ def flush(): Unit = Utils.tryAndWarnMsg[Unit] {
+ outputStream match {
+ case hdfs: HdfsDataOutputStream =>
+ // todo check
+ hdfs.hsync()
+ case _ =>
+ outputStream.flush()
+ }
+ }("Error encounters when flush log, ")
def close(): Unit = {
flush()
- if (outputStream != null) IOUtils.closeQuietly(outputStream)
-
+ if (outputStream != null) {
+ Utils.tryCatch{
+ outputStream.close()
+ }{
+ case t:Throwable => //ignore
+ }
+ }
}
}
@@ -68,11 +76,11 @@ abstract class AbstractLogWriter(logPath: String,
user: String,
charset: String) extends LogWriter(charset) {
if(StringUtils.isBlank(logPath)) throw new EntranceErrorException(20301, "logPath cannot be empty.")
- protected val fileSystem = FSFactory.getFs(new FsPath(logPath))
+ protected val fileSystem = FSFactory.getFsByProxyUser(new FsPath(logPath), user)
fileSystem.init(new util.HashMap[String, String]())
protected val outputStream: OutputStream = {
- FileSystemUtils.createNewFile(new FsPath(logPath), true)
+ FileSystemUtils.createNewFile(new FsPath(logPath), user, true)
fileSystem.write(new FsPath(logPath), true)
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LoopArray.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LoopArray.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LoopArray.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/LoopArray.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketCacheLogReader.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketCacheLogReader.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketCacheLogReader.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketCacheLogReader.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketLogWriter.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketLogWriter.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/WebSocketLogWriter.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala
new file mode 100644
index 0000000000..025da36543
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala
@@ -0,0 +1,219 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.parser
+
+import java.util
+import java.util.Date
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.entrance.exception.{EntranceErrorCode, EntranceIllegalParamException}
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.manager.label.builder.factory.{LabelBuilderFactory, LabelBuilderFactoryContext, StdLabelBuilderFactory}
+import com.webank.wedatasphere.linkis.manager.label.constant.LabelKeyConstant
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.manager.label.entity.engine.{EngineRunTypeLabel, EngineTypeLabel, UserCreatorLabel}
+import com.webank.wedatasphere.linkis.manager.label.utils.{EngineTypeLabelCreator, LabelUtils}
+import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
+import com.webank.wedatasphere.linkis.protocol.task.Task
+import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions.mapAsScalaMap
+import scala.collection.JavaConverters._
+
+
+class CommonEntranceParser extends AbstractEntranceParser with Logging {
+
+ private val labelBuilderFactory: LabelBuilderFactory = new StdLabelBuilderFactory
+
+ /**
+ * parse params to be a task
+ * params json data from frontend
+ *
+ */
+ override def parseToTask(params: util.Map[String, Any]): Task = {
+ if (!params.containsKey(TaskConstant.EXECUTION_CONTENT)) {
+ return parseToOldTask(params)
+ }
+
+ val task = new RequestPersistTask
+ task.setCreatedTime(new Date(System.currentTimeMillis))
+ task.setInstance(Sender.getThisInstance)
+ val umUser = params.get(TaskConstant.EXECUTE_USER).asInstanceOf[String]
+ val submitUser = params.get(TaskConstant.SUBMIT_USER).asInstanceOf[String]
+ task.setSubmitUser(submitUser)
+ if (StringUtils.isBlank(umUser)) task.setUmUser(submitUser)
+ task.setUmUser(umUser)
+ val executionContent = params.getOrDefault(TaskConstant.EXECUTION_CONTENT, null).asInstanceOf[util.Map[String, Object]]
+ val configMap = params.getOrDefault(TaskConstant.PARAMS, null).asInstanceOf[util.Map[String, Object]]
+ val labelMap = params.getOrDefault(TaskConstant.LABELS, null).asInstanceOf[util.Map[String, Object]]
+ val source = params.getOrDefault(TaskConstant.SOURCE, new util.HashMap[String, String]()).asInstanceOf[util.Map[String, String]]
+ val sourcePath = source.getOrDefault(TaskConstant.SCRIPTPATH, null)
+ if (null == sourcePath || null == labelMap) {
+ throw new EntranceIllegalParamException(EntranceErrorCode.PARAM_CANNOT_EMPTY.getErrCode, EntranceErrorCode.PARAM_CANNOT_EMPTY.getDesc + s", source : ${sourcePath}, labels : ${BDPJettyServerHelper.gson.toJson(labelMap)}")
+ }
+
+ var code: String = null
+ var runType: String = null
+ if (executionContent.containsKey(TaskConstant.CODE)) {
+ code = executionContent.get(TaskConstant.CODE).asInstanceOf[String]
+ runType = executionContent.get(TaskConstant.RUNTYPE).asInstanceOf[String]
+ if (StringUtils.isEmpty(code))
+ throw new EntranceIllegalParamException(20007, "param executionCode and scriptPath can not be empty at the same time")
+ } else {
+ // todo check
+ throw new EntranceIllegalParamException(20010, "Only code with runtype supported !")
+ }
+ val formatCode = params.get(TaskConstant.FORMATCODE).asInstanceOf[Boolean]
+ if (formatCode) code = format(code)
+ task.setExecutionCode(code)
+ var labels: util.Map[String, Label[_]] = buildLabel(labelMap)
+ val engineTypeLabel = labels.getOrElse(LabelKeyConstant.ENGINE_TYPE_KEY, null).asInstanceOf[EngineTypeLabel]
+ if (null == engineTypeLabel) {
+ val msg = s"Cannot create engineTypeLabel, labelMap : ${BDPJettyServerHelper.gson.toJson(labelMap)}"
+ error(msg)
+ throw new EntranceIllegalParamException(EntranceErrorCode.LABEL_PARAMS_INVALID.getErrCode,
+ EntranceErrorCode.LABEL_PARAMS_INVALID.getDesc + msg)
+ } else {
+ task.setEngineType(engineTypeLabel.getEngineType)
+ }
+ val engineRunTypeLabel = labels.getOrElse(LabelKeyConstant.ENGINE_RUN_TYPE_KEY, null).asInstanceOf[EngineRunTypeLabel]
+ if (null != engineRunTypeLabel) {
+ task.setRunType(engineRunTypeLabel.getRunType)
+ } else {
+ warn(s"RunType not set. EngineType : ${engineTypeLabel.getEngineType}")
+ }
+ var userCreatorLabel = labels.getOrElse(LabelKeyConstant.USER_CREATOR_TYPE_KEY, null).asInstanceOf[UserCreatorLabel]
+
+ if (null == userCreatorLabel) {
+ userCreatorLabel = labelBuilderFactory.createLabel(classOf[UserCreatorLabel])
+ val creator = EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue
+ userCreatorLabel.setUser(umUser)
+ userCreatorLabel.setCreator(creator)
+ labels.put(userCreatorLabel.getLabelKey, userCreatorLabel)
+ }
+ labels += (LabelKeyConstant.ENGINE_TYPE_KEY -> engineTypeLabel)
+ labels += (LabelKeyConstant.ENGINE_RUN_TYPE_KEY -> engineRunTypeLabel)
+ labels += (LabelKeyConstant.USER_CREATOR_TYPE_KEY -> userCreatorLabel)
+ task.setLabels(new util.ArrayList[Label[_]](labels.values()))
+ task.setSource(source)
+ task.setExecuteApplicationName(engineTypeLabel.getEngineType)
+ task.setRequestApplicationName(userCreatorLabel.getCreator)
+ task.setStatus(SchedulerEventState.Inited.toString)
+ task.setCreateService(EntranceConfiguration.DEFAULT_CREATE_SERVICE.getValue)
+ task.setParams(configMap)
+ task
+ }
+
+ private def parseToOldTask(params: util.Map[String, Any]): Task = {
+
+ val task = new RequestPersistTask
+ task.setCreatedTime(new Date(System.currentTimeMillis))
+ task.setInstance(Sender.getThisInstance)
+ val umUser = params.get(TaskConstant.UMUSER).asInstanceOf[String]
+ val submitUser = params.get(TaskConstant.SUBMIT_USER).asInstanceOf[String]
+ task.setSubmitUser(submitUser)
+ if (umUser == null) throw new EntranceIllegalParamException(20005, "umUser can not be null")
+ task.setUmUser(umUser)
+ var executionCode = params.get(TaskConstant.EXECUTIONCODE).asInstanceOf[String]
+ val _params = params.get(TaskConstant.PARAMS)
+ _params match {
+ case mapParams: java.util.Map[String, Object] => task.setParams(mapParams)
+ case _ =>
+ }
+ val formatCode = params.get(TaskConstant.FORMATCODE).asInstanceOf[Boolean]
+ var creator = params.get(TaskConstant.REQUESTAPPLICATIONNAME).asInstanceOf[String]
+ val source = params.getOrDefault(TaskConstant.SOURCE, new util.HashMap[String, String]()).asInstanceOf[util.Map[String, String]]
+ val executeApplicationName = params.get(TaskConstant.EXECUTEAPPLICATIONNAME).asInstanceOf[String]
+ if (StringUtils.isEmpty(creator)) creator = EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue
+ //if (StringUtils.isEmpty(executeApplicationName)) throw new EntranceIllegalParamException(20006, "param executeApplicationName can not be empty or null")
+ /* When the execution type is IDE, executionCode and scriptPath cannot be empty at the same time*/
+ /*当执行类型为IDE的时候，executionCode和scriptPath不能同时为空*/
+ if (EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue.equals(creator) && StringUtils.isEmpty(source.get(TaskConstant.SCRIPTPATH)) &&
+ StringUtils.isEmpty(executionCode))
+ throw new EntranceIllegalParamException(20007, "param executionCode and scriptPath can not be empty at the same time")
+ var runType: String = null
+ if (StringUtils.isNotEmpty(executionCode)) {
+ runType = params.get(TaskConstant.RUNTYPE).asInstanceOf[String]
+ if (StringUtils.isEmpty(runType)) runType = EntranceConfiguration.DEFAULT_RUN_TYPE.getValue
+ //If formatCode is not empty, we need to format it(如果formatCode 不为空的话，我们需要将其进行格式化)
+ if (formatCode) executionCode = format(executionCode)
+ task.setExecutionCode(executionCode)
+ }
+ task.setSource(source)
+ task.setEngineType(runType)
+ //为了兼容代码，让engineType和runType都有同一个属性
+ task.setRunType(runType)
+ task.setExecuteApplicationName(executeApplicationName)
+ task.setRequestApplicationName(creator)
+ task.setStatus(SchedulerEventState.Inited.toString)
+ task.setCreateService(EntranceConfiguration.DEFAULT_CREATE_SERVICE.getValue)
+ //封装Labels
+ task.setLabels(buildLabel(task))
+ task
+ }
+
+ private def buildLabel(task: RequestPersistTask): util.List[Label[_]] = {
+ val labelsList = new util.ArrayList[Label[_]]()
+ val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory
+ if (StringUtils.isNotBlank(task.getUmUser) && StringUtils.isNotBlank(task.getRequestApplicationName)) {
+ val userCreatorLabel = labelBuilderFactory.createLabel(classOf[UserCreatorLabel])
+ userCreatorLabel.setCreator(task.getRequestApplicationName)
+ userCreatorLabel.setUser(task.getUmUser)
+ labelsList.add(userCreatorLabel)
+ }
+
+ if (StringUtils.isNotBlank(task.getExecuteApplicationName)) {
+ val engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(task.getExecuteApplicationName)
+ labelsList.add(engineTypeLabel)
+ }
+
+ if (StringUtils.isNotBlank(task.getRunType)) {
+ val runTypeLabel = labelBuilderFactory.createLabel(classOf[EngineRunTypeLabel])
+ runTypeLabel.setRunType(task.getRunType)
+ labelsList.add(runTypeLabel)
+ }
+
+ val labels = TaskUtils.getLabelsMap(task.getParams.asInstanceOf[util.Map[String, Any]])
+ if (null != labels) {
+ LabelUtils.distinctLabel(labelBuilderFactory.getLabels(labels.asInstanceOf[util.Map[String, AnyRef]]), labelsList)
+ } else {
+ labelsList
+ }
+ }
+
+ private def buildLabel(labelMap: util.Map[String, Object]): util.Map[String, Label[_]] = {
+ val labelKeyValueMap = new util.HashMap[String, Label[_]]()
+ if (null != labelMap && !labelMap.isEmpty) {
+ val list: util.List[Label[_]] = labelBuilderFactory.getLabels(labelMap.asInstanceOf[util.Map[String, AnyRef]])
+ if (null != list) {
+ list.asScala.foreach {
+ label => labelKeyValueMap.put(label.getLabelKey, label)
+ }
+ }
+ }
+ labelKeyValueMap
+ }
+
+ //todo to format code using proper way
+ private def format(code: String): String = code
+
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala
new file mode 100644
index 0000000000..d64747bda6
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.persistence
+
+import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet
+import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.execute.StorePathExecuteRequest
+import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob
+import com.webank.wedatasphere.linkis.entrance.scheduler.cache.CacheOutputExecuteResponse
+import com.webank.wedatasphere.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse}
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter}
+import com.webank.wedatasphere.linkis.storage.utils.FileSystemUtils
+import org.apache.commons.io.IOUtils
+
+
+class EntranceResultSetEngine extends ResultSetEngine with Logging {
+ override def persistResultSet(job: Job, executeCompleted: OutputExecuteResponse): String = {
+
+
+ executeCompleted match {
+ case AliasOutputExecuteResponse(alias, output) =>
+ if (ResultSetFactory.getInstance.isResultSetPath(output)) output
+ else {
+ val resultSet = ResultSetFactory.getInstance.getResultSetByContent(output)
+ writeResult(alias, output, job, resultSet)
+ }
+ case CacheOutputExecuteResponse(alias, output) =>
+ if(ResultSetFactory.getInstance.isResultSet(output)){
+ val resultSet = ResultSetFactory.getInstance.getResultSetByContent(output)
+ writeResult(alias, output, job, resultSet)
+ } else {
+ val resultSet = ResultSetFactory.getInstance.getResultSetByPath(FsPath.getFsPath(output))
+ copyResult(alias, output, job, resultSet)
+ }
+ }
+ }
+
+ private def copyResult(alias: String, output: String, job: Job, resultSet:ResultSet[_ <: MetaData, _ <: Record]) : String = {
+ var user:String="hadoop"
+ val storePath = job match {
+ case j: EntranceExecutionJob => j.jobToExecuteRequest() match {
+ case s: StorePathExecuteRequest => {
+ user = j.getUser
+ s.storePath
+ }
+ case _ => null
+ }
+ case _ => null
+ }
+ if(storePath != null) {
+ val path = if(alias.contains("_")) resultSet.getResultSetPath(new FsPath(storePath), alias) else resultSet.getResultSetPath(new FsPath(storePath), "_" + alias)
+ FileSystemUtils.copyFile(path, FsPath.getFsPath(output), user)
+ path.getSchemaPath
+ } else null
+ }
+
+ private def writeResult(alias: String, output: String, job: Job, resultSet:ResultSet[_ <: MetaData, _ <: Record]) = {
+ var user:String="hadoop"
+ val storePath = job match {
+ case j: EntranceExecutionJob => j.jobToExecuteRequest() match {
+ case s: StorePathExecuteRequest => {
+ user = j.getUser
+ s.storePath
+ }
+ case _ => null
+ }
+ case _ => null
+ }
+ if(storePath != null) {
+ //TODO Remove _ stitching(去掉_拼接)
+ val path = if(alias.contains("_")) resultSet.getResultSetPath(new FsPath(storePath), alias) else resultSet.getResultSetPath(new FsPath(storePath), "_" + alias)
+
+ FileSystemUtils.createNewFile(path, user,true)
+ val writer = ResultSetWriter.getResultSetWriter(resultSet, 0, path, user)
+ Utils.tryFinally {
+ writer.addMetaDataAndRecordString(output)
+ writer.flush()
+ }{
+ IOUtils.closeQuietly(writer)
+ }
+ path.getSchemaPath
+ } else null
+ }
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/HaPersistenceTask.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/HaPersistenceTask.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/HaPersistenceTask.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/HaPersistenceTask.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/JobCompletedListener.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/JobCompletedListener.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/JobCompletedListener.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/JobCompletedListener.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceManager.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceManager.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceManager.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetEngine.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetEngine.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetEngine.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetEngine.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetListener.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetListener.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetListener.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/ResultSetListener.scala
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
similarity index 90%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
index 620af1c46a..6c1aa10e72 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
@@ -23,15 +23,16 @@ import javax.ws.rs.QueryParam
import javax.ws.rs.core.{Context, Response}
import org.springframework.web.bind.annotation.{PathVariable, RequestBody, RequestMapping, RequestMethod}
-/**
- * Created by enjoyyin on 2018/9/3.
- */
+
trait EntranceRestfulRemote {
@RequestMapping(value = Array("/entrance/execute"), method = Array(RequestMethod.POST))
def execute(@Context req: HttpServletRequest, @RequestBody json: util.Map[String, Any]): Response
-// @RequestMapping(value = Array("/api/entrance/{id}"), method = Array(RequestMethod.GET))
+ @RequestMapping(value = Array("/entrance/submit"), method = Array(RequestMethod.POST))
+ def submit(@Context req: HttpServletRequest, @RequestBody json: util.Map[String, Any]): Response
+
+ // @RequestMapping(value = Array("/api/entrance/{id}"), method = Array(RequestMethod.GET))
// def get(@PathVariable("id") id: String): Response
@RequestMapping(value = Array("/entrance/{id}/status"), method = Array(RequestMethod.GET))
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala
new file mode 100644
index 0000000000..1300086e1b
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.scheduler
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
+import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
+import com.webank.wedatasphere.linkis.entrance.persistence.HaPersistenceTask
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.governance.common.protocol.conf.{RequestQueryEngineConfig, ResponseQueryConfig}
+import com.webank.wedatasphere.linkis.manager.label.builder.factory.LabelBuilderFactoryContext
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+import com.webank.wedatasphere.linkis.manager.label.entity.engine.{ConcurrentEngineConnLabel, EngineTypeLabel, UserCreatorLabel}
+import com.webank.wedatasphere.linkis.manager.label.utils.EngineTypeLabelCreator
+import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
+import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelGroup
+import com.webank.wedatasphere.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent}
+import com.webank.wedatasphere.linkis.server.JMap
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions._
+
+
+class EntranceGroupFactory extends GroupFactory with Logging {
+
+ private val groupNameToGroups = new JMap[String, Group]
+ private val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory
+
+ override def getOrCreateGroup(groupName: String): Group = {
+ if (!groupNameToGroups.containsKey(groupName)) synchronized {
+ val initCapacity = 100
+ val maxCapacity = 100
+ var maxRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue
+ val maxAskExecutorTimes = EntranceConfiguration.MAX_ASK_EXECUTOR_TIME.getValue.toLong
+ if (groupName.startsWith(EntranceGroupFactory.CONCURRENT)) {
+ if (null == groupNameToGroups.get(groupName)) synchronized {
+ if (null == groupNameToGroups.get(groupName)) {
+ val group = new ParallelGroup(groupName, 100, EntranceConfiguration.CONCURRENT_FACTORY_MAX_CAPACITY.getValue)
+ group.setMaxRunningJobs(EntranceConfiguration.CONCURRENT_MAX_RUNNING_JOBS.getValue)
+ group.setMaxAskExecutorTimes(EntranceConfiguration.CONCURRENT_EXECUTOR_TIME.getValue)
+ groupNameToGroups.put(groupName, group)
+ return group
+ }
+ }
+ }
+ val groupNameSplits = groupName.split("_")
+ if (groupNameSplits.length < 3) {
+ logger.warn(s"name style of group: $groupName is not correct, we will set default value for the group")
+ } else {
+ val sender: Sender = Sender.getSender(EntranceConfiguration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue)
+ val creator = groupNameSplits(0)
+ val username = groupNameSplits(1)
+ val engineType = groupNameSplits(2)
+
+ logger.info(s"Getting parameters for $groupName(正在为 $groupName 获取参数) username: $username, creator:$creator, engineType: $engineType")
+ val userCreatorLabel = labelBuilderFactory.createLabel(classOf[UserCreatorLabel])
+ userCreatorLabel.setUser(username)
+ userCreatorLabel.setCreator(creator)
+ val engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(engineType)
+ try {
+ val keyAndValue = sender.ask(RequestQueryEngineConfig(userCreatorLabel, engineTypeLabel)).asInstanceOf[ResponseQueryConfig].getKeyAndValue
+
+ maxRunningJobs = Integer.parseInt(keyAndValue.get(EntranceConfiguration.WDS_LINKIS_INSTANCE.key))
+ } catch {
+ case t: Throwable => logger.warn("Get maxRunningJobs from configuration server failed! Next use the default value to continue.")
+ }
+ }
+ logger.info("groupName: {} => maxRunningJobs is {}", groupName, maxRunningJobs)
+ val group = new ParallelGroup(groupName, initCapacity, maxCapacity)
+ group.setMaxRunningJobs(maxRunningJobs)
+ group.setMaxAskExecutorTimes(maxAskExecutorTimes)
+ if (!groupNameToGroups.containsKey(groupName)) groupNameToGroups.put(groupName, group)
+ }
+ groupNameToGroups.get(groupName)
+ }
+
+
+ override def getGroupNameByEvent(event: SchedulerEvent): String = event match {
+ case job: EntranceJob =>
+ job.getTask match {
+ case HaPersistenceTask(task) =>
+ "HA"
+ case requestPersistTask: RequestPersistTask => {
+ val labels = requestPersistTask.getLabels
+ EntranceGroupFactory.getGroupNameByLabels(labels, job.getParams)
+ }
+ case _ => EntranceGroupFactory.getGroupName(job.getCreator, job.getUser, job.getParams)
+ }
+ }
+}
+
+object EntranceGroupFactory {
+
+ val CACHE = "_Cache"
+
+ val CONCURRENT = "Concurrent_"
+
+ def getGroupName(creator: String, user: String, params: util.Map[String, Any] = new util.HashMap[String, Any]): String = {
+ val runtime = TaskUtils.getRuntimeMap(params)
+ val cache = if (runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime.get(TaskConstant.READ_FROM_CACHE).asInstanceOf[Boolean]) CACHE else ""
+ if (StringUtils.isNotEmpty(creator)) creator + "_" + user + cache
+ else EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue + "_" + user + cache
+ }
+
+ def getGroupNameByLabels(labels: java.util.List[Label[_]], params: util.Map[String, Any] = new util.HashMap[String, Any]): String = {
+
+ val userCreator = labels.find(_.isInstanceOf[UserCreatorLabel])
+ val engineType = labels.find(_.isInstanceOf[EngineTypeLabel])
+ val concurrent = labels.find(_.isInstanceOf[ConcurrentEngineConnLabel])
+ if (userCreator.isEmpty || engineType.isEmpty) {
+ throw new EntranceErrorException(20001, "userCreator label or engineType label cannot null")
+ }
+
+ if (concurrent.isDefined) {
+
+ val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel]
+ val groupName = CONCURRENT + engineTypeLabel.getEngineType
+ groupName
+
+ } else {
+ val userCreatorLabel = userCreator.get.asInstanceOf[UserCreatorLabel]
+
+ val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel]
+
+ val runtime = TaskUtils.getRuntimeMap(params)
+ val cache = if (runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime.get(TaskConstant.READ_FROM_CACHE).asInstanceOf[Boolean]) CACHE else ""
+ val groupName = userCreatorLabel.getCreator + "_" + userCreatorLabel.getUser + "_" + engineTypeLabel.getEngineType + cache
+ groupName
+ }
+ }
+
+
+}
\ No newline at end of file
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala
similarity index 100%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala
new file mode 100644
index 0000000000..6b53a773b1
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.scheduler.cache
+
+import com.webank.wedatasphere.linkis.scheduler.executer.OutputExecuteResponse
+
+case class CacheOutputExecuteResponse(alias: String, output: String) extends OutputExecuteResponse{
+ override def getOutput: String = output
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala
new file mode 100644
index 0000000000..8a8091161d
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.scheduler.cache
+
+import java.util.concurrent.ExecutorService
+
+import com.google.common.collect.Lists
+import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.common.utils.Utils
+import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
+import com.webank.wedatasphere.linkis.entrance.persistence.PersistenceManager
+import com.webank.wedatasphere.linkis.entrance.utils.JobHistoryHelper
+import com.webank.wedatasphere.linkis.governance.common.entity.task.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
+import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
+import com.webank.wedatasphere.linkis.scheduler.SchedulerContext
+import com.webank.wedatasphere.linkis.scheduler.exception.SchedulerErrorException
+import com.webank.wedatasphere.linkis.scheduler.executer.SuccessExecuteResponse
+import com.webank.wedatasphere.linkis.scheduler.queue.Group
+import com.webank.wedatasphere.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer
+import com.webank.wedatasphere.linkis.storage.FSFactory
+import com.webank.wedatasphere.linkis.storage.fs.FileSystem
+import org.apache.commons.io.FilenameUtils
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions._
+
+class ReadCacheConsumer(schedulerContext: SchedulerContext,
+ executeService: ExecutorService, private var group: Group, persistenceManager: PersistenceManager) extends FIFOUserConsumer(schedulerContext, executeService, group) {
+
+ override protected def loop(): Unit = {
+ val event = Option(getConsumeQueue.take())
+ event.foreach{
+ case job: EntranceJob =>
+ job.getTask match {
+ case task: RequestPersistTask =>
+ Utils.tryCatch {
+ val readCacheBefore = TaskUtils.getRuntimeMap(job.getParams).getOrDefault(TaskConstant.READ_CACHE_BEFORE, 300L).asInstanceOf[Long]
+ val cacheResult = JobHistoryHelper.getCache(task.getExecutionCode, task.getExecuteApplicationName, task.getUmUser, readCacheBefore)
+ if (cacheResult != null && StringUtils.isNotBlank(cacheResult.getResultLocation)) {
+ val resultSets = listResults(cacheResult.getResultLocation, job.getUser)
+ if (resultSets.size() > 0) {
+ for (resultSet: FsPath <- resultSets) {
+ val alias = FilenameUtils.getBaseName(resultSet.getPath)
+ val output = FsPath.getFsPath(cacheResult.getResultLocation, FilenameUtils.getName(resultSet.getPath)).getSchemaPath
+ persistenceManager.onResultSetCreated(job, new CacheOutputExecuteResponse(alias, output))
+ }
+ persistenceManager.onResultSizeCreated(job, resultSets.size())
+ }
+ val runtime = TaskUtils.getRuntimeMap(job.getParams)
+ runtime.put(TaskConstant.CACHE, false)
+ TaskUtils.addRuntimeMap(job.getParams, runtime)
+ job.transitionCompleted(new SuccessExecuteResponse(), "Result found in cache")
+ } else {
+ info("Cache not found, submit to normal consumer.")
+ submitToExecute(job)
+ }
+ }{
+ case t: Throwable =>
+ warn("Read cache failed, submit to normal consumer: ", t)
+ submitToExecute(job)
+ }
+ case _ =>
+ }
+ case _ =>
+ }
+ }
+
+ private def listResults(resultLocation: String, user: String) = {
+ val dirPath = FsPath.getFsPath(resultLocation)
+ val fileSystem = FSFactory.getFsByProxyUser(dirPath, user).asInstanceOf[FileSystem]
+ Utils.tryFinally {
+ fileSystem.init(null)
+ if (fileSystem.exists(dirPath)) {
+ fileSystem.listPathWithError(dirPath).getFsPaths
+ } else {
+ Lists.newArrayList[FsPath]();
+ }
+ }(Utils.tryQuietly(fileSystem.close()))
+ }
+
+ private def submitToExecute(job: EntranceJob) = {
+ val runtime = TaskUtils.getRuntimeMap(job.getParams)
+ runtime.put(TaskConstant.READ_FROM_CACHE, false)
+ TaskUtils.addRuntimeMap(job.getParams, runtime)
+ val groupName = schedulerContext.getOrCreateGroupFactory.getGroupNameByEvent(job)
+ val consumer = schedulerContext.getOrCreateConsumerManager.getOrCreateConsumer(groupName)
+ val index = consumer.getConsumeQueue.offer(job)
+ //index.map(getEventId(_, groupName)).foreach(job.setId)
+ if (index.isEmpty) throw new SchedulerErrorException(12001, "The submission job failed and the queue is full!(提交作业失败，队列已满！)")
+ }
+
+ private def getEventId(index: Int, groupName: String): String = groupName + "_" + index
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala
new file mode 100644
index 0000000000..b4c39f1042
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.scheduler.cache
+
+import com.webank.wedatasphere.linkis.entrance.persistence.PersistenceManager
+import com.webank.wedatasphere.linkis.entrance.scheduler.EntranceGroupFactory
+import com.webank.wedatasphere.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer
+import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager
+
+class ReadCacheConsumerManager(maxParallelismUsers: Int, persistenceManager: PersistenceManager) extends ParallelConsumerManager(maxParallelismUsers){
+
+ override protected def createConsumer(groupName: String): FIFOUserConsumer = {
+ val group = getSchedulerContext.getOrCreateGroupFactory.getOrCreateGroup(groupName)
+ if(groupName.endsWith(EntranceGroupFactory.CACHE)){
+ info("Create cache consumer with group: " + groupName)
+ new ReadCacheConsumer(getSchedulerContext, getOrCreateExecutorService, group, persistenceManager)
+ } else {
+ info("Create normal consumer with group: " + groupName)
+ new FIFOUserConsumer(getSchedulerContext, getOrCreateExecutorService, group)
+ }
+ }
+
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala
similarity index 78%
rename from ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala
rename to linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala
index f330fcd4bf..17dcb8d145 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala
@@ -1,21 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package com.webank.wedatasphere.linkis.entrance.utils
import java.util
-import com.google.gson.Gson
import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.exception.JobHistoryFailedException
+import com.webank.wedatasphere.linkis.governance.common.entity.task.{RequestPersistTask, RequestQueryTask, RequestUpdateTask, ResponsePersist}
import com.webank.wedatasphere.linkis.protocol.query.cache.{CacheTaskResult, RequestReadCache}
-import com.webank.wedatasphere.linkis.protocol.query.{RequestPersistTask, RequestQueryTask, RequestUpdateTask, ResponsePersist}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState
-/**
- * created by cooperyang on 2020/1/2
- * Description:
- */
+
object JobHistoryHelper extends Logging{
private val sender = Sender.getSender(EntranceConfiguration.QUERY_PERSISTENCE_SPRING_APPLICATION_NAME.getValue)
@@ -61,13 +73,7 @@ object JobHistoryHelper extends Logging{
}else{
val data = responsePersist.getData
data.get(TASK_MAP_KEY) match {
- case tasks:util.List[util.Map[String, Object]] => tasks.get(0) match {
- case map:util.Map[String, Object] => val gson = new Gson()
- val json = gson.toJson(map)
- val requestPersistTask = gson.fromJson(json, classOf[RequestPersistTask])
- requestPersistTask
- case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct RequestPersistTask type taskId is $taskID")
- }
+ case tasks: util.List[RequestPersistTask] => tasks.get(0)
case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct List type taskId is $taskID")
}
}
diff --git a/linkis-computation-governance/linkis-jdbc-driver/pom.xml b/linkis-computation-governance/linkis-jdbc-driver/pom.xml
new file mode 100644
index 0000000000..d9ce050d0e
--- /dev/null
+++ b/linkis-computation-governance/linkis-jdbc-driver/pom.xml
@@ -0,0 +1,80 @@
+
+
+
+
+
+ 4.0.0
+
+ linkis
+ com.webank.wedatasphere.linkis
+ 1.0.0-RC1
+
+ linkis-jdbc-driver
+
+
+
+ com.webank.wedatasphere.linkis
+ linkis-computation-client
+ ${linkis.version}
+
+
+ junit
+ junit
+ RELEASE
+ test
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+ maven-assembly-plugin
+
+
+ jar-with-dependencies
+
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ **/*.yml
+ **/*.properties
+ **/*.sh
+ **/log4j2.xml
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${project.artifactId}-${project.version}
+
+
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/com/webank/wedatasphere/linkis/ujes/jdbc/LinkisBaseResultSet.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/com/webank/wedatasphere/linkis/ujes/jdbc/LinkisBaseResultSet.java
new file mode 100644
index 0000000000..dcedfa50a1
--- /dev/null
+++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/com/webank/wedatasphere/linkis/ujes/jdbc/LinkisBaseResultSet.java
@@ -0,0 +1,1088 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.ujes.jdbc;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.net.URL;
+import java.sql.*;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Data independed base class which implements the common part of
+ * all hive resultsets.
+ */
+public abstract class LinkisBaseResultSet implements ResultSet{
+ protected SQLWarning warningChain = null;
+ protected boolean wasNull = false;
+ protected List