From 001c2ce096be418bf23696f26b5a7ec439779d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 21 Mar 2023 08:44:33 +0000 Subject: [PATCH 1/4] Add active executions state --- app/http.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/http.php b/app/http.php index efd8d2e..c00e083 100644 --- a/app/http.php +++ b/app/http.php @@ -105,6 +105,7 @@ $table->column('hostname', Table::TYPE_STRING, 256); $table->column('status', Table::TYPE_STRING, 128); $table->column('key', Table::TYPE_STRING, 256); + $table->column('executions', Table::TYPE_INT, 11); // Amount of active executions $table->create(); return $table; @@ -311,6 +312,7 @@ function removeAllRuntimes(Table $activeRuntimes, Pool $orchestrationPool): void 'updated' => $startTime, 'status' => 'pending', 'key' => $secret, + 'executions' => 0 ]); /** @@ -445,6 +447,7 @@ function removeAllRuntimes(Table $activeRuntimes, Pool $orchestrationPool): void 'updated' => \microtime(true), 'status' => 'Up ' . \round($duration, 2) . 's', 'key' => $secret, + 'executions' => 0 ]); } catch (Throwable $th) { $localDevice->deletePath($tmpFolder); @@ -555,6 +558,8 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st $activeRuntimeId = $runtimeId; // Used with Swoole table (key) $runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name) + $activeRuntimes->incr($activeRuntimeId, 'executions', 1); + $variables = \array_merge($variables, [ 'INERNAL_EXECUTOR_HOSTNAME' => System::getHostname() ]); @@ -764,6 +769,8 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st $runtime['updated'] = \microtime(true); $activeRuntimes->set($activeRuntimeId, $runtime); + $activeRuntimes->decr($activeRuntimeId, 'executions', 1); + // Finish request $response ->setStatusCode(Response::STATUS_CODE_OK) @@ -914,7 +921,8 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st Console::info("Running maintenance task ..."); foreach ($activeRuntimes as $activeRuntimeId => $runtime) { $inactiveThreshold = \time() - \intval(App::getEnv('OPR_EXECUTOR_INACTIVE_TRESHOLD', '60')); - if ($runtime['updated'] < $inactiveThreshold) { + + if ($runtime['updated'] < $inactiveThreshold && $runtime['executions'] === 0) { go(function () use ($activeRuntimeId, $runtime, $orchestrationPool, $activeRuntimes) { try { $connection = $orchestrationPool->pop(); From f62d9edaa61e3735c11ad1a957a2568c96a6d29f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 21 Mar 2023 09:40:55 +0000 Subject: [PATCH 2/4] Fix bug --- app/http.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/http.php b/app/http.php index c00e083..bfc9b67 100644 --- a/app/http.php +++ b/app/http.php @@ -558,8 +558,6 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st $activeRuntimeId = $runtimeId; // Used with Swoole table (key) $runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name) - $activeRuntimes->incr($activeRuntimeId, 'executions', 1); - $variables = \array_merge($variables, [ 'INERNAL_EXECUTOR_HOSTNAME' => System::getHostname() ]); @@ -640,6 +638,8 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st } } + $activeRuntimes->incr($activeRuntimeId, 'executions', 1); + // Ensure runtime started for ($i = 0; $i < 5; $i++) { if ($activeRuntimes->get($activeRuntimeId)['status'] !== 'pending') { From b62b870debedbb4aab935cb917ff14e7ca0e6cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 24 May 2023 18:25:52 +0200 Subject: [PATCH 3/4] PR review changes --- app/http.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/http.php b/app/http.php index bfc9b67..c25ec48 100644 --- a/app/http.php +++ b/app/http.php @@ -765,9 +765,9 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st ]; // Update swoole table - $runtime = $activeRuntimes->get($activeRuntimeId); - $runtime['updated'] = \microtime(true); - $activeRuntimes->set($activeRuntimeId, $runtime); + $activeRuntimes->set($activeRuntimeId, [ + 'updated' => \microtime(true) + ]); $activeRuntimes->decr($activeRuntimeId, 'executions', 1); From a39458155ba4066a572d7bb7584ebf3e747839f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 24 May 2023 20:09:20 +0200 Subject: [PATCH 4/4] Add force kill --- .env | 1 + README.md | 2 ++ app/http.php | 11 ++++++++++- docker-compose.yml | 1 + 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 2e0c698..5b155e6 100644 --- a/.env +++ b/.env @@ -2,6 +2,7 @@ OPR_EXECUTOR_ENV=development OPR_EXECUTOR_RUNTIMES=php-8.1,dart-2.17,deno-1.24,node-18.0,python-3.9,ruby-3.1,cpp-17 OPR_EXECUTOR_CONNECTION_STORAGE=file://localhost OPR_EXECUTOR_INACTIVE_TRESHOLD=60 +OPR_EXECUTOR_RUNTIME_TRESHOLD=600 OPR_EXECUTOR_MAINTENANCE_INTERVAL=60 OPR_EXECUTOR_NETWORK=openruntimes-runtimes OPR_EXECUTOR_SECRET=executor-secret-key diff --git a/README.md b/README.md index c356b09..ab14567 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ services: - OPR_EXECUTOR_RUNTIMES - OPR_EXECUTOR_CONNECTION_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD + - OPR_EXECUTOR_RUNTIME_TRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK - OPR_EXECUTOR_SECRET @@ -78,6 +79,7 @@ OPR_EXECUTOR_ENV=development OPR_EXECUTOR_RUNTIMES=php-8.0 OPR_EXECUTOR_CONNECTION_STORAGE=file://localhost OPR_EXECUTOR_INACTIVE_TRESHOLD=60 +OPR_EXECUTOR_RUNTIME_TRESHOLD=60 OPR_EXECUTOR_MAINTENANCE_INTERVAL=60 OPR_EXECUTOR_NETWORK=openruntimes-runtimes OPR_EXECUTOR_SECRET=executor-secret-key diff --git a/app/http.php b/app/http.php index c25ec48..b5897d6 100644 --- a/app/http.php +++ b/app/http.php @@ -638,6 +638,11 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st } } + // Update swoole table + $activeRuntimes->set($activeRuntimeId, [ + 'updated' => \microtime(true) + ]); + $activeRuntimes->incr($activeRuntimeId, 'executions', 1); // Ensure runtime started @@ -921,8 +926,12 @@ function (string $runtimeId, string $payload, array $variables, int $timeout, st Console::info("Running maintenance task ..."); foreach ($activeRuntimes as $activeRuntimeId => $runtime) { $inactiveThreshold = \time() - \intval(App::getEnv('OPR_EXECUTOR_INACTIVE_TRESHOLD', '60')); + $runtimeThreshold = \time() - \intval(App::getEnv('OPR_EXECUTOR_RUNTIME_TRESHOLD', '600')); + + $softKill = $runtime['updated'] < $inactiveThreshold && $runtime['executions'] === 0; + $forceKill = $runtime['updated'] < $runtimeThreshold; - if ($runtime['updated'] < $inactiveThreshold && $runtime['executions'] === 0) { + if ($softKill || $forceKill) { go(function () use ($activeRuntimeId, $runtime, $orchestrationPool, $activeRuntimes) { try { $connection = $orchestrationPool->pop(); diff --git a/docker-compose.yml b/docker-compose.yml index 048fab6..9cca04a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,7 @@ services: - OPR_EXECUTOR_RUNTIMES - OPR_EXECUTOR_CONNECTION_STORAGE - OPR_EXECUTOR_INACTIVE_TRESHOLD + - OPR_EXECUTOR_RUNTIME_TRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK - OPR_EXECUTOR_SECRET