chore(version): update to version 'v0.8.0'.

.gitattributes

@@ -0,0 +1,9 @@
+# Match the .editorconfig
+*           text=auto eol=lf
+
+# Scripts
+*.bat       text eol=crlf
+*.sh        text eol=lf
+
+# Gradle wrapper
+/gradlew    text eol=lf

.github/workflows/main.yml

@@ -193,7 +193,7 @@ jobs:
       # Docker Build and push
       - name: Push to Docker Hub
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: true
@@ -287,12 +287,12 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
       - name: Copy gradle properties
-        run: >-
+        run: |
           cp gradle.properties core/src/main/resources/scripts/python/gradle.properties
           cp README.md core/src/main/resources/scripts/python/README.md
 

build.gradle

@@ -3,21 +3,21 @@ plugins {
     id "java"
     id 'java-library'
     id "idea"
-    id "com.github.johnrengelman.shadow" version "7.1.2"
+    id "com.github.johnrengelman.shadow" version "8.1.0"
     id "application"
 
     // test
     id 'com.adarshr.test-logger' version '3.2.0'
-    id 'org.gradle.test-retry' version '1.5.0'
+    id 'org.gradle.test-retry' version '1.5.2'
 
     // helper
-    id "com.github.ben-manes.versions" version "0.45.0"
+    id "com.github.ben-manes.versions" version "0.46.0"
 
     // front
     id 'org.siouan.frontend-jdk11' version '6.0.0' apply false
 
     // release
-    id "io.github.gradle-nexus.publish-plugin" version "1.1.0"
+    id "io.github.gradle-nexus.publish-plugin" version "1.3.0"
     id 'net.researchgate.release' version '3.0.2'
     id "com.gorylenko.gradle-git-properties" version "2.4.1"
     id 'signing'
@@ -82,14 +82,6 @@ allprojects {
         }
     }
 
-    // https://github.com/micronaut-projects/micronaut-openapi/issues/894
-    configurations.all {
-        resolutionStrategy {
-            force("io.micronaut.openapi:micronaut-openapi-bom:4.7.1")
-            force("io.micronaut.openapi:micronaut-openapi:4.7.1")
-        }
-    }
-
     // dependencies
     dependencies {
         // lombok
@@ -239,13 +231,17 @@ jar {
 }
 
 shadowJar {
-    dependsOn = ['ui:assembleFrontend']
-
     archiveClassifier.set(null)
     mergeServiceFiles()
     zip64 true
 }
 
+distZip.dependsOn shadowJar
+distTar.dependsOn shadowJar
+startScripts.dependsOn shadowJar
+startShadowScripts.dependsOn jar
+shadowJar.dependsOn 'ui:assembleFrontend'
+
 /**********************************************************************************************************************\
  * Executable Jar
  **********************************************************************************************************************/
@@ -361,9 +357,13 @@ subprojects {
     }
 
     task sourcesJar(type: Jar) {
+        dependsOn = [':core:copyGradleProperties']
+        dependsOn = [':ui:assembleFrontend']
         archiveClassifier.set('sources')
         from sourceSets.main.allSource
     }
+    sourcesJar.dependsOn ':core:copyGradleProperties'
+    sourcesJar.dependsOn ':ui:assembleFrontend'
 
     task javadocJar(type: Jar) {
         archiveClassifier.set('javadoc')

cli/src/main/java/io/kestra/cli/commands/AbstractServiceNamespaceUpdateCommand.java

@@ -1,6 +1,8 @@
 package io.kestra.cli.commands;
 
 import io.kestra.cli.AbstractApiCommand;
+import lombok.*;
+import lombok.extern.jackson.Jacksonized;
 import picocli.CommandLine;
 
 import java.nio.file.Path;
@@ -14,4 +16,12 @@ public abstract class AbstractServiceNamespaceUpdateCommand extends AbstractApiC
 
     @CommandLine.Option(names = {"--no-delete"}, description = "if missing should not be deleted")
     public boolean noDelete = false ;
+
+    @Builder
+    @Value
+    @Jacksonized
+    public static class UpdateResult {
+        String id;
+        String namespace;
+    }
 }

cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java

@@ -0,0 +1,37 @@
+package io.kestra.cli.commands.flows;
+
+import io.kestra.cli.AbstractCommand;
+import io.kestra.core.models.flows.Flow;
+import io.kestra.core.models.validations.ModelValidator;
+import io.kestra.core.serializers.YamlFlowParser;
+import jakarta.inject.Inject;
+import picocli.CommandLine;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+@CommandLine.Command(
+    name = "expand",
+    description = "expand a flow"
+)
+public class FlowExpandCommand extends AbstractCommand {
+
+    @CommandLine.Parameters(index = "0", description = "the flow file to expand")
+    private Path file;
+
+    @Inject
+    private YamlFlowParser yamlFlowParser;
+
+    @Inject
+    private ModelValidator modelValidator;
+
+    @Override
+    public Integer call() throws Exception {
+        super.call();
+        String content = IncludeHelperExpander.expand(Files.readString(file), file.getParent());
+        Flow flow = yamlFlowParser.parse(content, Flow.class);
+        modelValidator.validate(flow);
+        stdOut(content);
+        return 0;
+    }
+}

cli/src/main/java/io/kestra/cli/commands/flows/IncludeHelperExpander.java

@@ -0,0 +1,39 @@
+package io.kestra.cli.commands.flows;
+
+import com.google.common.io.Files;
+import lombok.SneakyThrows;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public abstract class IncludeHelperExpander {
+
+    public static String expand(String value, Path directory) throws IOException {
+        return value.lines()
+            .map(line -> line.contains("[[>") && line.contains("]]") ? expandLine(line, directory) : line)
+            .collect(Collectors.joining("\n"));
+    }
+
+    @SneakyThrows
+    private static String expandLine(String line, Path directory) {
+        String prefix = line.substring(0, line.indexOf("[[>"));
+        String suffix = line.substring(line.indexOf("]]") + 2, line.length());
+        String file = line.substring(line.indexOf("[[>") + 3 , line.indexOf("]]")).strip();
+        Path includePath = directory.resolve(file);
+        List<String> include = Files.readLines(includePath.toFile(), Charset.defaultCharset());
+
+        // handle single line directly with the suffix (should be between quotes or double-quotes
+        if(include.size() == 1) {
+            String singleInclude = include.get(0);
+            return prefix + singleInclude + suffix;
+        }
+
+        // multi-line will be expanded with the prefix but no suffix
+        return include.stream()
+            .map(includeLine -> prefix + includeLine)
+            .collect(Collectors.joining("\n"));
+    }
+}

cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java

@@ -2,7 +2,7 @@
 
 import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
 import io.kestra.cli.commands.flows.FlowValidateCommand;
-import io.kestra.core.models.flows.FlowWithSource;
+import io.kestra.cli.commands.flows.IncludeHelperExpander;
 import io.kestra.core.serializers.YamlFlowParser;
 import io.micronaut.core.type.Argument;
 import io.micronaut.http.HttpRequest;
@@ -42,7 +42,7 @@ public Integer call() throws Exception {
                 .filter(YamlFlowParser::isValidExtension)
                 .map(path -> {
                     try {
-                        return Files.readString(path, Charset.defaultCharset());
+                        return IncludeHelperExpander.expand(Files.readString(path, Charset.defaultCharset()), path.getParent());
                     } catch (IOException e) {
                         throw new RuntimeException(e);
                     }
@@ -59,9 +59,9 @@ public Integer call() throws Exception {
                 MutableHttpRequest<String> request = HttpRequest
                     .POST("/api/v1/flows/" + namespace + "?delete=" + !noDelete, body).contentType(MediaType.APPLICATION_YAML);
 
-                List<FlowWithSource> updated = client.toBlocking().retrieve(
+                List<UpdateResult> updated = client.toBlocking().retrieve(
                     this.requestOptions(request),
-                    Argument.listOf(FlowWithSource.class)
+                    Argument.listOf(UpdateResult.class)
                 );
 
                 stdOut(updated.size() + " flow(s) for namespace '" + namespace + "' successfully updated !");

cli/src/main/java/io/kestra/cli/commands/sys/RestoreQueueCommand.java

@@ -30,6 +30,9 @@ public class RestoreQueueCommand extends AbstractCommand {
     @CommandLine.Option(names = {"--no-triggers"}, description = "Don't send triggers")
     private boolean noTriggers = false;
 
+    @CommandLine.Option(names = {"--no-triggers-execution-id"}, description = "Remove executionId from trigger")
+    private boolean noTriggerExecutionId = false;
+
     @Override
     public Integer call() throws Exception {
         super.call();
@@ -50,7 +53,7 @@ public Integer call() throws Exception {
 
         // trigger
         if (!this.noTriggers) {
-            int size = restoreQueueService.triggers(noRecreate);
+            int size = restoreQueueService.triggers(noRecreate, noTriggerExecutionId);
             stdOut("Successfully send {0} triggers", size);
         }
 

cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java

@@ -47,9 +47,9 @@ public Integer call() throws Exception {
                 MutableHttpRequest<List<Template>> request = HttpRequest
                     .POST("/api/v1/templates/" + namespace + "?delete=" + !noDelete, templates);
 
-                List<Template> updated = client.toBlocking().retrieve(
+                List<UpdateResult> updated = client.toBlocking().retrieve(
                     this.requestOptions(request),
-                    Argument.listOf(Template.class)
+                    Argument.listOf(UpdateResult.class)
                 );
 
                 stdOut(updated.size() + " template(s) for namespace '" + namespace + "' successfully updated !");

cli/src/main/java/io/kestra/cli/services/RestoreQueueService.java

@@ -47,10 +47,20 @@ public int templates(boolean noRecreate) {
         return this.send(templates, QueueFactoryInterface.TEMPLATE_NAMED, Template.class, noRecreate);
     }
 
-    public int triggers(boolean noRecreate) {
+    public int triggers(boolean noRecreate, boolean noTriggerExecutionId) {
         TriggerRepositoryInterface triggerRepository = applicationContext.getBean(TriggerRepositoryInterface.class);
         List<Trigger> triggers = new ArrayList<>(triggerRepository.findAll());
 
+        if (noTriggerExecutionId) {
+            triggers = triggers
+                .stream()
+                .map(trigger -> trigger.toBuilder()
+                    .executionId(null)
+                    .build()
+                )
+                .collect(Collectors.toList());
+        }
+
         return this.send(triggers, QueueFactoryInterface.TRIGGER_NAMED, Trigger.class, noRecreate);
     }
 

cli/src/main/resources/application.yml

@@ -52,6 +52,8 @@ flyway:
       enabled: true
       locations:
         - classpath:migrations/postgres
+      # We must ignore missing migrations as a V6 wrong migration was created and replaced by the V11
+      ignore-migration-patterns: "*:missing"
     mysql:
       enabled: true
       locations:
@@ -60,6 +62,8 @@ flyway:
       enabled: true
       locations:
         - classpath:migrations/h2
+      # We must ignore missing migrations as a V8 wrong migration was created and replaced by the V11
+      ignore-migration-patterns: "*:missing"
 
 kestra:
   retries:
@@ -87,6 +91,9 @@ kestra:
       logs:
         table: "logs"
         cls: io.kestra.core.models.executions.LogEntry
+      metrics:
+        table: "metrics"
+        cls: io.kestra.core.models.executions.MetricEntry
       multipleconditions:
         table: "multipleconditions"
         cls: io.kestra.core.models.triggers.multipleflows.MultipleConditionWindow
@@ -116,45 +123,6 @@ kestra:
       fixed-delay: 1h
       retention: 7d
 
-  elasticsearch:
-    defaults:
-      indice-prefix: "kestra_"
-    indices:
-      flows:
-        index: "${kestra.elasticsearch.defaults.indice-prefix}flows"
-        cls: io.kestra.core.models.flows.Flow
-        mapping-file: flow
-      flows-revisions:
-        index: "${kestra.elasticsearch.defaults.indice-prefix}flows_revisions"
-        cls: io.kestra.core.models.flows.Flow
-        mapping-file: flow
-      executions:
-        index: "${kestra.elasticsearch.defaults.indice-prefix}executions"
-        cls: io.kestra.core.models.executions.Execution
-        mapping-file: execution
-      templates:
-        index: "${kestra.elasticsearch.defaults.indice-prefix}templates"
-        cls: io.kestra.core.models.templates.Template
-        mapping-file: template
-      triggers:
-        index: "${kestra.elasticsearch.defaults.indice-prefix}triggers"
-        cls: io.kestra.core.models.triggers.Trigger
-        mapping-file: trigger
-      logs:
-        index: "${kestra.elasticsearch.defaults.indice-prefix}logs"
-        cls: io.kestra.core.models.executions.LogEntry
-        mapping-file: log
-      settings:
-        index: "${kestra.elasticsearch.defaults.indice-prefix}settings"
-        cls: io.kestra.core.models.Setting
-        mapping-file: setting
-
-  indexer:
-    models:
-      - io.kestra.core.models.executions.Execution
-      - io.kestra.core.models.triggers.Trigger
-      - io.kestra.core.models.executions.LogEntry
-
   plugins:
     repositories:
       central: