diff --git a/build.gradle b/build.gradle index 5e840af70..d1e3293bf 100644 --- a/build.gradle +++ b/build.gradle @@ -70,6 +70,7 @@ dependencies { implementation "software.amazon.awssdk:ecrpublic" implementation 'software.amazon.awssdk:ses' implementation 'org.yaml:snakeyaml:2.0' + implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' //object storage dependency implementation("io.micronaut.objectstorage:micronaut-object-storage-aws") // include sts to allow the use of service account role - https://stackoverflow.com/a/73306570 diff --git a/configuration.md b/configuration.md index 9c8daf16b..b7523b17c 100644 --- a/configuration.md +++ b/configuration.md @@ -216,7 +216,7 @@ Wave offers a feature to provide a cache for Docker blobs, which improves the pe - **`wave.blobCache.enabled`**: whether to enable the blob cache. It is `false` by default. *Optional*. -- **`wave.blobCache.s5cmdImage`**: the Docker image that supplies the [s5cmd tool](https://github.com/peak/s5cmd). This tool is used to upload blob binaries to the S3 bucket. The default image used by Wave is `cr.seqera.io/public/wave/s5cmd:v2.2.2`. *Optional*. +- **`wave.blobCache.s5cmdImage`**: the Docker image that supplies the [s5cmd tool](https://github.com/peak/s5cmd). This tool is used to upload blob binaries to the S3 bucket. The default image used by Wave is `public.cr.seqera.io/wave/s5cmd:v2.2.2`. *Optional*. - **`wave.blobCache.status.delay`**: the time delay in checking the status of the transfer of the blob binary from the repository to the cache. Its default value is `5s`. *Optional*. diff --git a/docs/get-started.mdx b/docs/get-started.mdx index b779c1e6c..08c63b587 100644 --- a/docs/get-started.mdx +++ b/docs/get-started.mdx @@ -18,8 +18,8 @@ In this guide, you'll request a containerized Conda package from Seqera Containe ### Request a Conda package as a Seqera Container 1. Open [Seqera Containers][sc] in a browser. -1. In the search box, enter `faker`. -1. In the search results, select **Add** in the `conda-forge::faker` result, and then **Get Container** to initiate the container build. +1. In the search box, enter `samtools`. +1. In the search results, select **Add** in the `bioconda::samtools` result, and then **Get Container** to initiate the container build. 1. From the **Fetching container** modal, copy the the durable container image URI that Seqera Containers provides. 1. Optional: Select **View build details** to watch Seqera Containers build the requested container in real time. @@ -39,25 +39,25 @@ Nextflow can use the container that Seqera Containers built in the previous sect 1. Create a `main.nf` file with the following contents: ```groovy - process FAKER { + process SAMTOOLS { container '' debug true - """ - faker address + samtools --version-only """ } - workflow { - FAKER() + SAMTOOLS() } ``` - Substitute `` for the container URI that you received from Seqera Containers in the previous section. + Substitute `` for the container URI that you received from Seqera Containers in the previous section. e.g. + - `community.wave.seqera.io/library/samtools:1.20--b5dfbd93de237464` for linux/amd64. + - `community.wave.seqera.io/library/samtools:1.20--497854c5df637867` for linux/arm64. ### Run the Nextflow pipeline -To confirm that the `faker` command is available from your pipeline, run the following command: +To confirm that the `samtools` command is available from your pipeline, run the following command: ``` nextflow run main.nf @@ -66,12 +66,14 @@ nextflow run main.nf The output from a successful execution is displayed in the following example: ``` -Launching `main.nf` [jolly_edison] DSL2 - revision: 5c414bd927 + N E X T F L O W ~ version 24.04.4 + +Launching `samtools.nf` [furious_carlsson] DSL2 - revision: 04817f962f executor > local (1) -[86/0d56e8] faker | 1 of 1 ✔ -234 Nicholas Circle -Masonport, MS 98018 +[2f/d2ccc7] process > SAMTOOLS [100%] 1 of 1 ✔ +1.20+htslib-1.20 + ``` ## Nextflow diff --git a/s5cmd/Makefile b/s5cmd/Makefile index 2de4f13e6..45ea7ee29 100644 --- a/s5cmd/Makefile +++ b/s5cmd/Makefile @@ -5,5 +5,5 @@ build: --push \ --platform linux/amd64,linux/arm64 \ --build-arg version=${version} \ - --tag cr.seqera.io/public/wave/s5cmd:v${version} \ + --tag public.cr.seqera.io/wave/s5cmd:v${version} \ . diff --git a/s5cmd/dist.sh b/s5cmd/dist.sh new file mode 100644 index 000000000..af66e14cb --- /dev/null +++ b/s5cmd/dist.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# +# Wave, containers provisioning service +# Copyright (c) 2023-2024, Seqera Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +arch=$(uname -m) + +case $arch in + x86_64|amd64) + echo "https://github.com/peak/s5cmd/releases/download/v$1/s5cmd_$1_Linux-64bit.tar.gz" + ;; + aarch64|arm64) + echo "https://github.com/peak/s5cmd/releases/download/v$1/s5cmd_$1_Linux-arm64.tar.gz" + ;; + *) + echo "Unknown architecture: $arch" + ;; +esac diff --git a/src/main/groovy/io/seqera/wave/ErrorHandler.groovy b/src/main/groovy/io/seqera/wave/ErrorHandler.groovy index 602385f33..0d872918b 100644 --- a/src/main/groovy/io/seqera/wave/ErrorHandler.groovy +++ b/src/main/groovy/io/seqera/wave/ErrorHandler.groovy @@ -18,8 +18,6 @@ package io.seqera.wave -import java.util.function.BiFunction - import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.http.HttpRequest @@ -32,6 +30,7 @@ import io.seqera.wave.exception.DockerRegistryException import io.seqera.wave.exception.ForbiddenException import io.seqera.wave.exception.HttpResponseException import io.seqera.wave.exception.NotFoundException +import io.seqera.wave.exception.RegistryForwardException import io.seqera.wave.exception.SlowDownException import io.seqera.wave.exception.UnauthorizedException import io.seqera.wave.exception.WaveException @@ -46,10 +45,14 @@ import jakarta.inject.Singleton @Singleton class ErrorHandler { + static interface Mapper { + T apply(String message, String errorCode) + } + @Value('${wave.debug:false}') private Boolean debug - def HttpResponse handle(HttpRequest httpRequest, Throwable t, BiFunction responseFactory) { + def HttpResponse handle(HttpRequest httpRequest, Throwable t, Mapper responseFactory) { final errId = LongRndKey.rndHex() final request = httpRequest?.toString() def msg = t.message @@ -78,6 +81,14 @@ class ErrorHandler { log.error(render, t) } + if( t instanceof RegistryForwardException ) { + // report this error as it has been returned by the target registry + return HttpResponse + .status(HttpStatus.valueOf(t.statusCode)) + .body(t.response) + .headers(t.headers) + } + if( t instanceof DockerRegistryException ) { final resp = responseFactory.apply(msg, t.error) return HttpResponseFactory.INSTANCE.status(t.statusCode).body(resp) diff --git a/src/main/groovy/io/seqera/wave/WaveDefault.groovy b/src/main/groovy/io/seqera/wave/WaveDefault.groovy index 7b1fd6254..ec8748a5a 100644 --- a/src/main/groovy/io/seqera/wave/WaveDefault.groovy +++ b/src/main/groovy/io/seqera/wave/WaveDefault.groovy @@ -17,11 +17,14 @@ */ package io.seqera.wave + +import groovy.transform.CompileStatic /** * Wave app defaults * * @author Paolo Di Tommaso */ +@CompileStatic interface WaveDefault { final static public String DOCKER_IO = 'docker.io' @@ -38,7 +41,7 @@ interface WaveDefault { 'application/vnd.docker.distribution.manifest.list.v2+json' ) ) - final public static int[] HTTP_REDIRECT_CODES = List.of(301, 302, 303, 307, 308) + final public static List HTTP_REDIRECT_CODES = List.of(301, 302, 303, 307, 308) final public static List HTTP_SERVER_ERRORS = List.of(500, 502, 503, 504) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy index eca481b02..9e1c01082 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.auth +import io.seqera.wave.exception.RegistryUnauthorizedAccessException + /** * Declares container registry authentication & authorization operations * diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index bdc7fe6da..c74020994 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -35,6 +35,8 @@ import groovy.transform.PackageScope import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.configuration.HttpClientConfig +import io.seqera.wave.exception.RegistryForwardException +import io.seqera.wave.exception.RegistryUnauthorizedAccessException import io.seqera.wave.http.HttpClientFactory import io.seqera.wave.util.RegHelper import io.seqera.wave.util.Retryable @@ -42,7 +44,7 @@ import io.seqera.wave.util.StringUtils import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.WaveDefault.DOCKER_IO -import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS +import static io.seqera.wave.auth.RegistryUtils.isServerError /** * Implement Docker authentication & login service * @@ -110,7 +112,6 @@ class RegistryAuthServiceImpl implements RegistryAuthService { @Inject RegistryCredentialsFactory credentialsFactory - /** * Implements container registry login * @@ -143,10 +144,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService { .header("Authorization", "Basic $basic") .build() // retry strategy + // note: do not retry on 429 error code because it just continues to report the error + // for a while. better returning the error to the upstream client + // see also https://github.com/docker/hub-feedback/issues/1907#issuecomment-631028965 final retryable = Retryable .>of(httpConfig) - .retryIf( (response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS) - .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event}")) + .retryIf((response) -> isServerError(response)) + .onRetry((event) -> log.warn("Unable to connect '$endpoint' - attempt: ${event.attempt} status: ${event.result?.statusCode()}; body: ${event.result?.body()}")) // make the request final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString())) final body = response.body() @@ -230,10 +234,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService { log.trace "Token request=$req" // retry strategy + // note: do not retry on 429 error code because it just continues to report the error + // for a while. better returning the error to the upstream client + // see also https://github.com/docker/hub-feedback/issues/1907#issuecomment-631028965 final retryable = Retryable .>of(httpConfig) - .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_RETRYABLE_ERRORS ) - .onRetry((event) -> log.warn("Unable to connect '$login' - event: $event")) + .retryIf((response) -> isServerError(response)) + .onRetry((event) -> log.warn("Unable to connect '$login' - attempt: ${event.attempt} status: ${event.result?.statusCode()}; body: ${event.result?.body()}")) // submit http request final response = retryable.apply(()-> httpClient.send(req, HttpResponse.BodyHandlers.ofString())) // check the response @@ -248,8 +255,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { return token } } - - throw new RegistryUnauthorizedAccessException("Unable to authorize request: $login", response.statusCode(), body) + throw new RegistryForwardException("Unexpected response acquiring token for '$login' [${response.statusCode()}]", response) } String buildLoginUrl(URI realm, String image, String service){ diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index 02781c8ca..d1535d8b5 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -20,21 +20,24 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.util.concurrent.ExecutionException import java.util.concurrent.TimeUnit import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader import com.google.common.cache.LoadingCache +import com.google.common.util.concurrent.UncheckedExecutionException import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.configuration.HttpClientConfig +import io.seqera.wave.exception.RegistryForwardException import io.seqera.wave.http.HttpClientFactory import io.seqera.wave.util.Retryable import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.WaveDefault.DOCKER_IO import static io.seqera.wave.WaveDefault.DOCKER_REGISTRY_1 -import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS +import static io.seqera.wave.auth.RegistryUtils.isServerError /** * Lookup service for container registry. The role of this component * is to registry the retrieve the registry authentication realm @@ -81,13 +84,15 @@ class RegistryLookupServiceImpl implements RegistryLookupService { final httpClient = HttpClientFactory.followRedirectsHttpClient() final request = HttpRequest.newBuilder() .uri(endpoint) .GET() .build() // retry strategy + // note: do not retry on 429 error code because it just continues to report the error + // for a while. better returning the error to the upstream client + // see also https://github.com/docker/hub-feedback/issues/1907#issuecomment-631028965 final retryable = Retryable .>of(httpConfig) - .retryIf((response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS ) - .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event")) + .retryIf((response) -> isServerError(response)) + .onRetry((event) -> log.warn("Unable to connect '$endpoint' - attempt: ${event.attempt} status: ${event.result?.statusCode()}; body: ${event.result?.body()}")) // submit the request final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString())) - final body = response.body() // check response final code = response.statusCode() if( code == 401 ) { @@ -102,9 +107,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService { else if( code == 200 ) { return new RegistryAuth(endpoint) } - else { - throw new IllegalArgumentException("Request '$endpoint' unexpected response code: $code; message: ${body} ") - } + throw new RegistryForwardException("Unexpected response for '$endpoint' [${response.statusCode()}]", response) } /** @@ -117,8 +120,10 @@ class RegistryLookupServiceImpl implements RegistryLookupService { final auth = cache.get(endpoint) return new RegistryInfo(registry, endpoint, auth) } - catch (Throwable t) { - throw new RegistryLookupException("Unable to lookup authority for registry '$registry'", t) + catch (UncheckedExecutionException | ExecutionException e) { + // this catches the exception thrown in the cache loader lookup + // and throws the causing exception that should be `RegistryUnauthorizedAccessException` + throw e.cause } } diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryUtils.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryUtils.groovy new file mode 100644 index 000000000..5ae75f5ae --- /dev/null +++ b/src/main/groovy/io/seqera/wave/auth/RegistryUtils.groovy @@ -0,0 +1,37 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth + +import java.net.http.HttpResponse + +import groovy.transform.CompileStatic +import io.seqera.wave.WaveDefault +/** + * Utility class for registry functions + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class RegistryUtils { + + static boolean isServerError(HttpResponse response) { + response.statusCode() in WaveDefault.HTTP_SERVER_ERRORS + } + +} diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index 2dd816d41..c9b65c714 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -23,6 +23,7 @@ import java.time.Duration import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable /** @@ -31,6 +32,7 @@ import io.micronaut.core.annotation.Nullable * @author Paolo Di Tommaso */ @Slf4j +@Requires(property = 'wave.blobCache.enabled', value = 'true') @ToString(includeNames = true, includePackage = false, excludes = 'storageSecretKey', ignoreNulls = true) @CompileStatic class BlobCacheConfig { @@ -38,11 +40,19 @@ class BlobCacheConfig { @Value('${wave.blobCache.enabled:false}') boolean enabled - @Value('${wave.blobCache.status.delay:5s}') + /** + * The time interval every when the status of the blob transfer is checked + */ + @Value('${wave.blobCache.status.delay:2s}') Duration statusDelay - @Value('${wave.blobCache.failure.duration:4s}') - Duration failureDuration + /** + * How long a failed blob should survive in the cache. Note: this should be longer than + * {@link #statusDelay} otherwise the state can be lost. + */ + Duration getFailureDuration() { + return statusDelay.multipliedBy(3) + } @Value('${wave.blobCache.timeout:10m}') Duration transferTimeout @@ -87,12 +97,9 @@ class BlobCacheConfig { @Value('${wave.blobCache.url-signature-duration:30m}') Duration urlSignatureDuration - @Value('${wave.blobCache.retryAttempts:3}') + @Value('${wave.blobCache.retry-attempts:3}') Integer retryAttempts - @Value('${wave.blobCache.deleteAfterFinished:7d}') - Duration deleteAfterFinished - @Value('${wave.blobCache.k8s.pod.delete.timeout:20s}') Duration podDeleteTimeout diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index 2ea625fde..665f8283e 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -22,7 +22,6 @@ import javax.annotation.Nullable import javax.annotation.PostConstruct import groovy.transform.CompileStatic -import groovy.transform.Memoized import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.seqera.wave.api.SubmitContainerTokenRequest @@ -76,24 +75,14 @@ class BuildConfig { @Value('${wave.build.status.duration}') Duration statusDuration - @Memoized - Duration getStatusInitialDelay() { - final d1 = defaultTimeout.toMillis() * 2.5f - final d2 = trustedTimeout.toMillis() * 1.5f - return Duration.ofMillis(Math.round(Math.max(d1,d2))) - } + @Nullable + @Value('${wave.build.failure.duration}') + Duration failureDuration - @Memoized - Duration getStatusAwaitDuration() { - final d1 = defaultTimeout.toMillis() * 2.1f - final d2 = trustedTimeout.toMillis() * 1.1f - return Duration.ofMillis(Math.round(Math.max(d1,d2))) + Duration getFailureDuration() { + return failureDuration ?: statusDelay.multipliedBy(10) } - @Value('${wave.build.cleanup}') - @Nullable - String cleanup - @Value('${wave.build.reserved-words:[]}') Set reservedWords @@ -110,6 +99,13 @@ class BuildConfig { @Value('${wave.build.force-compression:false}') Boolean forceCompression + /** + * The number of times a build job should be retries. Since failures are expected due to + * invalid Dockerfile or Conda environment, retry is disabled. + */ + @Value('${wave.build.retry-attempts:0}') + int retryAttempts + @PostConstruct private void init() { log.info("Builder config: " + @@ -124,11 +120,12 @@ class BuildConfig { "build-trusted-timeout=${trustedTimeout}; " + "status-delay=${statusDelay}; " + "status-duration=${statusDuration}; " + + "failure-duration=${getFailureDuration()}; " + "record-duration=${recordDuration}; " + - "cleanup=${cleanup}; "+ "oci-mediatypes=${ociMediatypes}; " + "compression=${compression}; " + - "force-compression=${forceCompression}; ") + "force-compression=${forceCompression}; " + + "retry-attempts=${retryAttempts}") // minimal validation if( trustedTimeout < defaultTimeout ) { log.warn "Trusted build timeout should be longer than default timeout - check configuration setting 'wave.build.trusted-timeout'" diff --git a/src/main/groovy/io/seqera/wave/configuration/HttpClientConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/HttpClientConfig.groovy index 2ec87b0a2..cc2eb8988 100644 --- a/src/main/groovy/io/seqera/wave/configuration/HttpClientConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/HttpClientConfig.groovy @@ -50,8 +50,8 @@ class HttpClientConfig implements Retryable.Config { @Value('${wave.httpclient.retry.attempts:3}') int retryAttempts - @Value('${wave.httpclient.retry.multiplier:1.0}') - float retryMultiplier + @Value('${wave.httpclient.retry.multiplier:2.0}') + double retryMultiplier @Value('${wave.httpclient.retry.jitter:0.25}') double retryJitter @@ -73,4 +73,8 @@ class HttpClientConfig implements Retryable.Config { double getJitter() { retryJitter } int getStreamThreshold() { streamThreshold } + + double getMultiplier() { + return retryMultiplier + } } diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index 6312e0184..eee8e1d62 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -21,6 +21,8 @@ package io.seqera.wave.configuration import java.nio.file.Files import java.nio.file.Path import java.time.Duration + +import io.micronaut.context.annotation.Requires import io.micronaut.core.annotation.Nullable import javax.annotation.PostConstruct @@ -37,6 +39,7 @@ import jakarta.inject.Singleton @CompileStatic @Singleton @Slf4j +@Requires(property = 'wave.scan.enabled', value = 'true') class ScanConfig { /** @@ -66,6 +69,9 @@ class ScanConfig { @Nullable private String severity + @Value('${wave.scan.retry-attempts:1}') + int retryAttempts + String getScanImage() { return scanImage } @@ -95,6 +101,6 @@ class ScanConfig { @PostConstruct private void init() { - log.debug("Scanner config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity") + log.debug("Scanner config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; retry-attempts: $retryAttempts") } } diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 441234ece..2499bf925 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -25,6 +25,7 @@ import javax.annotation.PostConstruct import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpRequest import io.micronaut.http.HttpResponse import io.micronaut.http.annotation.Controller @@ -48,6 +49,8 @@ import io.seqera.wave.exception.BadRequestException import io.seqera.wave.exception.NotFoundException import io.seqera.wave.exchange.DescribeWaveContainerResponse import io.seqera.wave.model.ContainerCoordinates +import io.seqera.wave.ratelimit.AcquireRequest +import io.seqera.wave.ratelimit.RateLimiterService import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.UserService import io.seqera.wave.service.builder.BuildRequest @@ -147,6 +150,10 @@ class ContainerController { @Inject ContainerInclusionService inclusionService + @Inject + @Nullable + RateLimiterService rateLimiterService + @PostConstruct private void init() { log.info "Wave server url: $serverUrl; allowAnonymous: $allowAnonymous; tower-endpoint-url: $towerEndpointUrl; default-build-repo: $buildConfig.defaultBuildRepository; default-cache-repo: $buildConfig.defaultCacheRepository; default-public-repo: $buildConfig.defaultPublicRepository" @@ -231,6 +238,10 @@ class ContainerController { } final ip = addressResolver.resolve(httpRequest) + // check the rate limit before continuing + if( rateLimiterService ) + rateLimiterService.acquirePull(new AcquireRequest(identity.userId as String, ip)) + // create request data final data = makeRequestData(req, identity, ip) final token = tokenService.computeToken(data) final target = targetImage(token.value, data.coordinates()) diff --git a/src/main/groovy/io/seqera/wave/controller/ErrorController.groovy b/src/main/groovy/io/seqera/wave/controller/ErrorController.groovy index 07068a7d1..bceda35d2 100644 --- a/src/main/groovy/io/seqera/wave/controller/ErrorController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ErrorController.groovy @@ -41,7 +41,7 @@ class ErrorController { @Error(global = true) HttpResponse handleException(HttpRequest request, Throwable exception) { - handler.handle(request, exception, (msg, id) -> { return new JsonError(msg) }) + handler.handle(request, exception, (String message, String code)-> new JsonError(message)) } } diff --git a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy index f187078cb..54ba69702 100644 --- a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy @@ -104,7 +104,7 @@ class RegistryProxyController { @Error HttpResponse handleError(HttpRequest request, Throwable t) { - return errorHandler.handle(request, t, (msg, code) -> new RegistryErrorResponse(code,msg) ) + return errorHandler.handle(request, t, (String msg, String code) -> new RegistryErrorResponse(code,msg) ) } @Get diff --git a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy index ea684a220..830c60eb9 100644 --- a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy @@ -37,7 +37,7 @@ import jakarta.inject.Inject */ @Slf4j @CompileStatic -@Requires(property = 'wave.scan.enabled', value = 'true') +@Requires(bean = ContainerScanService) @Controller("/") @ExecuteOn(TaskExecutors.IO) class ScanController { @@ -45,7 +45,6 @@ class ScanController { @Inject private ContainerScanService containerScanService - @Get("/v1alpha1/scans/{scanId}") HttpResponse scanImage(String scanId){ final record = containerScanService.getScanResult(scanId) diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 1a4d9ba95..6edad5dd6 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -150,20 +150,7 @@ class ViewController { final binding = new HashMap(10) try { final result = persistenceService.loadScanResult(scanId) - binding.should_refresh = !result.isCompleted() - binding.scan_id = result.id - binding.scan_exist = true - binding.scan_completed = result.isCompleted() - binding.scan_status = result.status - binding.scan_failed = result.status == ScanResult.FAILED - binding.scan_succeeded = result.status == ScanResult.SUCCEEDED - binding.build_id = result.buildId - binding.build_url = "$serverUrl/view/builds/${result.buildId}" - binding.scan_time = formatTimestamp(result.startTime) ?: '-' - binding.scan_duration = formatDuration(result.duration) ?: '-' - if ( result.vulnerabilities ) - binding.vulnerabilities = result.vulnerabilities.toSorted().reverse() - + makeScanViewBinding(result, binding) } catch (NotFoundException e){ binding.scan_exist = false @@ -177,4 +164,22 @@ class ViewController { return HttpResponse.>ok(binding) } + Map makeScanViewBinding(ScanResult result, Map binding=new HashMap(10)) { + binding.should_refresh = !result.isCompleted() + binding.scan_id = result.id + binding.scan_container_image = result.containerImage ?: '-' + binding.scan_exist = true + binding.scan_completed = result.isCompleted() + binding.scan_status = result.status + binding.scan_failed = result.status == ScanResult.FAILED + binding.scan_succeeded = result.status == ScanResult.SUCCEEDED + binding.build_id = result.buildId + binding.build_url = "$serverUrl/view/builds/${result.buildId}" + binding.scan_time = formatTimestamp(result.startTime) ?: '-' + binding.scan_duration = formatDuration(result.duration) ?: '-' + if ( result.vulnerabilities ) + binding.vulnerabilities = result.vulnerabilities.toSorted().reverse() + + return binding + } } diff --git a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy index 3ecd517c1..ba9bb553f 100644 --- a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy +++ b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy @@ -40,6 +40,7 @@ import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.storage.DigestStore import io.seqera.wave.storage.Storage +import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.RegHelper import jakarta.inject.Inject import jakarta.inject.Singleton @@ -188,10 +189,10 @@ class RegistryProxyService { String getImageDigest(BuildRequest request, boolean retryOnNotFound=false) { try { - return getImageDigest0(request, retryOnNotFound) + return getImageDigest0(request.targetImage, request.identity, retryOnNotFound) } catch(Exception e) { - log.warn "Unable to retrieve digest for image '${request.getTargetImage()}' -- cause: ${e.message}" + log.warn "Unable to retrieve digest for image '${request.targetImage}' -- cause: ${e.message}" return null } } @@ -199,10 +200,9 @@ class RegistryProxyService { static private List RETRY_ON_NOT_FOUND = HTTP_RETRYABLE_ERRORS + 404 @Cacheable(value = 'cache-registry-proxy', atomic = true) - protected String getImageDigest0(BuildRequest request, boolean retryOnNotFound) { - final image = request.targetImage + protected String getImageDigest0(String image, PlatformId identity, boolean retryOnNotFound) { final coords = ContainerCoordinates.parse(image) - final route = RoutePath.v2manifestPath(coords, request.identity) + final route = RoutePath.v2manifestPath(coords, identity) final proxyClient = client(route) .withRetryableHttpErrors(retryOnNotFound ? RETRY_ON_NOT_FOUND : HTTP_RETRYABLE_ERRORS) final resp = proxyClient.head(route.path, WaveDefault.ACCEPT_HEADERS) diff --git a/src/main/groovy/io/seqera/wave/exception/RegistryForwardException.groovy b/src/main/groovy/io/seqera/wave/exception/RegistryForwardException.groovy new file mode 100644 index 000000000..3ccfb2df0 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/exception/RegistryForwardException.groovy @@ -0,0 +1,66 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.exception + +import java.net.http.HttpResponse + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +/** + * Exception thrown when a error response is returned by a target registry + * while authenticating a container request + * + * @author Paolo Di Tommaso + */ +@Canonical +@CompileStatic +class RegistryForwardException extends WaveException implements HttpError { + + final int statusCode + final String response + final Map headers + + RegistryForwardException(String message, int status, String body, Map> headers) { + super(message) + this.statusCode = status + this.response = body + this.headers = simpleMap(headers) ?: Map.of() + } + + RegistryForwardException(String message, HttpResponse resp) { + super(message) + this.statusCode = resp.statusCode() + this.response = resp.body() + this.headers = simpleMap(resp.headers().map()) + } + + private Map simpleMap(Map> h) { + final result = new LinkedHashMap() + for( Map.Entry> it : h.entrySet()) { + result.put(it.key, it.value?.first() ?: '') + } + return result + } + + String getMessage() { + def result = super.getMessage() + result += " - status=${statusCode}; response=${response}; headers=${headers}" + return result + } +} diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryUnauthorizedAccessException.groovy b/src/main/groovy/io/seqera/wave/exception/RegistryUnauthorizedAccessException.groovy similarity index 95% rename from src/main/groovy/io/seqera/wave/auth/RegistryUnauthorizedAccessException.groovy rename to src/main/groovy/io/seqera/wave/exception/RegistryUnauthorizedAccessException.groovy index ebc287701..c8873a55b 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryUnauthorizedAccessException.groovy +++ b/src/main/groovy/io/seqera/wave/exception/RegistryUnauthorizedAccessException.groovy @@ -16,10 +16,9 @@ * along with this program. If not, see . */ -package io.seqera.wave.auth +package io.seqera.wave.exception import groovy.transform.CompileStatic -import io.seqera.wave.exception.WaveException /** * Exception throw when the registry authorization failed diff --git a/src/main/groovy/io/seqera/wave/exchange/RegistryErrorResponse.groovy b/src/main/groovy/io/seqera/wave/exchange/RegistryErrorResponse.groovy index 51e1817d1..af63bbb00 100644 --- a/src/main/groovy/io/seqera/wave/exchange/RegistryErrorResponse.groovy +++ b/src/main/groovy/io/seqera/wave/exchange/RegistryErrorResponse.groovy @@ -20,8 +20,10 @@ package io.seqera.wave.exchange import groovy.transform.Canonical import groovy.transform.CompileStatic +import groovy.transform.ToString import io.micronaut.http.MediaType import io.micronaut.http.annotation.Produces +import io.seqera.wave.encoder.MoshiEncodeStrategy /** * Model a docker registry error response @@ -30,27 +32,56 @@ import io.micronaut.http.annotation.Produces */ @CompileStatic @Produces(MediaType.APPLICATION_JSON) +@ToString(includePackage = false, includeNames = true) class RegistryErrorResponse { @Canonical static class RegistryError { + /** + * The error code as defined in the registry API. see + * https://distribution.github.io/distribution/spec/api/#errors-2 + */ final String code + + /** + * The error message + */ final String message } - List errors = new ArrayList<>(10) + final List errors = new ArrayList<>(10) /** * Do not remove -- required for object de-serialisation */ - RegistryErrorResponse() { } - - RegistryErrorResponse(List errors) { - this.errors = errors - } + protected RegistryErrorResponse() { } + /** + * Creates a {@link RegistryErrorResponse} object with the specified error + * code and message. + * + * @param code The error code as string. + * @param message The error message as string + */ RegistryErrorResponse(String code, String message) { errors.add( new RegistryError(code, message) ) } + /** + * Parse a JSON error response into a {@link RegistryErrorResponse}. + * + * @param json + * The JSON error response as a string + * @return + * The corresponding {@link RegistryErrorResponse} object + */ + static RegistryErrorResponse parse(String json) throws IllegalArgumentException { + try { + final decoder = new MoshiEncodeStrategy() {} + return decoder.decode(json) + } + catch (Throwable t) { + throw new IllegalArgumentException("Unable to parse registry error response - offending value: $json", t) + } + } } diff --git a/src/main/groovy/io/seqera/wave/memstore/range/impl/RedisRangeProvider.groovy b/src/main/groovy/io/seqera/wave/memstore/range/impl/RedisRangeProvider.groovy index 20015adeb..22a5ede1a 100644 --- a/src/main/groovy/io/seqera/wave/memstore/range/impl/RedisRangeProvider.groovy +++ b/src/main/groovy/io/seqera/wave/memstore/range/impl/RedisRangeProvider.groovy @@ -47,15 +47,28 @@ class RedisRangeProvider implements RangeProvider { } } + private final static String SCRIPT = ''' + local elements = redis.call('ZRANGEBYSCORE', KEYS[1], ARGV[1], ARGV[2], 'LIMIT', ARGV[3], ARGV[4]) + if #elements > 0 then + redis.call('ZREM', KEYS[1], unpack(elements)) + end + return elements + ''' + @Override List getRange(String key, double min, double max, int count, boolean remove) { try(Jedis conn = pool.getResource()) { - List found = conn.zrangeByScoreWithScores(key, min, max, 0, count) - final result = new ArrayList(found.size()) - for( Tuple it : found ) { - result.add(it.element) - if( remove ) - conn.zrem(key, it.element) + final result = new ArrayList() + if( remove ) { + final entries = conn.eval(SCRIPT, 1, key, min.toString(), max.toString(), '0', count.toString()) + if( entries instanceof List ) + result.addAll((List) entries) + } + else { + List found = conn.zrangeByScoreWithScores(key, min, max, 0, count) + for( Tuple it : found ) { + result.add(it.element) + } } return result } diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy index f58c3a688..8f1664a72 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy @@ -30,10 +30,11 @@ import io.micronaut.http.HttpMethod import io.micronaut.http.MutableHttpRequest import io.micronaut.http.server.exceptions.InternalServerException import io.micronaut.reactor.http.client.ReactorStreamingHttpClient +import io.seqera.wave.exception.RegistryForwardException import io.seqera.wave.auth.RegistryAuthService import io.seqera.wave.auth.RegistryCredentials import io.seqera.wave.auth.RegistryInfo -import io.seqera.wave.auth.RegistryUnauthorizedAccessException +import io.seqera.wave.exception.RegistryUnauthorizedAccessException import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.core.ContainerPath import io.seqera.wave.util.RegHelper @@ -239,7 +240,7 @@ class ProxyClient { // just re-throw it so that it's managed by the retry policy throw e } - catch (RegistryUnauthorizedAccessException e) { + catch (RegistryForwardException | RegistryUnauthorizedAccessException e) { // just re-throw it because it's a known error condition throw e } diff --git a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy index 8511cc4ce..e00a28440 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy @@ -22,18 +22,25 @@ import java.time.Instant import groovy.transform.Canonical import groovy.transform.CompileStatic -import groovy.transform.ToString import groovy.util.logging.Slf4j +import io.seqera.wave.service.job.JobRecord + /** * Model a blob cache metadata entry * * @author Paolo Di Tommaso */ @Slf4j -@ToString(includePackage = false, includeNames = true, excludes = ['headers','logs']) @Canonical @CompileStatic -class BlobCacheInfo { +class BlobCacheInfo implements JobRecord { + + enum State { CREATED, CACHED, COMPLETED, ERRORED, UNKNOWN } + + /** + * The Blob state + */ + final State state /** * The HTTP location from the where the cached container blob can be retrieved @@ -95,6 +102,7 @@ class BlobCacheInfo { locationUri && exitStatus==0 } + @Override boolean done() { locationUri && completionTime!=null } @@ -113,7 +121,7 @@ class BlobCacheInfo { final type = headerString0(response, 'Content-Type') final cache = headerString0(response, 'Cache-Control') final creationTime = Instant.now() - return new BlobCacheInfo(locationUri, objectUri, headers0, length, type, cache, creationTime, null, null, null) + return new BlobCacheInfo(State.CREATED, locationUri, objectUri, headers0, length, type, cache, creationTime, null, null, null) } static String headerString0(Map> headers, String name) { @@ -130,8 +138,28 @@ class BlobCacheInfo { } } + @Override + String toString() { + if( state==State.UNKNOWN ) { + return "BlobCacheInfo(UNKNOWN)" + } + + return "BlobCacheInfo(" + + "state=" + state + + ", locationUri='" + locationUri + "'" + + ", objectUri='" + objectUri + "'" + + ", contentLength=" + contentLength + + ", contentType='" + contentType + "'" + + ", cacheControl='" + cacheControl + "'" + + ", creationTime=" + creationTime + + ", completionTime=" + completionTime + + ", exitStatus=" + exitStatus + + ')' + } + BlobCacheInfo cached() { new BlobCacheInfo( + State.CACHED, locationUri, objectUri, headers, @@ -147,6 +175,7 @@ class BlobCacheInfo { BlobCacheInfo completed(int status, String logs) { new BlobCacheInfo( + State.COMPLETED, locationUri, objectUri, headers, @@ -160,8 +189,9 @@ class BlobCacheInfo { ) } - BlobCacheInfo failed(String logs) { + BlobCacheInfo errored(String logs) { new BlobCacheInfo( + State.ERRORED, locationUri, objectUri, headers, @@ -177,6 +207,7 @@ class BlobCacheInfo { BlobCacheInfo withLocation(String location) { new BlobCacheInfo( + state, location, objectUri, headers, @@ -191,7 +222,7 @@ class BlobCacheInfo { } static BlobCacheInfo unknown(String logs) { - new BlobCacheInfo(null, null, null, null, null, null, Instant.ofEpochMilli(0), Instant.ofEpochMilli(0), null, logs) { + new BlobCacheInfo(State.UNKNOWN, null, null, null, null, null, null, Instant.ofEpochMilli(0), Instant.ofEpochMilli(0), null, logs) { @Override BlobCacheInfo withLocation(String uri) { // prevent the change of location for unknown status @@ -200,4 +231,5 @@ class BlobCacheInfo { } } + } diff --git a/src/main/groovy/io/seqera/wave/service/blob/BlobStore.groovy b/src/main/groovy/io/seqera/wave/service/blob/BlobStore.groovy index ce431312b..b5ed02c8e 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/BlobStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/BlobStore.groovy @@ -55,16 +55,6 @@ interface BlobStore { */ void storeBlob(String key, BlobCacheInfo info) - /** - * Store the blob cache info object with the specified key. The object is evicted after the specified - * duration is reached - * - * @param key The unique to be used to store the blob cache info - * @param info The {@link BlobCacheInfo} object modelling the container blob information - * @param ttl How long the object is allowed to stay in the cache - */ - void storeBlob(String key, BlobCacheInfo info, Duration ttl) - /** * Store a blob cache location only if the specified key does not exit * diff --git a/src/main/groovy/io/seqera/wave/service/job/JobStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/TransferStrategy.groovy similarity index 87% rename from src/main/groovy/io/seqera/wave/service/job/JobStrategy.groovy rename to src/main/groovy/io/seqera/wave/service/blob/TransferStrategy.groovy index 618c54fd4..d4db35320 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/TransferStrategy.groovy @@ -16,18 +16,14 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.job +package io.seqera.wave.service.blob /** * Defines the contract to transfer a layer blob into a remote object storage * * @author Paolo Di Tommaso */ -interface JobStrategy { +interface TransferStrategy { void launchJob(String jobName, List command) - JobState status(JobId jobId) - - void cleanup(JobId jobId, Integer exitStatus) - } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index a6c8fbeda..eed90d17d 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -20,7 +20,6 @@ package io.seqera.wave.service.blob.impl import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse -import java.time.Duration import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -36,8 +35,8 @@ import io.seqera.wave.service.blob.BlobCacheService import io.seqera.wave.service.blob.BlobSigningService import io.seqera.wave.service.blob.BlobStore import io.seqera.wave.service.job.JobHandler -import io.seqera.wave.service.job.JobId import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState import io.seqera.wave.util.Escape import io.seqera.wave.util.Retryable @@ -57,7 +56,7 @@ import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS @Singleton @Requires(property = 'wave.blobCache.enabled', value = 'true') @CompileStatic -class BlobCacheServiceImpl implements BlobCacheService, JobHandler { +class BlobCacheServiceImpl implements BlobCacheService, JobHandler { @Value('${wave.debug:false}') private Boolean debug @@ -189,9 +188,9 @@ class BlobCacheServiceImpl implements BlobCacheService, JobHandler { } catch (Throwable t) { log.warn "== Blob cache failed for object '${blob.objectUri}' - cause: ${t.message}", t - final result = blob.failed(t.message) + final result = blob.errored(t.message) // update the blob status - blobStore.storeBlob(blob.id(), result, blobConfig.failureDuration) + blobStore.storeBlob(blob.id(), result) } } @@ -269,63 +268,31 @@ class BlobCacheServiceImpl implements BlobCacheService, JobHandler { // ============ handles transfer job events ============ @Override - Duration jobMaxDuration(JobId job) { - return blobConfig.transferTimeout + BlobCacheInfo getJobRecord(JobSpec job) { + blobStore.getBlob(job.recordId) } @Override - void onJobCompletion(JobId job, JobState state) { - final blob = blobStore.getBlob(job.id) - if( !blob ) { - log.error "== Blob cache entry unknown for job=$job [1]" - return - } - if( blob.done() ) { - log.warn "== Blob cache entry already marked as completed for job=$job [1] - entry=$blob; state=$state" - return - } - // use a short time-to-live for failed downloads - // this is needed to allow re-try caching of failure transfers - final ttl = state.succeeded() - ? blobConfig.statusDuration - : blobConfig.failureDuration + void onJobCompletion(JobSpec job, BlobCacheInfo blob, JobState state) { // update the blob status final result = state.succeeded() ? blob.completed(state.exitCode, state.stdout) - : blob.failed(state.stdout) - blobStore.storeBlob(blob.id(), result, ttl) - log.debug "== Blob cache completed for object '${blob.objectUri}'; id=${blob.objectUri}; status=${result.exitStatus}; duration=${result.duration()}" + : blob.errored(state.stdout) + blobStore.storeBlob(blob.id(), result) + log.debug "== Blob cache completed for object '${blob.objectUri}'; operation=${job.operationName}; status=${result.exitStatus}; duration=${result.duration()}" } @Override - void onJobException(JobId job, Throwable error) { - final blob = blobStore.getBlob(job.id) - if( !blob ) { - log.error "== Blob cache entry unknown for job=$job [2]" - return - } - if( blob.done() ) { - log.warn "== Blob cache entry already marked as completed for job=$job [2] - entry=$blob; error=${error.message}" - return - } - final result = blob.failed("Unexpected error caching blob '${blob.locationUri}' - job name '${job.schedulerId}'") - log.error("== Blob cache exception for object '${blob.objectUri}'; job name=${job.schedulerId}; cause=${error.message}", error) - blobStore.storeBlob(job.id, result, blobConfig.failureDuration) + void onJobException(JobSpec job, BlobCacheInfo blob, Throwable error) { + final result = blob.errored("Unexpected error caching blob '${blob.locationUri}' - operation '${job.operationName}'") + log.error("== Blob cache exception for object '${blob.objectUri}'; operation=${job.operationName}; cause=${error.message}", error) + blobStore.storeBlob(blob.id(), result) } @Override - void onJobTimeout(JobId job) { - final blob = blobStore.getBlob(job.id) - if( !blob ) { - log.error "== Blob cache entry unknown for job=$job [3]" - return - } - if( blob.done() ) { - log.warn "== Blob cache entry already marked as completed for job=$job [3] - entry=$blob; duration=${blob.duration()}" - return - } - final result = blob.failed("Blob cache transfer timed out ${blob.objectUri}") - log.warn "== Blob cache completed for object '${blob.objectUri}'; job name=${job.schedulerId}; duration=${result.duration()}" - blobStore.storeBlob(blob.id(), result, blobConfig.failureDuration) + void onJobTimeout(JobSpec job, BlobCacheInfo blob) { + final result = blob.errored("Blob cache transfer timed out ${blob.objectUri}") + log.warn "== Blob cache timed out for object '${blob.objectUri}'; operation=${job.operationName}; duration=${result.duration()}" + blobStore.storeBlob(blob.id(), result) } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheStore.groovy index 835dc0288..a75d148c6 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheStore.groovy @@ -80,11 +80,10 @@ class BlobCacheStore extends AbstractCacheStore implements BlobSt @Override void storeBlob(String key, BlobCacheInfo info) { - put(key, info) - } - - @Override - void storeBlob(String key, BlobCacheInfo info, Duration ttl) { + final ttl = info.state == BlobCacheInfo.State.ERRORED + ? blobConfig.failureDuration + : blobConfig.statusDuration put(key, info, ttl) } + } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy index 0f97ebdc1..eb7275014 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy @@ -18,19 +18,16 @@ package io.seqera.wave.service.blob.impl -import groovy.transform.Canonical + import groovy.transform.CompileStatic -import groovy.transform.ToString import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.service.job.JobId -import io.seqera.wave.service.job.JobState -import io.seqera.wave.service.job.JobStrategy +import io.seqera.wave.service.blob.TransferStrategy import jakarta.inject.Inject import jakarta.inject.Singleton /** - * Implements {@link JobStrategy} that runs s5cmd using a docker + * Implements {@link TransferStrategy} that runs s5cmd using a docker * container. Meant for development purposes * * @author Paolo Di Tommaso @@ -40,7 +37,7 @@ import jakarta.inject.Singleton @Singleton @Requires(missingProperty = 'wave.build.k8s') @Requires(property = 'wave.blobCache.enabled', value = 'true') -class DockerTransferStrategy implements JobStrategy { +class DockerTransferStrategy implements TransferStrategy { @Inject private BlobCacheConfig blobConfig @@ -48,19 +45,20 @@ class DockerTransferStrategy implements JobStrategy { @Override void launchJob(String jobName, List command) { // create a unique name for the container - createProcess(command, jobName, blobConfig.transferTimeout.toSeconds()) - .start() + final process = createProcess(command, jobName) .start() + if( process.waitFor()!=0 ) { + throw new IllegalStateException("Unable to launch transfer container - exitCode=${process.exitValue()}; output=${process.text}") + } } - protected ProcessBuilder createProcess(List command, String name, long timeoutSecs) { + protected ProcessBuilder createProcess(List command, String name) { // compose the docker command final cli = new ArrayList(10) cli.add('docker') cli.add('run') + cli.add('--detach') cli.add('--name') cli.add(name) - cli.add('--stop-timeout') - cli.add(String.valueOf(timeoutSecs)) cli.add('-e') cli.add('AWS_ACCESS_KEY_ID') cli.add('-e') @@ -77,81 +75,5 @@ class DockerTransferStrategy implements JobStrategy { return builder } - @Override - JobState status(JobId job) { - final state = getDockerContainerState(job.schedulerId) - log.trace "Docker transfer status name=$job.schedulerId; state=$state" - - if (state.status == 'running') { - return JobState.running() - } - else if (state.status == 'exited') { - final logs = getDockerContainerLogs(job.schedulerId) - return JobState.completed(state.exitCode, logs) - } - else if (state.status == 'created' || state.status == 'paused') { - return JobState.pending() - } - else { - final logs = getDockerContainerLogs(job.schedulerId) - return JobState.unknown(logs) - } - } - - @Override - void cleanup(JobId jobId, Integer exitStatus) { - final cli = new ArrayList() - cli.add('docker') - cli.add('rm') - cli.add(jobId.schedulerId) - - final builder = new ProcessBuilder(cli) - builder.redirectErrorStream(true) - final process = builder.start() - process.waitFor() - } - - @ToString(includePackage = false, includeNames = true) - @Canonical - static class State { - String status - Integer exitCode - - static State parse(String result) { - final ret = result.tokenize(',') - final status = ret[0] - final exit = ret[1] ? Integer.valueOf(ret[1]) : null - new State(status,exit) - } - } - - private static State getDockerContainerState(String containerName) { - final cli = new ArrayList() - cli.add('docker') - cli.add('inspect') - cli.add('--format') - cli.add('{{.State.Status}},{{.State.ExitCode}}') - cli.add(containerName) - - final builder = new ProcessBuilder(cli) - builder.redirectErrorStream(true) - final process = builder.start() - process.waitFor() - final result = process.inputStream.text.trim() - return State.parse(result) - } - - private static String getDockerContainerLogs(String containerName) { - final cli = new ArrayList() - cli.add('docker') - cli.add('logs') - cli.add(containerName) - - final builder = new ProcessBuilder(cli) - builder.redirectErrorStream(true) - final process = builder.start() - process.waitFor() - process.inputStream.text - } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy index 3fbe68112..55262ed62 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy @@ -18,25 +18,16 @@ package io.seqera.wave.service.blob.impl -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires -import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.service.cleanup.CleanupStrategy -import io.seqera.wave.service.job.JobId -import io.seqera.wave.service.job.JobState -import io.seqera.wave.service.job.JobState.Status -import io.seqera.wave.service.job.JobStrategy +import io.seqera.wave.service.blob.TransferStrategy import io.seqera.wave.service.k8s.K8sService -import io.seqera.wave.service.k8s.K8sService.JobStatus import jakarta.inject.Inject -import jakarta.inject.Named /** - * Implements {@link JobStrategy} that runs s5cmd using a + * Implements {@link TransferStrategy} that runs s5cmd using a * Kubernetes job * * @author Paolo Di Tommaso @@ -45,7 +36,7 @@ import jakarta.inject.Named @CompileStatic @Requires(property = 'wave.build.k8s') @Requires(property = 'wave.blobCache.enabled', value = 'true') -class KubeTransferStrategy implements JobStrategy { +class KubeTransferStrategy implements TransferStrategy { @Inject private BlobCacheConfig blobConfig @@ -53,67 +44,10 @@ class KubeTransferStrategy implements JobStrategy { @Inject private K8sService k8sService - @Inject - private CleanupStrategy cleanup - - @Inject - @Named(TaskExecutors.IO) - private ExecutorService executor - @Override void launchJob(String jobName, List command) { // run the transfer job - k8sService.launchJob(jobName, blobConfig.s5Image, command, blobConfig) - } - - @Override - void cleanup(JobId job, Integer exitStatus) { - if( cleanup.shouldCleanup(exitStatus) ) { - CompletableFuture.supplyAsync (() -> k8sService.deleteJob(job.schedulerId), executor) - } + k8sService.launchTransferJob(jobName, blobConfig.s5Image, command, blobConfig) } - @Override - JobState status(JobId job) { - final status = k8sService.getJobStatus(job.schedulerId) - if( !status || !status.completed() ) { - return new JobState(mapToStatus(status)) - } - - // Find the latest created pod among the pods associated with the job - final pod = k8sService.getLatestPodForJob(job.schedulerId) - if( !pod ) - throw new IllegalStateException("Missing carried pod for job: ${job.schedulerId}") - - // determine exit code and logs - final exitCode = pod - .status - ?.containerStatuses - ?.first() - ?.state - ?.terminated - ?.exitCode - final stdout = k8sService.logsPod(pod) - return new JobState(mapToStatus(status), exitCode, stdout) - } - - /** - * Map Kubernetes job status to Transfer status - * @param jobStatus - * @return - */ - static Status mapToStatus(JobStatus jobStatus) { - switch (jobStatus) { - case JobStatus.Pending: - return Status.PENDING - case JobStatus.Running: - return Status.RUNNING - case JobStatus.Succeeded: - return Status.SUCCEEDED - case JobStatus.Failed: - return Status.FAILED - default: - return Status.UNKNOWN - } - } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy index edd7c6583..cdd914598 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy @@ -27,34 +27,33 @@ import groovy.util.logging.Slf4j import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.exception.BuildTimeoutException import io.seqera.wave.service.cache.AbstractCacheStore import io.seqera.wave.service.cache.impl.CacheProvider import jakarta.inject.Named import jakarta.inject.Singleton /** - * Implements Cache store for {@link BuildResult} + * Implements Cache store for {@link BuildStoreEntry} * * @author Paolo Di Tommaso */ @Slf4j @Singleton @CompileStatic -class BuildCacheStore extends AbstractCacheStore implements BuildStore { +class BuildCacheStore extends AbstractCacheStore implements BuildStore { private BuildConfig buildConfig private ExecutorService ioExecutor BuildCacheStore(CacheProvider provider, BuildConfig buildConfig, @Named(TaskExecutors.IO) ExecutorService ioExecutor) { - super(provider, new MoshiEncodeStrategy() {}) + super(provider, new MoshiEncodeStrategy() {}) this.buildConfig = buildConfig this.ioExecutor = ioExecutor } @Override protected String getPrefix() { - return 'wave-build/v1:' + return 'wave-build/v2:' } @Override @@ -63,26 +62,28 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt } @Override - BuildResult getBuild(String imageName) { + BuildStoreEntry getBuild(String imageName) { return get(imageName) } @Override - void storeBuild(String imageName, BuildResult result) { - put(imageName, result) + BuildResult getBuildResult(String imageName) { + return getBuild(imageName)?.getResult() } @Override - void storeBuild(String imageName, BuildResult result, Duration ttl) { + void storeBuild(String imageName, BuildStoreEntry buildStoreEntry) { + put(imageName, buildStoreEntry) + } + + @Override + void storeBuild(String imageName, BuildStoreEntry result, Duration ttl) { put(imageName, result, ttl) } @Override - boolean storeIfAbsent(String imageName, BuildResult build) { - // store up 2.5 time the build timeout to prevent a missed cache - // update on job termination remains too long in the store - // note: this should be longer than the max await time used in the Waiter#awaitCompletion method - return putIfAbsent(imageName, build, buildConfig.statusInitialDelay) + boolean storeIfAbsent(String imageName, BuildStoreEntry build) { + return putIfAbsent(imageName, build, buildConfig.statusDuration) } @Override @@ -92,10 +93,10 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt @Override CompletableFuture awaitBuild(String imageName) { - final result = getBuild(imageName) + final result = getBuildResult(imageName) if( !result ) return null - return CompletableFuture.supplyAsync(() -> Waiter.awaitCompletion(this,imageName,result), ioExecutor) + return CompletableFuture.supplyAsync(() -> Waiter.awaitCompletion(this, imageName, result), ioExecutor) } /** @@ -105,11 +106,6 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt static BuildResult awaitCompletion(BuildCacheStore store, String imageName, BuildResult current) { final await = store.buildConfig.statusDelay - final beg = System.currentTimeMillis() - // await nearly double of the build timeout time because the build job - // can require additional time, other than the build time, to be scheduled - // note: see also #storeIfAbsent method - final max = store.buildConfig.statusAwaitDuration.toMillis() while( true ) { if( current==null ) { return BuildResult.unknown() @@ -119,14 +115,10 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt if( current.done() ) { return current } - // check if it's timed out - final delta = System.currentTimeMillis()-beg - if( delta > max ) - throw new BuildTimeoutException("Build of container '$imageName' timed out") // sleep a bit Thread.sleep(await.toMillis()) // fetch the build status again - current = store.getBuild(imageName) + current = store.getBuildResult(imageName) } } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy index f6e246a06..b172d45c2 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy @@ -31,38 +31,50 @@ import groovy.transform.CompileStatic interface BuildStore { /** - * Retrieve a container image {@link BuildResult} + * Retrieve the build entry {@link BuildStoreEntry} for a given container image name * - * @param imageName The container image name - * @return The corresponding {@link BuildResult} or {@code null} otherwise + * @param imageName + * The container image name + * @return + * The corresponding {@link BuildStoreEntry} or {@code null} otherwise + */ + BuildStoreEntry getBuild(String imageName) + + /** + * Retrieve the build entry {@link BuildResult} for a given container image name + * + * @param imageName + * The container image name + * @return + * The corresponding {@link BuildStoreEntry} or {@code null} otherwise */ - BuildResult getBuild(String imageName) + BuildResult getBuildResult(String imageName) /** * Store a container image build request * * @param imageName The container image name - * @param request The {@link BuildResult} object associated to the image name + * @param request The {@link BuildStoreEntry} object associated to the image name */ - void storeBuild(String imageName, BuildResult result) + void storeBuild(String imageName, BuildStoreEntry result) /** * Store a container image build request using the specified time-to-live duration * * @param imageName The container image name - * @param result The {@link BuildResult} object associated to the image name + * @param result The {@link BuildStoreEntry} object associated to the image name * @param ttl The {@link Duration} after which the entry is expired */ - void storeBuild(String imageName, BuildResult result, Duration ttl) + void storeBuild(String imageName, BuildStoreEntry result, Duration ttl) /** * Store a build result only if the specified key does not exit * * @param imageName The container image unique key - * @param build The {@link BuildResult} desired status to be stored - * @return {@code true} if the {@link BuildResult} was stored, {@code false} otherwise + * @param build The {@link BuildStoreEntry} desired status to be stored + * @return {@code true} if the {@link BuildStoreEntry} was stored, {@code false} otherwise */ - boolean storeIfAbsent(String imageName, BuildResult build) + boolean storeIfAbsent(String imageName, BuildStoreEntry build) /** * Remove the build status for the given image name diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStoreEntry.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStoreEntry.groovy new file mode 100644 index 000000000..b95d8658d --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStoreEntry.groovy @@ -0,0 +1,55 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import io.seqera.wave.service.job.JobRecord + +/** + * Class to store build request and result in cache + * + * @author Munish Chouhan + */ +@ToString(includePackage = false, includeNames = true) +@EqualsAndHashCode +@CompileStatic +class BuildStoreEntry implements JobRecord { + + final BuildRequest request + + final BuildResult result + + @Override + boolean done() { + return result.done() + } + + protected BuildStoreEntry() {} + + BuildStoreEntry(BuildRequest request, BuildResult result) { + this.request = request + this.result = result + } + + BuildStoreEntry withResult(BuildResult result) { + new BuildStoreEntry(request, result) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 723d2611a..fb5a25cb4 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -35,14 +35,10 @@ abstract class BuildStrategy { @Inject private BuildConfig buildConfig - abstract BuildResult build(BuildRequest req) + abstract void build(String jobName, BuildRequest req) static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh' - void cleanup(BuildRequest req) { - req.workDir?.deleteDir() - } - List launchCmd(BuildRequest req) { if(req.formatDocker()) { dockerLaunchCmd(req) diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy index dfb8c3a74..f63d3ec1d 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy @@ -55,6 +55,10 @@ interface ContainerBuildService { */ CompletableFuture buildResult(String targetImage) + default CompletableFuture buildResult(BuildRequest request) { + return buildResult(request.targetImage) + } + /** * Get a completable future that holds the build result * diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 7e57ae642..cc57d4e57 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -36,11 +36,15 @@ import io.seqera.wave.auth.RegistryLookupService import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.configuration.SpackConfig +import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.exception.HttpServerRetryableErrorException import io.seqera.wave.ratelimit.AcquireRequest import io.seqera.wave.ratelimit.RateLimiterService import io.seqera.wave.service.builder.store.BuildRecordStore -import io.seqera.wave.service.cleanup.CleanupStrategy +import io.seqera.wave.service.job.JobHandler +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState import io.seqera.wave.service.metric.MetricsService import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord @@ -55,7 +59,6 @@ import jakarta.inject.Named import jakarta.inject.Singleton import static io.seqera.wave.util.RegHelper.layerDir import static io.seqera.wave.util.RegHelper.layerName -import static io.seqera.wave.util.StringUtils.indent import static java.nio.file.StandardOpenOption.CREATE import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING import static java.nio.file.StandardOpenOption.WRITE @@ -66,8 +69,9 @@ import static java.nio.file.StandardOpenOption.WRITE */ @Slf4j @Singleton +@Named('Build') @CompileStatic -class ContainerBuildServiceImpl implements ContainerBuildService { +class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler { @Inject private BuildConfig buildConfig @@ -89,7 +93,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { private RegistryCredentialsProvider credentialsProvider @Inject - private BuildStrategy buildStrategy + private JobService jobService @Inject @Nullable @@ -101,9 +105,6 @@ class ContainerBuildServiceImpl implements ContainerBuildService { @Inject private HttpClientConfig httpClientConfig - @Inject - private CleanupStrategy cleanup - @Inject private StreamService streamService @@ -111,13 +112,16 @@ class ContainerBuildServiceImpl implements ContainerBuildService { private BuildCounterStore buildCounter @Inject - PersistenceService persistenceService + private PersistenceService persistenceService @Inject private MetricsService metricsService @Inject - BuildRecordStore buildRecordStore + private BuildRecordStore buildRecordStore + + @Inject + private RegistryProxyService proxyService /** * Build a container image for the given {@link BuildRequest} @@ -169,9 +173,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { } } - protected BuildResult launch(BuildRequest req) { - // launch an external process to build the container - BuildResult resp=null + protected void launch(BuildRequest req) { try { // create the workdir path Files.createDirectories(req.workDir) @@ -200,34 +202,17 @@ class ContainerBuildServiceImpl implements ContainerBuildService { if( req.containerConfig ) { saveLayersToContext(req, context) } - resp = buildStrategy.build(req) - def msg = "== Build request ${req.buildId} completed with status=$resp.exitStatus" - if( log.isTraceEnabled() ) - msg += "; stdout: (see below)\n${indent(resp.logs)}" - log.info(msg) - return resp + // launch the container build + jobService.launchBuild(req) } catch (Throwable e) { - log.error "== Ouch! Unable to build container req=$req", e - return resp = BuildResult.failed(req.buildId, e.message, req.startTime) - } - finally { - // use a short time-to-live for failed build - // this is needed to allow re-try builds failed for - // temporary error conditions e.g. expired credentials - final ttl = resp.failed() - ? buildConfig.statusDelay.multipliedBy(10) - : buildConfig.statusDuration - // update build status store - buildStore.storeBuild(req.targetImage, resp, ttl) - // cleanup build context - if( cleanup.shouldCleanup(resp) ) - buildStrategy.cleanup(req) + log.error "== Container build unexpected exception: ${e.message} - request=$req", e + final result = BuildResult.failed(req.buildId, e.message, req.startTime) + buildStore.storeBuild(req.targetImage, new BuildStoreEntry(req, result), buildConfig.failureDuration) } } - - protected CompletableFuture launchAsync(BuildRequest request) { + protected void launchAsync(BuildRequest request) { // check the build rate limit try { if( rateLimiterService ) @@ -246,12 +231,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { // launch the build async CompletableFuture - .supplyAsync(() -> launch(request), executor) - .thenApply((result) -> { notifyCompletion(request,result); return result }) - } - - protected notifyCompletion(BuildRequest request, BuildResult result) { - eventPublisher.publishEvent(new BuildEvent(request, result)) + .runAsync(() -> launch(request), executor) } protected BuildTrack checkOrSubmit(BuildRequest request) { @@ -261,17 +241,17 @@ class ContainerBuildServiceImpl implements ContainerBuildService { // try to store a new build status for the given target image // this returns true if and only if such container image was not set yet final ret1 = BuildResult.create(request) - if( buildStore.storeIfAbsent(request.targetImage, ret1) ) { + if( buildStore.storeIfAbsent(request.targetImage, new BuildStoreEntry(request, ret1)) ) { // go ahead - log.info "== Submit build request: $request" + log.info "== Container build submitted - request=$request" launchAsync(request) return new BuildTrack(ret1.id, request.targetImage, false) } // since it was unable to initialise the build result status // this means the build status already exists, retrieve it - final ret2 = buildStore.getBuild(request.targetImage) + final ret2 = buildStore.getBuildResult(request.targetImage) if( ret2 ) { - log.info "== Hit build cache for request: $request" + log.info "== Container build hit cache - request=$request" // note: mark as cached only if the build result is 'done' // if the build is still in progress it should be marked as not cached // so that the client will wait for the container completion @@ -346,6 +326,54 @@ class ContainerBuildServiceImpl implements ContainerBuildService { .onRetry((event)-> log.warn("$message - event: $event")) } + // ************************************************************** + // ** build job handle implementation + // ************************************************************** + + @Override + BuildStoreEntry getJobRecord(JobSpec job) { + buildStore.getBuild(job.recordId) + } + + @Override + void onJobCompletion(JobSpec job, BuildStoreEntry build, JobState state) { + final buildId = build.request.buildId + final digest = state.succeeded() + ? proxyService.getImageDigest(build.request, true) + : null + // use a short time-to-live for failed build + // this is needed to allow re-try builds failed for + // temporary error conditions e.g. expired credentials + final ttl = state.succeeded() + ? buildConfig.statusDuration + : buildConfig.failureDuration + // update build status store + final exit = state.exitCode!=null ? state.exitCode : -1 + final result = state.completed() + ? BuildResult.completed(buildId, exit, state.stdout, job.creationTime, digest) + : BuildResult.failed(buildId, state.stdout, job.creationTime) + buildStore.storeBuild(job.recordId, build.withResult(result), ttl) + eventPublisher.publishEvent(new BuildEvent(build.request, result)) + log.info "== Container build completed '${build.request.targetImage}' - operation=${job.operationName}; exit=${exit}; status=${state.status}; duration=${result.duration}" + } + + @Override + void onJobException(JobSpec job, BuildStoreEntry build, Throwable error) { + final result= BuildResult.failed(build.request.buildId, error.message, job.creationTime) + buildStore.storeBuild(job.recordId, build.withResult(result), buildConfig.failureDuration) + eventPublisher.publishEvent(new BuildEvent(build.request, result)) + log.error("== Container build exception '${build.request.targetImage}' - operation=${job.operationName}; cause=${error.message}", error) + } + + @Override + void onJobTimeout(JobSpec job, BuildStoreEntry build) { + final buildId = build.request.buildId + final result= BuildResult.failed(buildId, "Container image build timed out '${build.request.targetImage}'", job.creationTime) + buildStore.storeBuild(job.recordId, build.withResult(result), buildConfig.failureDuration) + eventPublisher.publishEvent(new BuildEvent(build.request, result)) + log.warn "== Container build time out '${build.request.targetImage}'; operation=${job.operationName}; duration=${result.duration}" + } + // ************************************************************** // ** build record implementation // ************************************************************** diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 966f3e2aa..82ee363b9 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -20,7 +20,6 @@ package io.seqera.wave.service.builder import java.nio.file.Files import java.nio.file.Path -import java.util.concurrent.TimeUnit import groovy.json.JsonOutput import groovy.transform.CompileStatic @@ -61,7 +60,7 @@ class DockerBuildStrategy extends BuildStrategy { RegistryProxyService proxyService @Override - BuildResult build(BuildRequest req) { + void build(String jobName, BuildRequest req) { Path configFile = null // save docker config for creds @@ -80,7 +79,7 @@ class DockerBuildStrategy extends BuildStrategy { } // command the docker build command - final buildCmd= buildCmd(req, configFile) + final buildCmd= buildCmd(jobName, req, configFile) log.debug "Build run command: ${buildCmd.join(' ')}" // save docker cli for debugging purpose if( debug ) { @@ -89,39 +88,33 @@ class DockerBuildStrategy extends BuildStrategy { CREATE, WRITE, TRUNCATE_EXISTING) } - final proc = new ProcessBuilder() - .command(buildCmd) - .directory(req.workDir.toFile()) - .redirectErrorStream(true) - .start() - - final timeout = req.maxDuration ?: buildConfig.defaultTimeout - final completed = proc.waitFor(timeout.toSeconds(), TimeUnit.SECONDS) - final stdout = proc.inputStream.text - if( completed ) { - final digest = proc.exitValue()==0 ? proxyService.getImageDigest(req, true) : null - return BuildResult.completed(req.buildId, proc.exitValue(), stdout, req.startTime, digest) - } - else { - return BuildResult.failed(req.buildId, stdout, req.startTime) + final process = new ProcessBuilder() + .command(buildCmd) + .directory(req.workDir.toFile()) + .redirectErrorStream(true) + .start() + + if( process.waitFor()!=0 ) { + throw new IllegalStateException("Unable to launch build container - exitCode=${process.exitValue()}; output=${process.text}") } } - protected List buildCmd(BuildRequest req, Path credsFile) { + protected List buildCmd(String jobName, BuildRequest req, Path credsFile) { final spack = req.isSpackBuild ? spackConfig : null final dockerCmd = req.formatDocker() - ? cmdForBuildkit( req.workDir, credsFile, spack, req.platform) - : cmdForSingularity( req.workDir, credsFile, spack, req.platform) + ? cmdForBuildkit(jobName, req.workDir, credsFile, spack, req.platform) + : cmdForSingularity(jobName, req.workDir, credsFile, spack, req.platform) return dockerCmd + launchCmd(req) } - protected List cmdForBuildkit(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { + protected List cmdForBuildkit(String name, Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', - '--rm', + '--detach', + '--name', name, '--privileged', '-v', "$workDir:$workDir".toString(), '--entrypoint', @@ -149,10 +142,11 @@ class DockerBuildStrategy extends BuildStrategy { return wrapper } - protected List cmdForSingularity(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform) { + protected List cmdForSingularity(String name, Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform) { final wrapper = ['docker', 'run', - '--rm', + '--detach', + '--name', name, '--privileged', "--entrypoint", '', '-v', "$workDir:$workDir".toString()] diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index 7ac6d20c5..4d4e9fa06 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -71,12 +71,9 @@ class KubeBuildStrategy extends BuildStrategy { @Inject private RegistryProxyService proxyService - protected String podName(BuildRequest req) { - return "build-${req.buildId}".toString().replace('_', '-') - } @Override - BuildResult build(BuildRequest req) { + void build(String jobName, BuildRequest req) { Path configFile = null if( req.configJson ) { @@ -96,20 +93,10 @@ class KubeBuildStrategy extends BuildStrategy { try { final buildImage = getBuildImage(req) final buildCmd = launchCmd(req) - final name = podName(req) final timeout = req.maxDuration ?: buildConfig.defaultTimeout final selector= getSelectorLabel(req.platform, nodeSelectorMap) final spackCfg0 = req.isSpackBuild ? spackConfig : null - final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, timeout, spackCfg0, selector) - final exitCode = k8sService.waitPodCompletion(pod, timeout.toMillis()) - final stdout = k8sService.logsPod(pod) - if( exitCode!=null ) { - final digest = exitCode==0 ? proxyService.getImageDigest(req, true) : null - return BuildResult.completed(req.buildId, exitCode, stdout, req.startTime, digest) - } - else { - return BuildResult.failed(req.buildId, stdout, req.startTime) - } + k8sService.launchBuildJob(jobName, buildImage, buildCmd, req.workDir, configFile, timeout, spackCfg0, selector) } catch (ApiException e) { throw new BadRequestException("Unexpected build failure - ${e.responseBody}", e) @@ -128,16 +115,4 @@ class KubeBuildStrategy extends BuildStrategy { throw new IllegalArgumentException("Unexpected container platform: ${buildRequest.platform}") } - @Override - void cleanup(BuildRequest req) { - super.cleanup(req) - final name = podName(req) - try { - k8sService.deletePod(name) - } - catch (Exception e) { - log.warn ("Unable to delete pod=$name - cause: ${e.message ?: e}", e) - } - } - } diff --git a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupConfig.groovy b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupConfig.groovy new file mode 100644 index 000000000..1dddc77d4 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupConfig.groovy @@ -0,0 +1,62 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cleanup + +import java.time.Duration +import javax.annotation.Nullable + +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.micronaut.context.annotation.Value +import io.seqera.wave.util.DurationUtils +import jakarta.inject.Singleton + +/** + * Model configuration settings for resources cleanup + * + * @author Paolo Di Tommaso + */ +@ToString(includePackage = false, includeNames = true) +@CompileStatic +@Singleton +class CleanupConfig { + + @Value('${wave.cleanup.strategy}') + @Nullable + String strategy + + @Value('${wave.cleanup.succeeded:5m}') + Duration succeededDuration + + @Value('${wave.cleanup.failed:1d}') + Duration failedDuration + + @Value('${wave.cleanup.range:200}') + int cleanupRange + + @Value('${wave.cleanup.startup-delay:10s}') + Duration cleanupStartupDelay + + @Value('${wave.cleanup.run-interval:30s}') + Duration cleanupRunInterval + + Duration getCleanupStartupDelayRandomized() { + DurationUtils.randomDuration(cleanupStartupDelay, 0.4f) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupService.groovy b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupService.groovy new file mode 100644 index 000000000..e5520d409 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupService.groovy @@ -0,0 +1,32 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cleanup + + +import io.seqera.wave.service.job.JobSpec +/** + * Define the contract for resources cleanup service + * + * @author Paolo Di Tommaso + */ +interface CleanupService { + + void cleanupJob(JobSpec job, Integer exitStatus) + +} diff --git a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy new file mode 100644 index 000000000..1b8baba53 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy @@ -0,0 +1,138 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cleanup + +import java.nio.file.Path +import java.time.Instant + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Context +import io.micronaut.scheduling.TaskScheduler +import io.seqera.wave.service.job.JobOperation +import io.seqera.wave.service.job.JobSpec +import jakarta.annotation.PostConstruct +import jakarta.inject.Inject +/** + * Implement a service for resources cleanup + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Context +@CompileStatic +class CleanupServiceImpl implements Runnable, CleanupService { + + static final private String DIR_PREFIX = 'dir:' + + static final private String JOB_PREFIX = 'job:' + + @Inject + private TaskScheduler scheduler + + @Inject + private CleanupStore store + + @Inject + private CleanupConfig config + + @Inject + private CleanupStrategy cleanupStrategy + + @Inject + private JobOperation operation + + @PostConstruct + private init() { + log.info "Creating cleanup service - config=$config" + // use randomize initial delay to prevent multiple replicas running at the same time + scheduler.scheduleWithFixedDelay( + config.cleanupStartupDelayRandomized, + config.cleanupRunInterval, + this ) + } + + @Override + void run() { + final now = Instant.now() + final keys = store.getRange(0, now.epochSecond, config.cleanupRange) + + for( String it : keys ) { + try { + cleanupEntry(it) + } + catch (InterruptedException e) { + Thread.interrupted() + } + catch (Throwable t) { + log.error("Unexpected error in JWT heartbeat while processing key: $it", t) + } + } + } + + protected void cleanupEntry(String entry) { + log.debug "Cleaning up entry $entry" + if( entry.startsWith(JOB_PREFIX) ) { + cleanupJob0(entry.substring(4)) + } + else if( entry.startsWith(DIR_PREFIX) ) { + cleanupDir0(entry.substring(4)) + } + else { + log.error "Unknown cleanup entry - offending value: $entry" + } + } + + protected void cleanupJob0(String jobName) { + try { + operation.cleanup(jobName) + } + catch (Throwable t) { + log.error("Unexpected error deleting job=$jobName - cause: ${t.message}", t) + } + } + + protected void cleanupDir0(String path) { + try { + Path.of(path).deleteDir() + } + catch (Throwable t) { + log.error("Unexpected error deleting path=$path - cause: ${t.message}", t) + } + } + + @Override + void cleanupJob(JobSpec job, Integer exitStatus) { + if( !cleanupStrategy.shouldCleanup(exitStatus) ) { + return + } + + final ttl = exitStatus==0 + ? config.succeededDuration + : config.failedDuration + final expirationSecs = Instant.now().plus(ttl).epochSecond + // schedule the job deletion + store.add(JOB_PREFIX + job.operationName, expirationSecs) + // schedule work dir path deletion + if( job.workDir ) { + store.add(DIR_PREFIX + job.workDir, expirationSecs) + } + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupStore.groovy b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupStore.groovy new file mode 100644 index 000000000..2f02028ab --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupStore.groovy @@ -0,0 +1,44 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cleanup + +import groovy.transform.CompileStatic +import io.seqera.wave.memstore.range.AbstractRangeStore +import io.seqera.wave.memstore.range.impl.RangeProvider +import jakarta.inject.Singleton + +/** + * Implements a timed range to store and retrieve IDs + * of resources to be cleaned up. + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +class CleanupStore extends AbstractRangeStore { + + CleanupStore(RangeProvider provider) { + super(provider) + } + + @Override + protected String getKey() { + return 'cleanup-store/v1:' + } +} diff --git a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupStrategy.groovy b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupStrategy.groovy index 6a75d88b2..feeb76505 100644 --- a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupStrategy.groovy @@ -21,8 +21,6 @@ package io.seqera.wave.service.cleanup import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value -import io.seqera.wave.configuration.BuildConfig -import io.seqera.wave.service.builder.BuildResult import jakarta.inject.Inject import jakarta.inject.Singleton /** @@ -36,27 +34,23 @@ import jakarta.inject.Singleton class CleanupStrategy { @Value('${wave.debug:false}') - Boolean debugMode + private Boolean debugMode @Inject - BuildConfig buildConfig - - boolean shouldCleanup(BuildResult result) { - shouldCleanup(result?.exitStatus) - } + private CleanupConfig config boolean shouldCleanup(Integer exitStatus) { - final cleanup = buildConfig.cleanup + final cleanup = config.strategy if( cleanup==null ) return !debugMode - if( cleanup == 'true' ) + if( cleanup == 'always' ) return true - if( cleanup == 'false' ) + if( cleanup == 'never' ) return false if( cleanup.toLowerCase() == 'onsuccess' ) { return exitStatus==0 } - log.debug "Invalid cleanup value: '$cleanup'" + log.debug "Invalid cleanup strategy: '$cleanup' - check setting 'wave.cleanup.strategy'" return true } diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy index 54331bcb6..262ce3151 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy @@ -41,7 +41,7 @@ abstract class AbstractMessageStream implements Closeable { static final private AtomicInteger count = new AtomicInteger() - final private Map> listeners = new ConcurrentHashMap<>() + final private Map> listeners = new ConcurrentHashMap<>() final private ExponentialAttempt attempt = new ExponentialAttempt() @@ -97,10 +97,15 @@ abstract class AbstractMessageStream implements Closeable { * @param consumer * The {@link Predicate} to be invoked when a stream message is consumed (read from) the stream. */ - void consume(String streamId, Predicate consumer) { - final value = listeners.put(streamId, consumer) - if( value!=null ) - throw new IllegalStateException("Only one consumer can be defined for each stream - offending streamId=$streamId; consumer=$consumer") + void addConsumer(String streamId, MessageConsumer consumer) { + synchronized (listeners) { + if( listeners.containsKey(streamId)) + throw new IllegalStateException("Only one consumer can be defined for each stream - offending streamId=$streamId; consumer=$consumer") + // initialize the stream + stream.init(streamId) + // then add the consumer to the listeners + final value = listeners.put(streamId, consumer) + } } /** @@ -117,11 +122,11 @@ abstract class AbstractMessageStream implements Closeable { * @return * The result of the consumer {@link Predicate} operation. */ - protected boolean processMessage(String msg, Predicate consumer, AtomicInteger count) { + protected boolean processMessage(String msg, MessageConsumer consumer, AtomicInteger count) { count.incrementAndGet() final decoded = encoder.decode(msg) log.trace "Message streaming - receiving message=$msg; decoded=$decoded" - return consumer.test(decoded) + return consumer.accept(decoded) } /** @@ -132,7 +137,7 @@ abstract class AbstractMessageStream implements Closeable { while( !thread.interrupted() ) { try { final count=new AtomicInteger() - for( Map.Entry> entry : listeners.entrySet() ) { + for( Map.Entry> entry : listeners.entrySet() ) { final streamId = entry.key final consumer = entry.value stream.consume(streamId, (String msg)-> processMessage(msg, consumer, count)) diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/MessageConsumer.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/MessageConsumer.groovy new file mode 100644 index 000000000..eae10a76a --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/data/stream/MessageConsumer.groovy @@ -0,0 +1,40 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +/** + * Defines the message consumer interface + * + * @author Paolo Di Tommaso + */ +interface MessageConsumer { + + /** + * Consume a message from the stream + * + * @param message + * A message payload ready to be consumed + * @return + * {@link true} to acknowledge the consumer has effectively consumed the message, + * so that it's not made available to other consumers. {@link false} the message + * has been consumed, therefore other consumers will ultimately receive it. + */ + boolean accept(T message) + +} diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy index 279183991..cb77741b4 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy @@ -17,9 +17,6 @@ */ package io.seqera.wave.service.data.stream - - -import java.util.function.Predicate /** * Define the contract for a generic message stream * able to add message and consume them asynchronously. @@ -28,6 +25,13 @@ import java.util.function.Predicate */ interface MessageStream { + /** + * Initialize the stream with the given Id + * + * @param streamId The uniqur ID of the stream to be initialized + */ + void init(String streamId) + /** * Offer a message to the stream. * @@ -37,16 +41,16 @@ interface MessageStream { void offer(String streamId, M message) /** - * Consume a message from the stream and invoke the specified predicate + * Consume a message from the stream and invoke the specified consumer predicate * * @param streamId * The target stream ID * @param consumer - * The {@link Predicate} instance to be invoked to consume the message + * The {@link MessageConsumer} instance to be invoked to consume the message * @return * {code true} when the message has been processed successfully, * otherwise {@code false} when the message needs to be further processed */ - boolean consume(String streamId, Predicate consumer) + boolean consume(String streamId, MessageConsumer consumer) } diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy index 01d971080..ce3c349be 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy @@ -20,11 +20,11 @@ package io.seqera.wave.service.data.stream.impl import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.LinkedBlockingQueue -import java.util.function.Predicate import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires +import io.seqera.wave.service.data.stream.MessageConsumer import io.seqera.wave.service.data.stream.MessageStream import jakarta.inject.Singleton /** @@ -41,17 +41,22 @@ class LocalMessageStream implements MessageStream { private ConcurrentHashMap> delegate = new ConcurrentHashMap<>() + @Override + void init(String streamId) { + delegate.put(streamId, new LinkedBlockingQueue<>()) + } + @Override void offer(String streamId, String message) { delegate - .computeIfAbsent(streamId, (it)-> new LinkedBlockingQueue<>()) + .get(streamId) .offer(message) } @Override - boolean consume(String streamId, Predicate consumer) { + boolean consume(String streamId, MessageConsumer consumer) { final message = delegate - .computeIfAbsent(streamId, (it)-> new LinkedBlockingQueue<>()) + .get(streamId) .poll() if( message==null ) { return false @@ -59,7 +64,7 @@ class LocalMessageStream implements MessageStream { def result = false try { - result = consumer.test(message) + result = consumer.accept(message) } catch (Throwable e) { result = false diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy index e9402e8ec..a8150962c 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy @@ -19,13 +19,12 @@ package io.seqera.wave.service.data.stream.impl import java.time.Duration -import java.util.concurrent.ConcurrentHashMap -import java.util.function.Predicate import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value +import io.seqera.wave.service.data.stream.MessageConsumer import io.seqera.wave.service.data.stream.MessageStream import io.seqera.wave.util.LongRndKey import jakarta.annotation.PostConstruct @@ -57,42 +56,42 @@ class RedisMessageStream implements MessageStream { private static final String DATA_FIELD = 'data' - private final ConcurrentHashMap group0 = new ConcurrentHashMap<>() - @Inject private JedisPool pool - @Value('${wave.message-stream.claim-timeout:10s}') + @Value('${wave.message-stream.claim-timeout:5s}') private Duration claimTimeout private String consumerName @PostConstruct - private void init() { + private void create() { consumerName = "consumer-${LongRndKey.rndLong()}" log.info "Creating Redis message stream - consumer=${consumerName}; claim-timeout=${claimTimeout}" } - protected void createGroup(Jedis jedis, String stream, String group) { - // use a concurrent hash map to create it only the very first time - group0.computeIfAbsent("$stream/$group".toString(),(it)-> createGroup0(jedis,stream,group)) - } - - protected boolean createGroup0(Jedis jedis, String stream, String group) { + protected boolean initGroup0(Jedis jedis, String streamId, String group) { + log.debug "Initializing Redis group='$group'; streamId='$streamId'" try { - jedis.xgroupCreate(stream, group, STREAM_ENTRY_ZERO, true) + jedis.xgroupCreate(streamId, group, STREAM_ENTRY_ZERO, true) return true } catch (JedisDataException e) { if (e.message.contains("BUSYGROUP")) { // The group already exists, so we can safely ignore this exception - log.debug "Redis message stream - consume group=$group already exists" + log.info "Redis message stream - consume group=$group already exists" return true } throw e } } + void init(String streamId) { + try (Jedis jedis = pool.getResource()) { + initGroup0(jedis, streamId, CONSUMER_GROUP_NAME) + } + } + @Override void offer(String streamId, String message) { try (Jedis jedis = pool.getResource()) { @@ -101,12 +100,12 @@ class RedisMessageStream implements MessageStream { } @Override - boolean consume(String streamId, Predicate consumer) { + boolean consume(String streamId, MessageConsumer consumer) { try (Jedis jedis = pool.getResource()) { - createGroup(jedis, streamId, CONSUMER_GROUP_NAME) final entry = claimMessage(jedis,streamId) ?: readMessage(jedis, streamId) - if( entry && consumer.test(entry.getFields().get(DATA_FIELD)) ) { - // Acknowledge the job after processing + if( entry && consumer.accept(entry.getFields().get(DATA_FIELD)) ) { + // acknowledge the job after processing + // this remove permanently the entry from the stream jedis.xack(streamId, CONSUMER_GROUP_NAME, entry.getID()) return true } @@ -129,7 +128,8 @@ class RedisMessageStream implements MessageStream { Map.of(streamId, StreamEntryID.UNRECEIVED_ENTRY) ) final entry = messages?.first()?.value?.first() - log.trace "Redis stream id=$streamId; read entry=$entry" + if( entry!=null ) + log.trace "Redis stream id=$streamId; read entry=$entry" return entry } @@ -147,7 +147,8 @@ class RedisMessageStream implements MessageStream { params ) final entry = messages?.getValue()?[0] - log.trace "Redis stream id=$streamId; claimed entry=$entry" + if( entry!=null ) + log.trace "Redis stream id=$streamId; claimed entry=$entry" return entry } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobConfig.groovy b/src/main/groovy/io/seqera/wave/service/job/JobConfig.groovy index 504ba9506..a2c5d3db9 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobConfig.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobConfig.groovy @@ -33,7 +33,7 @@ import jakarta.inject.Singleton @Singleton class JobConfig { - @Value('${wave.job-manager.grace-interval:20s}') + @Value('${wave.job-manager.grace-interval:30s}') Duration graceInterval @Value('${wave.job-manager.poll-interval:200ms}') diff --git a/src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy b/src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy index 44b8ae519..9b00b7146 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy @@ -18,68 +18,74 @@ package io.seqera.wave.service.job -import java.time.Duration +import java.util.function.BiConsumer import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.ApplicationContext -import io.micronaut.inject.qualifiers.Qualifiers import jakarta.annotation.PostConstruct import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton /** * Concrete implementation of {@link JobHandler} that dispatcher event invocations - * to the target implementation based on the job {@link JobId#type} + * to the target implementation based on the job {@link JobSpec#type} * * @author Paolo Di Tommaso */ @Slf4j @Singleton @CompileStatic -class JobDispatcher implements JobHandler { +class JobDispatcher { @Inject private ApplicationContext context - private Map dispatch = new HashMap<>() + private Map> dispatch = new HashMap<>() + /** + * Load all available implementations of {@link JobHandler}. Job handler should be: + * 1. declared as a @Singleton + * 2. annotated with the {@link Named} annotation + * 3. the Named value should be a literal matching the corresponding {@JobSpec.Type} + */ @PostConstruct - void init() { - // implementation should be added here - add(JobId.Type.Transfer, dispatch, false) + protected init() { + final handlers = context.getBeansOfType(JobHandler) + for( JobHandler it : handlers ) { + final qualifier = it.getClass().getAnnotation(Named)?.value() + if( !qualifier ) + throw new IllegalStateException("Missing 'Named' annotation for handler ${it.class.name}") + final type = JobSpec.Type.valueOf(qualifier) + log.info "Adding job handler for type: $type; handler=${it.class.simpleName}" + dispatch.put(type, it) + } } - protected void add(JobId.Type type, Map map, boolean required) { - final handler = context.findBean(JobHandler.class, Qualifiers.byName(type.toString())) - if( handler.isPresent() ) { - log.debug "Adding job handler for type: $type; handler=$handler" - map.put(type, handler.get()) + protected void apply(JobSpec job, BiConsumer consumer) { + final handler = dispatch.get(job.type) + final record = handler.getJobRecord(job) + if( !record ) { + log.error "== ${job.type} record unknown for job=${job.recordId}" } - else if( required ) { - throw new IllegalStateException("Unable to find Job handler for type: $type") + else if( record.done() ) { + log.warn "== ${job.type} record already marked as completed for job=${job.recordId}" } else { - log.debug "Disabled job handler for type: $type" + consumer.accept(handler, record) } } - @Override - Duration jobMaxDuration(JobId job) { - return dispatch.get(job.type).jobMaxDuration(job) + void notifyJobCompletion(JobSpec job, JobState state) { + apply(job, (handler, record)-> handler.onJobCompletion(job, record, state)) } - @Override - void onJobCompletion(JobId job, JobState state) { - dispatch.get(job.type).onJobCompletion(job, state) + void notifyJobException(JobSpec job, Throwable error) { + apply(job, (handler, record)-> handler.onJobException(job, record, error)) } - @Override - void onJobException(JobId job, Throwable error) { - dispatch.get(job.type).onJobException(job, error) + void notifyJobTimeout(JobSpec job) { + apply(job, (handler, record)-> handler.onJobTimeout(job, record)) } - @Override - void onJobTimeout(JobId job) { - dispatch.get(job.type).onJobTimeout(job) - } } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy new file mode 100644 index 000000000..c01158145 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy @@ -0,0 +1,89 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import java.time.Instant +import javax.annotation.Nullable + +import com.google.common.hash.Hashing +import groovy.transform.CompileStatic +import io.seqera.wave.configuration.BlobCacheConfig +import io.seqera.wave.configuration.ScanConfig +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.scan.ScanRequest +import jakarta.inject.Inject +import jakarta.inject.Singleton +/** + * Simple factory for {@link JobSpec} objects + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +class JobFactory { + + @Inject + @Nullable + private BlobCacheConfig blobConfig + + @Inject + @Nullable + private ScanConfig scanConfig + + JobSpec transfer(String stateId) { + JobSpec.transfer( + stateId, + generate("transfer", stateId, Instant.now()), + Instant.now(), + blobConfig.transferTimeout, + ) + } + + JobSpec build(BuildRequest request) { + JobSpec.build( + request.targetImage, + "build-" + request.buildId.replace('_', '-'), + request.startTime, + request.maxDuration, + request.workDir + ) + } + + JobSpec scan(ScanRequest request) { + JobSpec.scan( + request.id, + "scan-${request.id}", + request.creationTime, + scanConfig.timeout, + request.workDir + ) + } + + static private String generate(String type, String id, Instant creationTime) { + final prefix = type.toLowerCase() + return prefix + '-' + Hashing + .sipHash24() + .newHasher() + .putUnencodedChars(id) + .putUnencodedChars(type.toString()) + .putUnencodedChars(creationTime.toString()) + .hash() + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy b/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy index 33898e01d..1630c0f26 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy @@ -17,22 +17,19 @@ */ package io.seqera.wave.service.job - -import java.time.Duration - /** * Define events and properties for jobs managed via {@link JobManager} * * @author Paolo Di Tommaso */ -interface JobHandler { +interface JobHandler { - Duration jobMaxDuration(JobId job) + R getJobRecord(JobSpec job) - void onJobCompletion(JobId job, JobState state) + void onJobCompletion(JobSpec job, R jobRecord, JobState state) - void onJobException(JobId job, Throwable error) + void onJobException(JobSpec job, R jobRecord, Throwable error) - void onJobTimeout(JobId job) + void onJobTimeout(JobSpec job, R jobRecord) } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobId.groovy b/src/main/groovy/io/seqera/wave/service/job/JobId.groovy deleted file mode 100644 index b0f40c3c5..000000000 --- a/src/main/groovy/io/seqera/wave/service/job/JobId.groovy +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.job - -import java.time.Instant - -import com.google.common.hash.Hashing -import groovy.transform.Canonical - -/** - * Model a unique job to be managed - * - * @author Paolo Di Tommaso - */ -@Canonical -class JobId { - enum Type { Transfer, Build, Scan } - - final Type type - final String id - final Instant creationTime - final String schedulerId - - JobId( Type type, String id, Instant creationTime ) { - this.type = type - this.id = id - this.creationTime = creationTime - schedulerId = generate(type, id, creationTime) - } - - static JobId transfer(String id) { - new JobId(Type.Transfer, id, Instant.now()) - } - - static JobId build(String id) { - new JobId(Type.Build, id, Instant.now()) - } - - static JobId scan(String id) { - new JobId(Type.Scan, id, Instant.now()) - } - - static private String generate(Type type, String id, Instant creationTime) { - final prefix = type.toString().toLowerCase() - return prefix + '-' + Hashing - .sipHash24() - .newHasher() - .putUnencodedChars(id) - .putUnencodedChars(type.toString()) - .putUnencodedChars(creationTime.toString()) - .hash() - } - -} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy index 2970e6aa5..070888743 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy @@ -21,10 +21,11 @@ package io.seqera.wave.service.job import java.time.Duration import java.time.Instant +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Context -import io.micronaut.context.annotation.Requires import jakarta.annotation.PostConstruct import jakarta.inject.Inject /** @@ -35,11 +36,10 @@ import jakarta.inject.Inject @Slf4j @Context @CompileStatic -@Requires(property = 'wave.blobCache.enabled', value = 'true') class JobManager { @Inject - private JobStrategy jobStrategy + private JobService jobService @Inject private JobQueue queue @@ -50,49 +50,73 @@ class JobManager { @Inject private JobConfig config + private Cache debounceCache + @PostConstruct void init() { log.info "Creating job manager - config=$config" - queue.consume((job)-> processJob(job)) + debounceCache = Caffeine.newBuilder().expireAfterWrite(config.graceInterval.multipliedBy(2)).build() + queue.addConsumer((job)-> processJob(job)) } - protected boolean processJob(JobId jobId) { + + protected boolean processJob(JobSpec jobSpec) { try { - return processJob0(jobId) + return processJob0(jobSpec) } catch (Throwable err) { // in the case of an expected exception report the error condition by using `onJobException` - dispatcher.onJobException(jobId, err) - // finally return `true` to signal the job should not be processed anymore + dispatcher.notifyJobException(jobSpec, err) + // note: return `true` to signal the job should not be processed anymore return true } } - protected boolean processJob0(JobId jobId) { - final duration = Duration.between(jobId.creationTime, Instant.now()) - final state = jobStrategy.status(jobId) - log.trace "Job status id=${jobId.schedulerId}; state=${state}" - final done = - state.completed() || - // considered failed when remain in unknown status too long - (state.status==JobState.Status.UNKNOWN && duration>config.graceInterval) - if( done ) { + protected JobState state(JobSpec job) { + return state0(job, config.graceInterval, debounceCache) + } + + protected JobState state0(final JobSpec job, final Duration graceInterval, final Cache cache) { + final key = job.operationName + final state = jobService.status(job) + // return directly non-unknown statuses + if( state.status != JobState.Status.UNKNOWN ) { + cache.invalidate(key) + return state + } + // check how long it returns an unknown persistently + final initial = cache.get(key, (String)-> Instant.now()) + final delta = Duration.between(initial, Instant.now()) + // if it's less than the grace period return it + if( delta <= graceInterval ) { + return state + } + // if longer then the grace period, return a FAILED status to force an error + cache.invalidate(key) + return new JobState(JobState.Status.FAILED, null, state.stdout) + } + + protected boolean processJob0(JobSpec jobSpec) { + final duration = Duration.between(jobSpec.creationTime, Instant.now()) + final state = state(jobSpec) + log.trace "Job status id=${jobSpec.operationName}; state=${state}" + if( state.completed() ) { // publish the completion event - dispatcher.onJobCompletion(jobId, state) + dispatcher.notifyJobCompletion(jobSpec, state) // cleanup the job - jobStrategy.cleanup(jobId, state.exitCode) + jobService.cleanup(jobSpec, state.exitCode) return true } - // set the await timeout nearly double as the blob transfer timeout, this because the - // transfer pod can spend `timeout` time in pending status awaiting to be scheduled - // and the same `timeout` time amount carrying out the transfer (upload) operation - final max = (dispatcher.jobMaxDuration(jobId).toMillis() * 2.10) as long + // set the await timeout nearly double as the job timeout, this because the + // job can spend `timeout` time in pending status awaiting to be scheduled + // and the same `timeout` time amount carrying out job operation + final max = (jobSpec.maxDuration.toMillis() * 2.10) as long if( duration.toMillis()>max ) { - dispatcher.onJobTimeout(jobId) + dispatcher.notifyJobTimeout(jobSpec) return true } else { - log.trace "== Job pending for completion $jobId" + log.trace "== Job pending for completion ${jobSpec}" return false } } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobOperation.groovy b/src/main/groovy/io/seqera/wave/service/job/JobOperation.groovy new file mode 100644 index 000000000..804084537 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobOperation.groovy @@ -0,0 +1,33 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job +/** + * Define job operations contract + * + * @author Paolo Di Tommaso + */ +interface JobOperation { + + JobState status(JobSpec jobSpec) + + void cleanup(JobSpec jobSpec) + + void cleanup(String jobName) + +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy b/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy index 9eba8ecc4..c37d47881 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy @@ -19,10 +19,11 @@ package io.seqera.wave.service.job import java.time.Duration -import java.util.function.Predicate import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j import io.seqera.wave.service.data.stream.AbstractMessageStream +import io.seqera.wave.service.data.stream.MessageConsumer import io.seqera.wave.service.data.stream.MessageStream import jakarta.annotation.PreDestroy import jakarta.inject.Singleton @@ -31,17 +32,19 @@ import jakarta.inject.Singleton * * @author Paolo Di Tommaso */ +@Slf4j @Singleton @CompileStatic -class JobQueue extends AbstractMessageStream { +class JobQueue extends AbstractMessageStream { - final private static String STREAM_NAME = 'jobs-queue/v1' + final private static String STREAM_NAME = 'jobs-queue/v1:' private volatile JobConfig config JobQueue(MessageStream target, JobConfig config) { super(target) this.config = config + log.debug "Created job queue" } @Override @@ -54,16 +57,17 @@ class JobQueue extends AbstractMessageStream { return config.pollInterval } - final void offer(JobId job) { - super.offer(STREAM_NAME, job) + final void offer(JobSpec jobSpec) { + super.offer(STREAM_NAME, jobSpec) } - final void consume(Predicate consumer) { - super.consume(STREAM_NAME, consumer) + final void addConsumer(MessageConsumer consumer) { + super.addConsumer(STREAM_NAME, consumer) } @PreDestroy void destroy() { + log.debug "Shutting down job queue" this.close() } } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobRecord.groovy b/src/main/groovy/io/seqera/wave/service/job/JobRecord.groovy new file mode 100644 index 000000000..4bd9f4ac9 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobRecord.groovy @@ -0,0 +1,28 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +/** + * Marker interface for persisted state record associated with a job execution + * + * @author Paolo Di Tommaso + */ +interface JobRecord { + boolean done() +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy index 9c1c6f2e1..91bd19b77 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy @@ -19,6 +19,8 @@ package io.seqera.wave.service.job import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.scan.ScanRequest /** * Define the contract for submitting and monitoring jobs @@ -27,6 +29,14 @@ import io.seqera.wave.service.blob.BlobCacheInfo */ interface JobService { - JobId launchTransfer(BlobCacheInfo blob, List command) + JobSpec launchTransfer(BlobCacheInfo blob, List command) + + JobSpec launchBuild(BuildRequest request) + + JobSpec launchScan(ScanRequest request) + + JobState status(JobSpec jobSpec) + + void cleanup(JobSpec jobSpec, Integer exitStatus) } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy index 902e17c7b..8e093af1d 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy @@ -18,35 +18,103 @@ package io.seqera.wave.service.job +import javax.annotation.Nullable + import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.blob.TransferStrategy +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildStrategy +import io.seqera.wave.service.cleanup.CleanupService +import io.seqera.wave.service.scan.ScanRequest +import io.seqera.wave.service.scan.ScanStrategy import jakarta.inject.Inject import jakarta.inject.Singleton - /** * Implement a service for job creation and execution * * @author Paolo Di Tommaso */ +@Slf4j @Singleton @CompileStatic class JobServiceImpl implements JobService { @Inject - private JobStrategy jobStrategy + private CleanupService cleanupService + + @Inject + private JobOperation operations + + @Inject + @Nullable + private TransferStrategy transferStrategy + + @Inject + private BuildStrategy buildStrategy + + @Inject + @Nullable + private ScanStrategy scanStrategy @Inject private JobQueue jobQueue + @Inject + private JobFactory jobFactory + @Override - JobId launchTransfer(BlobCacheInfo blob, List command) { + JobSpec launchTransfer(BlobCacheInfo blob, List command) { + if( !transferStrategy ) + throw new IllegalStateException("Blob cache service is not available - check configuration setting 'wave.blobCache.enabled'") // create the ID for the job transfer - final job = JobId.transfer(blob.id()) + final job = jobFactory.transfer(blob.id()) // submit the job execution - jobStrategy.launchJob(job.schedulerId, command) - // signal the transfer to be started + transferStrategy.launchJob(job.operationName, command) + // signal the transfer has been submitted + jobQueue.offer(job) + return job + } + + @Override + JobSpec launchBuild(BuildRequest request) { + // create the unique job id for the build + final job = jobFactory.build(request) + // launch the build job + buildStrategy.build(job.operationName, request) + // signal the build has been submitted + jobQueue.offer(job) + return job + } + + @Override + JobSpec launchScan(ScanRequest request) { + if( !scanStrategy ) + throw new IllegalStateException("Container scan service is not available - check configuration setting 'wave.scan.enabled'") + + // create the unique job id for the build + final job = jobFactory.scan(request) + // launch the scan job + scanStrategy.scanContainer(job.operationName, request) + // signal the build has been submitted jobQueue.offer(job) return job } + @Override + JobState status(JobSpec job) { + try { + return operations.status(job) + } + catch (Throwable t) { + log.warn "Unable to obtain status for job=${job.operationName} - cause: ${t.message}", t + return new JobState(JobState.Status.UNKNOWN, null, t.message) + } + } + + @Override + void cleanup(JobSpec job, Integer exitStatus) { + cleanupService.cleanupJob(job, exitStatus) + } } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy new file mode 100644 index 000000000..90bae336c --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy @@ -0,0 +1,124 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import io.seqera.wave.util.LongRndKey + +/** + * Model a unique job to be managed + * + * @author Paolo Di Tommaso + */ +@EqualsAndHashCode +@ToString(includePackage = false, includeNames = true) +@CompileStatic +class JobSpec { + + enum Type { Transfer, Build, Scan } + + /** + * The job unique identifier + */ + final String id + + /** + * The type of the job. See {@link Type} + */ + final Type type + + /** + * The unique id of the state record associated with this job + */ + final String recordId + + /** + * The unique name of the underlying infra operation associated with this job + * e.g. the K8s job name or Docker container name + */ + final String operationName + + /** + * The instant when this job was created + */ + final Instant creationTime + + /** + * The max time to live of the job + */ + final Duration maxDuration + + /** + * The temporary path associated with this job (optional). This is expected to be deleted + * once the job execution terminates. + */ + final Path workDir + + protected JobSpec(String id, Type type, String recordId, String operationName, Instant createdAt, Duration maxDuration, Path dir) { + this.id = id + this.type = type + this.recordId = recordId + this.operationName = operationName + this.maxDuration = maxDuration + this.creationTime = createdAt + this.workDir = dir + } + + static JobSpec transfer(String stateId, String operationName, Instant creationTime, Duration maxDuration) { + new JobSpec( + LongRndKey.rndHex(), + Type.Transfer, + stateId, + operationName, + creationTime, + maxDuration, + null + ) + } + + static JobSpec scan(String stateId, String operationName, Instant creationTime, Duration maxDuration, Path dir) { + new JobSpec( + LongRndKey.rndHex(), + Type.Scan, + stateId, + operationName, + creationTime, + maxDuration, + dir + ) + } + + static JobSpec build(String stateId, String operationName, Instant creationTime, Duration maxDuration, Path dir) { + new JobSpec( + LongRndKey.rndHex(), + Type.Build, + stateId, + operationName, + creationTime, + maxDuration, + dir + ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobState.groovy b/src/main/groovy/io/seqera/wave/service/job/JobState.groovy index bd24bbf2e..5c3dc2036 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobState.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobState.groovy @@ -22,7 +22,7 @@ import groovy.transform.Canonical import groovy.transform.ToString /** - * Model a transfer operation state + * Model the state of a job execution * * @author Paolo Di Tommaso */ @@ -37,11 +37,11 @@ class JobState { final Integer exitCode final String stdout - final boolean completed() { + boolean completed() { return status==Status.SUCCEEDED || status==Status.FAILED } - final boolean succeeded() { + boolean succeeded() { status==Status.SUCCEEDED && exitCode==0 } diff --git a/src/main/groovy/io/seqera/wave/service/job/impl/DockerJobOperation.groovy b/src/main/groovy/io/seqera/wave/service/job/impl/DockerJobOperation.groovy new file mode 100644 index 000000000..993d173f2 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/impl/DockerJobOperation.groovy @@ -0,0 +1,124 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job.impl + + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires +import io.seqera.wave.service.job.JobOperation +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import jakarta.inject.Singleton +/** + * Docker implementation for {@link io.seqera.wave.service.job.JobService} + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +@Singleton +@Requires(missingProperty = 'wave.build.k8s') +class DockerJobOperation implements JobOperation { + + @Override + JobState status(JobSpec jobSpec) { + final state = getDockerContainerState(jobSpec.operationName) + log.trace "Docker container status name=$jobSpec.operationName; state=$state" + + if (state.status == 'running') { + return JobState.running() + } + else if (state.status == 'exited') { + final logs = getDockerContainerLogs(jobSpec.operationName) + return JobState.completed(state.exitCode, logs) + } + else if (state.status == 'created' || state.status == 'paused') { + return JobState.pending() + } + else { + final logs = getDockerContainerLogs(jobSpec.operationName) + return JobState.unknown(logs) + } + } + + @Override + void cleanup(JobSpec jobSpec) { + cleanup(jobSpec.operationName) + } + + @Override + void cleanup(String operationName) { + final cli = new ArrayList() + cli.add('docker') + cli.add('rm') + cli.add(operationName) + + final builder = new ProcessBuilder(cli) + builder.redirectErrorStream(true) + final process = builder.start() + process.waitFor() + } + + @ToString(includePackage = false, includeNames = true) + @Canonical + static class State { + String status + Integer exitCode + + static State parse(String result) { + final ret = result.tokenize(',') + final status = ret[0] + final exit = ret[1] ? Integer.valueOf(ret[1]) : null + new State(status,exit) + } + } + + private static State getDockerContainerState(String containerName) { + final cli = new ArrayList() + cli.add('docker') + cli.add('inspect') + cli.add('--format') + cli.add('{{.State.Status}},{{.State.ExitCode}}') + cli.add(containerName) + + final builder = new ProcessBuilder(cli) + builder.redirectErrorStream(true) + final process = builder.start() + process.waitFor() + final result = process.inputStream.text.trim() + return State.parse(result) + } + + private static String getDockerContainerLogs(String containerName) { + final cli = new ArrayList() + cli.add('docker') + cli.add('logs') + cli.add(containerName) + + final builder = new ProcessBuilder(cli) + builder.redirectErrorStream(true) + final process = builder.start() + process.waitFor() + process.inputStream.text + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/job/impl/K8sJobOperation.groovy b/src/main/groovy/io/seqera/wave/service/job/impl/K8sJobOperation.groovy new file mode 100644 index 000000000..6ef1e0b9e --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/impl/K8sJobOperation.groovy @@ -0,0 +1,98 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job.impl + + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires +import io.seqera.wave.service.job.JobOperation +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.k8s.K8sService +import jakarta.inject.Inject +/** + * Kubernetes implementation for {@link io.seqera.wave.service.job.JobService} + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +@Requires(property = 'wave.build.k8s') +class K8sJobOperation implements JobOperation { + + @Inject + private K8sService k8sService + + @Override + void cleanup(JobSpec job) { + k8sService.deleteJob(job.operationName) + } + + @Override + void cleanup(String jobName) { + k8sService.deleteJob(jobName) + } + + @Override + JobState status(JobSpec job) { + final status = k8sService.getJobStatus(job.operationName) + if( !status || !status.completed() ) { + return new JobState(mapToStatus(status)) + } + + // Find the latest created pod among the pods associated with the job + final pod = k8sService.getLatestPodForJob(job.operationName) + if( !pod ) { + log.warn "K8s missing carrier pod for job ${job.operationName}" + return new JobState(JobState.Status.UNKNOWN) + } + + // determine exit code and logs + final exitCode = pod + .status + ?.containerStatuses + ?.first() + ?.state + ?.terminated + ?.exitCode + final stdout = k8sService.logsPod(pod) + return new JobState(mapToStatus(status), exitCode, stdout) + } + + /** + * Map Kubernetes job status to Transfer status + * @param jobStatus + * @return + */ + static JobState.Status mapToStatus(K8sService.JobStatus jobStatus) { + switch (jobStatus) { + case K8sService.JobStatus.Pending: + return JobState.Status.PENDING + case K8sService.JobStatus.Running: + return JobState.Status.RUNNING + case K8sService.JobStatus.Succeeded: + return JobState.Status.SUCCEEDED + case K8sService.JobStatus.Failed: + return JobState.Status.FAILED + default: + return JobState.Status.UNKNOWN + } + } +} diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 7a292dad1..6817caffd 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -42,14 +42,19 @@ interface K8sService { void deletePod(String name) + @Deprecated V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) + @Deprecated V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) + @Deprecated Integer waitPodCompletion(V1Pod pod, long timeout) + @Deprecated void deletePodWhenReachStatus(String podName, String statusName, long timeout) + @Deprecated V1Job createJob(String name, String containerImage, List args) V1Job getJob(String name) @@ -58,8 +63,13 @@ interface K8sService { void deleteJob(String name) - V1Job launchJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) + V1Job launchTransferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) + V1Job launchBuildJob(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) + + V1Job launchScanJob(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) + + @Deprecated V1PodList waitJob(V1Job job, Long timeout) V1Pod getLatestPodForJob(String jobName) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 7872cc097..77fe61b6b 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -32,6 +32,7 @@ import io.kubernetes.client.openapi.models.V1EnvVar import io.kubernetes.client.openapi.models.V1HostPathVolumeSource import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1JobBuilder +import io.kubernetes.client.openapi.models.V1JobStatus import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodBuilder @@ -152,6 +153,7 @@ class K8sServiceImpl implements K8sService { */ @Override @CompileDynamic + @Deprecated V1Job createJob(String name, String containerImage, List args) { V1Job body = new V1JobBuilder() @@ -203,15 +205,29 @@ class K8sServiceImpl implements K8sService { final job = k8sClient .batchV1Api() .readNamespacedJob(name, namespace, null) - if( !job ) + if( !job ) { + log.warn "K8s job=$name - unknown" return null - if( job.status.succeeded==1 ) + } + + final result = jobStatus0(job.status, job.spec?.backoffLimit) + log.trace "K8s job=$name - result=$result; backoff-limit=${job.spec?.backoffLimit}; status=${job.status}" + return result + } + + private JobStatus jobStatus0(V1JobStatus status, Integer backoffLimit) { + if( status.succeeded ) return JobStatus.Succeeded - if( job.status.failed>0 ) - return JobStatus.Failed + if( status.active ) + return JobStatus.Pending + if( status.failed ) { + if( status.completionTime!=null ) + return JobStatus.Failed + if( backoffLimit!=null && status.failed > backoffLimit ) + return JobStatus.Failed + } return JobStatus.Pending } - /** * Get pod description * @@ -332,6 +348,7 @@ class K8sServiceImpl implements K8sService { * The {@link V1Pod} description the submitted pod */ @Override + @Deprecated V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, spackConfig, nodeSelector) return k8sClient @@ -433,6 +450,7 @@ class K8sServiceImpl implements K8sService { * or timeout was reached. */ @Override + @Deprecated Integer waitPodCompletion(V1Pod pod, long timeout) { final start = System.currentTimeMillis() // wait for termination @@ -474,7 +492,7 @@ class K8sServiceImpl implements K8sService { logs.streamNamespacedPodLog(namespace, pod.metadata.name, pod.spec.containers.first().name).getText() } catch (Exception e) { - log.error "Unable to fetch logs for pod: ${pod.metadata.name}", e + log.error "Unable to fetch logs for pod: ${pod.metadata.name} - cause: ${e.message}" return null } } @@ -499,6 +517,7 @@ class K8sServiceImpl implements K8sService { * @param timeout The max wait time in milliseconds */ @Override + @Deprecated void deletePodWhenReachStatus(String podName, String statusName, long timeout){ final pod = getPod(podName) final start = System.currentTimeMillis() @@ -512,6 +531,7 @@ class K8sServiceImpl implements K8sService { } @Override + @Deprecated V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) { final spec = scanSpec(name, containerImage, args, workDir, creds, scanConfig, nodeSelector) return k8sClient @@ -519,6 +539,7 @@ class K8sServiceImpl implements K8sService { .createNamespacedPod(namespace, spec, null, null, null,null) } + @Deprecated V1Pod scanSpec(String name, String containerImage, List args, Path workDir, Path credsFile, ScanConfig scanConfig, Map nodeSelector) { final mounts = new ArrayList(5) @@ -585,7 +606,7 @@ class K8sServiceImpl implements K8sService { * The {@link V1Job} description the submitted job */ @Override - V1Job launchJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) { + V1Job launchTransferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) { final spec = createTransferJobSpec(name, containerImage, args, blobConfig) return k8sClient @@ -613,11 +634,9 @@ class K8sServiceImpl implements K8sService { //spec section def spec = builder.withNewSpec() .withBackoffLimit(blobConfig.retryAttempts) - .withTtlSecondsAfterFinished(blobConfig.deleteAfterFinished.toSeconds() as Integer) .withNewTemplate() .editOrNewSpec() .withServiceAccount(serviceAccount) - .withActiveDeadlineSeconds(blobConfig.transferTimeout.toSeconds()) .withRestartPolicy("Never") //container section .addNewContainer() @@ -634,6 +653,168 @@ class K8sServiceImpl implements K8sService { return spec.build() } + /** + * Create a container for container image building via buildkit + * + * @param name + * The name of pod + * @param containerImage + * The container image to be used + * @param args + * The build command to be performed + * @param workDir + * The build context directory + * @param creds + * The target container repository credentials + * @return + * The {@link V1Pod} description the submitted pod + */ + @Override + V1Job launchBuildJob(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { + final spec = buildJobSpec(name, containerImage, args, workDir, creds, timeout, spackConfig, nodeSelector) + return k8sClient + .batchV1Api() + .createNamespacedJob(namespace, spec, null, null, null,null) + } + + V1Job buildJobSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { + + // dirty dependency to avoid introducing another parameter + final singularity = containerImage.contains('singularity') + + // required volumes + final mounts = new ArrayList(5) + mounts.add(mountBuildStorage(workDir, storageMountPath, true)) + + final volumes = new ArrayList(5) + volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) + + if( credsFile ){ + mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json')) + } + + if( spackConfig ) { + mounts.add(mountSpackSecretFile(spackConfig.secretKeyFile, storageMountPath, spackConfig.secretMountPath)) + } + + V1JobBuilder builder = new V1JobBuilder() + + //metadata section + builder.withNewMetadata() + .withNamespace(namespace) + .withName(name) + .addToLabels(labels) + .endMetadata() + + //spec section + def spec = builder + .withNewSpec() + .withBackoffLimit(buildConfig.retryAttempts) + .withNewTemplate() + .withNewMetadata() + .addToAnnotations(getBuildkitAnnotations(name,singularity)) + .endMetadata() + .editOrNewSpec() + .withNodeSelector(nodeSelector) + .withServiceAccount(serviceAccount) + .withActiveDeadlineSeconds( timeout.toSeconds() ) + .withRestartPolicy("Never") + .addAllToVolumes(volumes) + + final requests = new V1ResourceRequirements() + if( requestsCpu ) + requests.putRequestsItem('cpu', new Quantity(requestsCpu)) + if( requestsMemory ) + requests.putRequestsItem('memory', new Quantity(requestsMemory)) + + // container section + final container = new V1ContainerBuilder() + .withName(name) + .withImage(containerImage) + .withVolumeMounts(mounts) + .withResources(requests) + + if( singularity ) { + container + // use 'command' to override the entrypoint of the container + .withCommand(args) + .withNewSecurityContext().withPrivileged(true).endSecurityContext() + } else { + container + //required by buildkit rootless container + .withEnv(toEnvList(BUILDKIT_FLAGS)) + // buildCommand is to set entrypoint for buildkit + .withCommand(BUILDKIT_ENTRYPOINT) + .withArgs(args) + } + + // spec section + spec.withContainers(container.build()).endSpec().endTemplate().endSpec() + + return builder.build() + } + + @Override + V1Job launchScanJob(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) { + final spec = scanJobSpec(name, containerImage, args, workDir, creds, scanConfig, nodeSelector) + return k8sClient + .batchV1Api() + .createNamespacedJob(namespace, spec, null, null, null,null) + } + + V1Job scanJobSpec(String name, String containerImage, List args, Path workDir, Path credsFile, ScanConfig scanConfig, Map nodeSelector) { + + final mounts = new ArrayList(5) + mounts.add(mountBuildStorage(workDir, storageMountPath, false)) + mounts.add(mountScanCacheDir(scanConfig.cacheDirectory, storageMountPath)) + + final volumes = new ArrayList(5) + volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) + + if( credsFile ){ + mounts.add(0, mountHostPath(credsFile, storageMountPath, Trivy.CONFIG_MOUNT_PATH)) + } + + V1JobBuilder builder = new V1JobBuilder() + + //metadata section + builder.withNewMetadata() + .withNamespace(namespace) + .withName(name) + .addToLabels(labels) + .endMetadata() + + //spec section + def spec = builder + .withNewSpec() + .withBackoffLimit(scanConfig.retryAttempts) + .withNewTemplate() + .editOrNewSpec() + .withNodeSelector(nodeSelector) + .withServiceAccount(serviceAccount) + .withRestartPolicy("Never") + .addAllToVolumes(volumes) + + final requests = new V1ResourceRequirements() + if( scanConfig.requestsCpu ) + requests.putRequestsItem('cpu', new Quantity(scanConfig.requestsCpu)) + if( scanConfig.requestsMemory ) + requests.putRequestsItem('memory', new Quantity(scanConfig.requestsMemory)) + + // container section + final container = new V1ContainerBuilder() + .withName(name) + .withImage(containerImage) + .withArgs(args) + .withVolumeMounts(mounts) + .withResources(requests) + + // spec section + spec.withContainers(container.build()).endSpec().endTemplate().endSpec() + + return builder.build() + } + protected List toEnvList(Map env) { final result = new ArrayList(env.size()) for( Map.Entry it : env ) @@ -649,6 +830,7 @@ class K8sServiceImpl implements K8sService { * Max wait time in milliseconds * @return list of pods created by the job */ + @Deprecated @Override V1PodList waitJob(V1Job job, Long timeout) { sleep 5_000 @@ -686,11 +868,11 @@ class K8sServiceImpl implements K8sService { .coreV1Api() .listNamespacedPod(namespace, null, null, null, null, "job-name=${jobName}", null, null, null, null, null, null) - if( !allPods ) + if( !allPods || !allPods.items ) return null // Find the latest created pod among the pods associated with the job - def latest = allPods.getItems().get(0) + def latest = allPods.items.get(0) for (def pod : allPods.items) { if (pod.metadata?.creationTimestamp?.isAfter(latest.metadata.creationTimestamp)) { latest = pod diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index 0469fc37b..7612a1be6 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -119,6 +119,7 @@ interface PersistenceService { return ScanResult.create( scanRecord.id, scanRecord.buildId, + scanRecord.containerImage, scanRecord.startTime, scanRecord.duration, scanRecord.status, diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index 871bbfff1..7c2831d7a 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -25,6 +25,7 @@ import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import groovy.util.logging.Slf4j +import io.seqera.wave.service.job.JobRecord import io.seqera.wave.service.scan.ScanResult import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.StringUtils @@ -37,26 +38,35 @@ import io.seqera.wave.util.StringUtils @ToString(includeNames = true, includePackage = false) @EqualsAndHashCode @CompileStatic -class WaveScanRecord { +class WaveScanRecord implements JobRecord { + String id String buildId + String containerImage Instant startTime Duration duration String status List vulnerabilities + @Override + boolean done() { + return duration != null + } + /* required by jackson deserialization - do not remove */ WaveScanRecord() {} - WaveScanRecord(String id, String buildId, Instant startTime) { + WaveScanRecord(String id, String buildId, String containerImage, Instant startTime) { this.id = StringUtils.surrealId(id) this.buildId = buildId + this.containerImage = containerImage this.startTime = startTime } - WaveScanRecord(String id, String buildId, Instant startTime, Duration duration, String status, List vulnerabilities) { + WaveScanRecord(String id, String buildId, String containerImage, Instant startTime, Duration duration, String status, List vulnerabilities) { this.id = StringUtils.surrealId(id) this.buildId = buildId + this.containerImage = containerImage this.startTime = startTime this.duration = duration this.status = status @@ -68,6 +78,7 @@ class WaveScanRecord { WaveScanRecord(String id, ScanResult scanResult) { this.id = StringUtils.surrealId(id) this.buildId = scanResult.buildId + this.containerImage = scanResult.containerImage this.startTime = scanResult.startTime this.duration = scanResult.duration this.status = scanResult.status diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index a04ecc05c..578c3fe12 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.scan +import java.nio.file.NoSuchFileException import java.time.Instant import java.util.concurrent.CompletableFuture import java.util.concurrent.ExecutorService @@ -29,8 +30,10 @@ import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.service.builder.BuildEvent -import io.seqera.wave.service.builder.ContainerBuildServiceImpl -import io.seqera.wave.service.cleanup.CleanupStrategy +import io.seqera.wave.service.job.JobHandler +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveScanRecord import jakarta.inject.Inject @@ -43,16 +46,11 @@ import static io.seqera.wave.service.builder.BuildFormat.DOCKER * @author Munish Chouhan */ @Slf4j +@Named("Scan") @Requires(property = 'wave.scan.enabled', value = 'true') @Singleton @CompileStatic -class ContainerScanServiceImpl implements ContainerScanService { - - @Inject - private ContainerBuildServiceImpl containerBuildService - - @Inject - private ScanStrategy scanStrategy +class ContainerScanServiceImpl implements ContainerScanService, JobHandler { @Inject private ScanConfig scanConfig @@ -65,7 +63,8 @@ class ContainerScanServiceImpl implements ContainerScanService { private PersistenceService persistenceService @Inject - private CleanupStrategy cleanup + private JobService jobService + @EventListener void onBuildEvent(BuildEvent event) { @@ -75,7 +74,7 @@ class ContainerScanServiceImpl implements ContainerScanService { } } catch (Exception e) { - log.warn "Unable to run the container scan - reason: ${e.message?:e}" + log.warn "Unable to run the container scan - image=${event.request.targetImage}; reason=${e.message?:e}" } } @@ -83,47 +82,83 @@ class ContainerScanServiceImpl implements ContainerScanService { void scan(ScanRequest request) { //start scanning of build container CompletableFuture - .supplyAsync(() -> launch(request), executor) - .thenAcceptAsync((scanResult) -> completeScan(scanResult), executor) + .runAsync(() -> launch(request), executor) } @Override WaveScanRecord getScanResult(String scanId) { try{ return persistenceService.loadScanRecord(scanId) - }catch (Throwable t){ - log.error "Unable to load the scan results for scanId: ${scanId}", t + } + catch (Throwable t){ + log.error("Unable to load the scan result - id=${scanId}", t) return null } } - protected ScanResult launch(ScanRequest request) { - ScanResult scanResult = null + protected void launch(ScanRequest request) { try { // create a record to mark the beginning - persistenceService.createScanRecord(new WaveScanRecord(request.id, request.buildId, Instant.now())) + persistenceService.createScanRecord(new WaveScanRecord(request.id, request.buildId, request.targetImage, Instant.now())) //launch container scan - scanResult = scanStrategy.scanContainer(request) + jobService.launchScan(request) } catch (Throwable e){ - log.warn "Unable to launch the scan results for scan id: ${request.id} - cause: ${e.message}", e + log.warn "Unable to save scan result - id=${request.id}; cause=${e.message}", e + updateScanRecord(ScanResult.failure(request)) + } + } + + + // ************************************************************** + // ** scan job handle implementation + // ************************************************************** + + @Override + WaveScanRecord getJobRecord(JobSpec job) { + persistenceService.loadScanRecord(job.recordId) + } + + @Override + void onJobCompletion(JobSpec job, WaveScanRecord scan, JobState state) { + ScanResult result + if( state.completed() ) { + try { + result = ScanResult.success(scan, TrivyResultProcessor.process(job.workDir.resolve(Trivy.OUTPUT_FILE_NAME))) + log.info("Container scan succeeded - id=${scan.id}; exit=${state.exitCode}; stdout=${state.stdout}") + } + catch (NoSuchFileException e) { + result = ScanResult.failure(scan) + log.warn("Container scan failed - id=${scan.id}; exit=${state.exitCode}; stdout=${state.stdout}; exception: NoSuchFile=${e.message}") + } } - finally{ - // cleanup build context - if( cleanup.shouldCleanup(scanResult?.isSucceeded() ? 0 : 1) ) - request.workDir?.deleteDir() + else{ + result = ScanResult.failure(scan) + log.warn("Container scan failed - id=${scan.id}; exit=${state.exitCode}; stdout=${state.stdout}") } - return scanResult + + updateScanRecord(result) } - protected void completeScan(ScanResult scanResult) { + @Override + void onJobException(JobSpec job, WaveScanRecord scan, Throwable e) { + log.error("Container scan exception - id=${scan.id} - cause=${e.getMessage()}", e) + updateScanRecord(ScanResult.failure(scan)) + } + + @Override + void onJobTimeout(JobSpec job, WaveScanRecord scan) { + log.warn("Container scan timed out - id=${scan.id}") + updateScanRecord(ScanResult.failure(scan)) + } + + protected void updateScanRecord(ScanResult result) { try{ //save scan results - persistenceService.updateScanRecord(new WaveScanRecord(scanResult.id, scanResult)) + persistenceService.updateScanRecord(new WaveScanRecord(result.id, result)) } catch (Throwable t){ - log.error "Unable to save results for scan id: ${scanResult.id}", t + log.error("Unable to save result - id=${result.id}; cause=${t.message}", t) } } - } diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy index 0a909f889..93870f195 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy @@ -21,7 +21,6 @@ package io.seqera.wave.service.scan import java.nio.file.FileAlreadyExistsException import java.nio.file.Files import java.nio.file.Path -import java.time.Instant import groovy.json.JsonOutput import groovy.transform.CompileStatic @@ -52,60 +51,50 @@ class DockerScanStrategy extends ScanStrategy { } @Override - ScanResult scanContainer(ScanRequest req) { + void scanContainer(String jobName, ScanRequest req) { log.info("Launching container scan for buildId: ${req.buildId} with scanId ${req.id}") - final startTime = Instant.now() + // create the scan dir try { - // create the scan dir - try { - Files.createDirectory(req.workDir) - } - catch (FileAlreadyExistsException e) { - log.warn("Container scan directory already exists: $e") - } - - // save the config file with docker auth credentials - Path configFile = null - if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) - } - - // outfile file name - final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) - // create the launch command - final dockerCommand = dockerWrapper(req.workDir, configFile) - final trivyCommand = List.of(scanConfig.scanImage) + scanCommand(req.targetImage, reportFile, scanConfig) - final command = dockerCommand + trivyCommand - - //launch scanning - log.debug("Container scan command: ${command.join(' ')}") - final process = new ProcessBuilder() - .command(command) - .redirectErrorStream(true) - .start() - - final exitCode = process.waitFor() - if ( exitCode != 0 ) { - log.warn("Container scan failed to scan container, it exited with code: ${exitCode} - cause: ${process.text}") - return ScanResult.failure(req, startTime) - } - else{ - log.info("Container scan completed with id: ${req.id}") - return ScanResult.success(req, startTime, TrivyResultProcessor.process(reportFile.text)) - } + Files.createDirectory(req.workDir) } - catch (Throwable e){ - log.error("Container scan failed to scan container - cause: ${e.getMessage()}", e) - return ScanResult.failure(req, startTime) + catch (FileAlreadyExistsException e) { + log.warn("Container scan directory already exists: $e") + } + + // save the config file with docker auth credentials + Path configFile = null + if( req.configJson ) { + configFile = req.workDir.resolve('config.json') + Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + } + + // outfile file name + final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) + // create the launch command + final dockerCommand = dockerWrapper(jobName, req.workDir, configFile) + final trivyCommand = List.of(scanConfig.scanImage) + scanCommand(req.targetImage, reportFile, scanConfig) + final command = dockerCommand + trivyCommand + + //launch scanning + log.debug("Container scan command: ${command.join(' ')}") + final process = new ProcessBuilder() + .command(command) + .redirectErrorStream(true) + .start() + + if( process.waitFor()!=0 ) { + throw new IllegalStateException("Unable to launch scan container - exitCode=${process.exitValue()}; output=${process.text}") } } - protected List dockerWrapper(Path scanDir, Path credsFile) { + protected List dockerWrapper(String jobName, Path scanDir, Path credsFile) { + + final wrapper = ['docker','run'] + wrapper.add('--detach') + wrapper.add('--name') + wrapper.add(jobName) - final wrapper = ['docker','run', '--rm'] - // scan work dir wrapper.add('-w') wrapper.add(scanDir.toString()) diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index 2490f14ed..00d17f6f0 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -21,8 +21,6 @@ package io.seqera.wave.service.scan import java.nio.file.FileAlreadyExistsException import java.nio.file.Files import java.nio.file.Path -import java.time.Instant -import io.micronaut.core.annotation.Nullable import groovy.json.JsonOutput import groovy.transform.CompileStatic @@ -31,6 +29,7 @@ import io.kubernetes.client.openapi.ApiException import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Property import io.micronaut.context.annotation.Requires +import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService @@ -65,11 +64,8 @@ class KubeScanStrategy extends ScanStrategy { } @Override - ScanResult scanContainer(ScanRequest req) { + void scanContainer(String jobName, ScanRequest req) { log.info("Launching container scan for buildId: ${req.buildId} with scanId ${req.id}") - final startTime = Instant.now() - - final podName = "scan-${req.id}" try{ // create the scan dir try { @@ -90,28 +86,11 @@ class KubeScanStrategy extends ScanStrategy { final trivyCommand = scanCommand(req.targetImage, reportFile, scanConfig) final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.workDir, configFile, scanConfig, selector) - final exitCode = k8sService.waitPodCompletion(pod, scanConfig.timeout.toMillis()) - if( exitCode==0 ) { - log.info("Container scan completed for id: ${req.id}") - return ScanResult.success(req, startTime, TrivyResultProcessor.process(reportFile.text)) - } - else{ - final stdout = k8sService.logsPod(pod) - log.info("Container scan failed for scan id: ${req.id} - stdout: $stdout") - return ScanResult.failure(req, startTime) - } + k8sService.launchScanJob(jobName, scanConfig.scanImage, trivyCommand, req.workDir, configFile, scanConfig, selector) } catch (ApiException e) { throw new BadRequestException("Unexpected scan failure: ${e.responseBody}", e) } - catch (Throwable e){ - log.error("Error while scanning with id: ${req.id} - cause: ${e.getMessage()}", e) - return ScanResult.failure(req, startTime) - } - finally { - cleanup(podName) - } } void cleanup(String podName) { diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index b00a8011d..3fec1d4b9 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.service.scan import java.nio.file.Path +import java.time.Instant import groovy.transform.Canonical import groovy.transform.CompileStatic @@ -38,10 +39,11 @@ class ScanRequest { final String targetImage final ContainerPlatform platform final Path workDir + final Instant creationTime static ScanRequest fromBuild(BuildRequest request) { final id = request.scanId final workDir = request.workDir.resolveSibling("scan-${id}") - return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir) + return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir, Instant.now()) } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanResult.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanResult.groovy index 4f8dbea7c..bf7d7ef72 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanResult.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanResult.groovy @@ -31,6 +31,7 @@ import groovy.transform.CompileStatic */ import groovy.transform.ToString +import io.seqera.wave.service.persistence.WaveScanRecord @ToString(includePackage = false, includeNames = true) @Canonical @@ -42,14 +43,16 @@ class ScanResult { String id String buildId + String containerImage Instant startTime Duration duration String status List vulnerabilities - private ScanResult(String id, String buildId, Instant startTime, Duration duration, String status, List vulnerabilities) { + private ScanResult(String id, String buildId, String containerImage, Instant startTime, Duration duration, String status, List vulnerabilities) { this.id = id this.buildId = buildId + this.containerImage = containerImage this.startTime = startTime this.duration = duration this.status = status @@ -60,15 +63,19 @@ class ScanResult { boolean isCompleted() { duration!=null } - static ScanResult success(ScanRequest request, Instant startTime, List vulnerabilities){ - return new ScanResult(request.id, request.buildId, startTime, Duration.between(startTime, Instant.now()), SUCCEEDED, vulnerabilities) + static ScanResult success(WaveScanRecord scan, List vulnerabilities){ + return new ScanResult(scan.id, scan.buildId, scan.containerImage, scan.startTime, Duration.between(scan.startTime, Instant.now()), SUCCEEDED, vulnerabilities) } - static ScanResult failure(ScanRequest request, Instant startTime){ - return new ScanResult(request.id, request.buildId, startTime, Duration.between(startTime, Instant.now()), FAILED, List.of()) + static ScanResult failure(WaveScanRecord scan){ + return new ScanResult(scan.id, scan.buildId, scan.containerImage, scan.startTime, Duration.between(scan.startTime, Instant.now()), FAILED, List.of()) } - static ScanResult create(String scanId, String buildId, Instant startTime, Duration duration1, String status, List vulnerabilities){ - return new ScanResult(scanId, buildId, startTime, duration1, status, vulnerabilities) + static ScanResult failure(ScanRequest request){ + return new ScanResult(request.id, request.buildId, request.targetImage, request.creationTime, Duration.between(request.creationTime, Instant.now()), FAILED, List.of()) + } + + static ScanResult create(String scanId, String buildId, String containerImage, Instant startTime, Duration duration1, String status, List vulnerabilities){ + return new ScanResult(scanId, buildId, containerImage, startTime, duration1, status, vulnerabilities) } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy index 3938af4ec..e333141bc 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy @@ -22,6 +22,7 @@ import java.nio.file.Path import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires import io.seqera.wave.configuration.ScanConfig /** * Implements ScanStrategy for Docker @@ -30,9 +31,10 @@ import io.seqera.wave.configuration.ScanConfig */ @Slf4j @CompileStatic +@Requires(bean = ScanConfig) abstract class ScanStrategy { - abstract ScanResult scanContainer(ScanRequest request) + abstract void scanContainer(String jobName, ScanRequest request) protected List scanCommand(String targetImage, Path outputFile, ScanConfig config) { def cmd = ['--quiet', diff --git a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy index 58e21675d..71ea0d720 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.service.scan +import java.nio.file.Path + import groovy.json.JsonSlurper import groovy.util.logging.Slf4j import io.seqera.wave.exception.ScanRuntimeException @@ -29,7 +31,11 @@ import io.seqera.wave.exception.ScanRuntimeException @Slf4j class TrivyResultProcessor { - static List process(String trivyResult){ + static List process(Path reportFile) { + process(reportFile.getText()) + } + + static List process(String trivyResult) { final slurper = new JsonSlurper() try{ final jsonMap = slurper.parseText(trivyResult) as Map diff --git a/src/main/groovy/io/seqera/wave/tower/auth/JwtConfig.groovy b/src/main/groovy/io/seqera/wave/tower/auth/JwtConfig.groovy index e6fd0b7a6..9e040da47 100644 --- a/src/main/groovy/io/seqera/wave/tower/auth/JwtConfig.groovy +++ b/src/main/groovy/io/seqera/wave/tower/auth/JwtConfig.groovy @@ -23,10 +23,12 @@ import java.time.Duration import groovy.transform.CompileStatic import groovy.transform.ToString import io.micronaut.context.annotation.Value +import io.seqera.wave.util.DurationUtils import jakarta.inject.Singleton /** - * + * Model JWT refreshing service config + * * @author Paolo Di Tommaso */ @Singleton @@ -59,4 +61,8 @@ class JwtConfig { @Value('${wave.jwt.monitor.count:10}') int monitorCount + Duration getMonitorDelayRandomized() { + DurationUtils.randomDuration(monitorDelay, 0.4f) + } + } diff --git a/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy b/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy index 41262b847..0fee156f7 100644 --- a/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy +++ b/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy @@ -63,7 +63,11 @@ class JwtMonitor implements Runnable { @PostConstruct private init() { log.info "Creating JWT heartbeat - $jwtConfig" - scheduler.scheduleAtFixedRate(jwtConfig.monitorDelay, jwtConfig.monitorInterval, this) + // use randomize initial delay to prevent multiple replicas running at the same time + scheduler.scheduleAtFixedRate( + jwtConfig.monitorDelayRandomized, + jwtConfig.monitorInterval, + this ) } void run() { diff --git a/src/main/groovy/io/seqera/wave/util/DurationUtils.groovy b/src/main/groovy/io/seqera/wave/util/DurationUtils.groovy new file mode 100644 index 000000000..1d5f13034 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/util/DurationUtils.groovy @@ -0,0 +1,61 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.util + +import java.time.Duration +import java.util.concurrent.ThreadLocalRandom + +import groovy.transform.CompileStatic +/** + * Utility functions for handling duration + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class DurationUtils { + + static Duration randomDuration(Duration min, Duration max) { + if (min > max) { + throw new IllegalArgumentException("Min duration must be less than or equal to max duration") + } + + long minNanos = min.toNanos() + long maxNanos = max.toNanos() + long randomNanos = ThreadLocalRandom.current().nextLong(minNanos, maxNanos + 1) + + return Duration.ofNanos(randomNanos) + } + + static Duration randomDuration(Duration reference, float intervalPercentage) { + if (intervalPercentage < 0 || intervalPercentage > 1) { + throw new IllegalArgumentException("Interval percentage must be between 0 and 1") + } + + long refNanos = reference.toNanos() + long intervalNanos = (long)(refNanos * intervalPercentage) + + long minNanos = Math.max(0, refNanos - intervalNanos) + long maxNanos = refNanos + intervalNanos + + long randomNanos = ThreadLocalRandom.current().nextLong(minNanos, maxNanos + 1) + + return Duration.ofNanos(randomNanos) + } + +} diff --git a/src/main/groovy/io/seqera/wave/util/Retryable.groovy b/src/main/groovy/io/seqera/wave/util/Retryable.groovy index 2d7941231..db4a6f9eb 100644 --- a/src/main/groovy/io/seqera/wave/util/Retryable.groovy +++ b/src/main/groovy/io/seqera/wave/util/Retryable.groovy @@ -20,7 +20,6 @@ package io.seqera.wave.util import java.net.http.HttpResponse import java.time.Duration -import java.time.temporal.ChronoUnit import java.util.function.Consumer import java.util.function.Predicate @@ -34,7 +33,6 @@ import dev.failsafe.function.CheckedSupplier import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j - /** * Implements a retry strategy based on Fail safe * @@ -49,6 +47,7 @@ class Retryable { Duration getMaxDelay() int getMaxAttempts() double getJitter() + double getMultiplier() } @Canonical @@ -68,12 +67,13 @@ class Retryable { static final private int DEFAULT_MAX_ATTEMPTS = 5 static final private double DEFAULT_JITTER = 0.25d static final private Predicate DEFAULT_CONDITION = (e -> e instanceof IOException) as Predicate + static final private double DEFAULT_MULTIPLIER = 2.0 private Config config private Predicate condition - private Consumer retryEvent + private Consumer> retryEvent private Predicate handleResult @@ -92,7 +92,7 @@ class Retryable { return this } - Retryable onRetry(Consumer event) { + Retryable onRetry(Consumer> event) { this.retryEvent = event return this } @@ -121,10 +121,11 @@ class Retryable { final a = config.maxAttempts ?: DEFAULT_MAX_ATTEMPTS final j = config.jitter ?: DEFAULT_JITTER final c = condition ?: DEFAULT_CONDITION + final r = config.multiplier ?: DEFAULT_MULTIPLIER final RetryPolicyBuilder policy = RetryPolicy.builder() .handleIf(c) - .withBackoff(d.toMillis(), m.toMillis(), ChronoUnit.MILLIS) + .withBackoff(d, m, r) .withMaxAttempts(a) .withJitter(j) .onRetry(retry0) diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index b47e4812b..297fe0279 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -16,6 +16,10 @@ wave: interval: '10s' monitor: interval: '5s' + cleanup: + succeeded: '10s' + failed: '45s' + strategy: 'onsuccess' --- endpoints: metrics: @@ -40,5 +44,7 @@ logger: io.seqera.wave.service.data: DEBUG io.seqera.wave.service.pairing: TRACE io.seqera.wave.tower.client.connector: TRACE + io.seqera.wave.service.job: TRACE + io.seqera.wave.service.k8s.K8sServiceImpl: TRACE # io.seqera.wave.tower.auth: 'TRACE' ... diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7c795d5f5..e5a3b0832 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -7,7 +7,7 @@ micronaut: # Using `expire-after-read` can cause an entry to be retained in the cache more than expected if it is hit continuously # with a frequency shorter than the declared cache period. cache-tower-client: - expire-after-write: 20s + expire-after-write: 60s record-stats: true cache-registry-proxy: expire-after-write: 20s @@ -75,7 +75,7 @@ wave: timeout: 900s status: delay: 5s - duration: 1d + duration: 90m httpclient: retry: delay: '500ms' @@ -83,9 +83,9 @@ wave: multiplier: '1.75' scan: image: - name: aquasec/trivy:0.53.0 + name: aquasec/trivy:0.55.0 blobCache: - s5cmdImage: cr.seqera.io/public/wave/s5cmd:v2.2.2 + s5cmdImage: public.cr.seqera.io/wave/s5cmd:v2.2.2 --- jackson: serialization: diff --git a/src/main/resources/io/seqera/wave/scan-view.hbs b/src/main/resources/io/seqera/wave/scan-view.hbs index ba345dd7c..ae91aa05e 100644 --- a/src/main/resources/io/seqera/wave/scan-view.hbs +++ b/src/main/resources/io/seqera/wave/scan-view.hbs @@ -44,6 +44,10 @@ Build ID {{build_id}} + + Container image + {{scan_container_image}} + Status {{scan_status}} diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryLookupServiceTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryLookupServiceTest.groovy index 65f4525da..074950b95 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryLookupServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryLookupServiceTest.groovy @@ -29,7 +29,8 @@ import jakarta.inject.Inject @MicronautTest class RegistryLookupServiceTest extends Specification { - @Inject RegistryLookupServiceImpl service + @Inject + RegistryLookupServiceImpl service def 'should find registry realm' () { given: diff --git a/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy index f71941ea6..aacb51586 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy @@ -56,36 +56,6 @@ class BuildConfigTest extends Specification { config.singularityImage( ContainerPlatform.of('arm64') ) == 'bar' } - @Unroll - def 'should validate initial duration' () { - given: - def config = new BuildConfig(defaultTimeout:DEFAULT, trustedTimeout:TRUSTED) - expect: - config.statusInitialDelay == EXPECTED - - where: - DEFAULT | TRUSTED | EXPECTED - Duration.ofMillis(1000) | Duration.ofMillis(1000) | Duration.ofMillis(Math.round(1000 * 2.5)) - Duration.ofMinutes(15) | Duration.ofMinutes(15) | Duration.ofMillis(Math.round( 37.5 * 60 * 1000)) - Duration.ofMinutes(15) | Duration.ofMinutes(25) | Duration.ofMillis(Math.round( 2.5 * 15 * 60 * 1000)) - Duration.ofMinutes(15) | Duration.ofMinutes(25) | Duration.ofMillis(Math.round( 1.5 * 25 * 60 * 1000)) - Duration.ofMinutes(15) | Duration.ofMinutes(30) | Duration.ofMillis(Math.round( 1.5 * 30 * 60 * 1000)) - } - - @Unroll - def 'should validate await duration' () { - given: - def config = new BuildConfig(defaultTimeout:DEFAULT, trustedTimeout:TRUSTED) - expect: - config.statusAwaitDuration == EXPECTED - - where: - DEFAULT | TRUSTED | EXPECTED - Duration.ofMillis(1000) | Duration.ofMillis(1000) | Duration.ofMillis(Math.round(1000 * 2.1)) - Duration.ofMinutes(15) | Duration.ofMinutes(15) | Duration.ofMillis(Math.round( 2.1 * 15 * 60 * 1000)) - Duration.ofMinutes(15) | Duration.ofMinutes(25) | Duration.ofMillis(Math.round( 2.1 * 15 * 60 * 1000)) - Duration.ofMinutes(15) | Duration.ofMinutes(30) | Duration.ofMillis(Math.round( 1.1 * 30 * 60 * 1000)) - } @Unroll def 'should validate build max duration' () { diff --git a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy index 15344253e..fc69fc7b2 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy @@ -59,7 +59,6 @@ class BuildControllerTest extends Specification { Mock(BuildLogService) } - @Inject @Client("/") HttpClient client diff --git a/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy index 67201b912..dc4c46b49 100644 --- a/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy @@ -18,7 +18,7 @@ package io.seqera.wave.controller -import spock.lang.Shared + import spock.lang.Specification import spock.lang.Timeout @@ -26,7 +26,6 @@ import java.time.Instant import java.util.concurrent.CompletableFuture import java.util.concurrent.TimeUnit -import io.micronaut.context.ApplicationContext import io.micronaut.http.HttpRequest import io.micronaut.http.HttpResponse import io.micronaut.http.HttpStatus @@ -61,10 +60,6 @@ class CustomImageControllerTest extends Specification implements DockerRegistryC @Client("/") HttpClient client; - @Inject - @Shared - ApplicationContext applicationContext - BuildResult expected boolean resolveImageAsync = false @@ -125,7 +120,7 @@ class CustomImageControllerTest extends Specification implements DockerRegistryC } def setupSpec() { - initRegistryContainer(applicationContext) + initRegistryContainer() } void 'should fails head manifest when no image'() { diff --git a/src/test/groovy/io/seqera/wave/controller/ErrorHandlingTest.groovy b/src/test/groovy/io/seqera/wave/controller/ErrorHandlingTest.groovy index 2778470ae..70ff7d61b 100644 --- a/src/test/groovy/io/seqera/wave/controller/ErrorHandlingTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ErrorHandlingTest.groovy @@ -39,7 +39,7 @@ class ErrorHandlingTest extends Specification { @Inject @Client("/") - HttpClient client; + HttpClient client void 'should handle an error'() { when: diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy index 36a7eba9e..ef37f10ba 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy @@ -18,11 +18,10 @@ package io.seqera.wave.controller -import spock.lang.Shared + import spock.lang.Specification import groovy.json.JsonSlurper -import io.micronaut.context.ApplicationContext import io.micronaut.http.HttpRequest import io.micronaut.http.HttpResponse import io.micronaut.http.HttpStatus @@ -43,17 +42,13 @@ class RegistryControllerLocalTest extends Specification implements DockerRegistr @Inject @Client("/") - HttpClient client; - - @Inject - @Shared - ApplicationContext applicationContext + HttpClient client @Inject ManifestCacheStore storage def setupSpec() { - initRegistryContainer(applicationContext) + initRegistryContainer() } void 'should get manifest'() { diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerLookupFailureTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLookupFailureTest.groovy new file mode 100644 index 000000000..9b97d1787 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLookupFailureTest.groovy @@ -0,0 +1,73 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.controller + +import spock.lang.Specification + +import io.micronaut.http.HttpRequest +import io.micronaut.http.MediaType +import io.micronaut.http.client.HttpClient +import io.micronaut.http.client.annotation.Client +import io.micronaut.http.client.exceptions.HttpClientResponseException +import io.micronaut.test.annotation.MockBean +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.auth.RegistryLookupService +import io.seqera.wave.exception.RegistryForwardException +import io.seqera.wave.model.ContentType +import io.seqera.wave.test.DockerRegistryContainer +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class RegistryControllerLookupFailureTest extends Specification implements DockerRegistryContainer { + + @Inject + @Client("/") + HttpClient client + + @MockBean(RegistryLookupService) + RegistryLookupService lookupService = Mock(RegistryLookupService) + + def setupSpec() { + initRegistryContainer() + } + + def 'should report registry lookup' () { + when: + def MESSAGE = 'Registry response body here' + def HEADERS = ['x-foo': ['this'], 'x-bar': ['that']] + lookupService.lookup(_) >> { throw new RegistryForwardException('Oops.. something went wrong', 400, MESSAGE, HEADERS) } + and: + HttpRequest request = HttpRequest.GET("/v2/library/hello-world/manifests/sha256:53641cd209a4fecfc68e21a99871ce8c6920b2e7502df0a20671c6fccc73a7c6").headers({ h-> + h.add('Accept', ContentType.DOCKER_MANIFEST_V2_TYPE) + h.add('Accept', ContentType.DOCKER_MANIFEST_V1_JWS_TYPE) + h.add('Accept', MediaType.APPLICATION_JSON) + }) + client.toBlocking().exchange(request,String) + then: + def e = thrown(HttpClientResponseException) + e.status.code == 400 + e.response.body() == MESSAGE + e.response.headers.get('x-foo') == 'this' + e.response.headers.get('x-bar') == 'that' + } + +} diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy index 33b52493d..cc3439c23 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy @@ -18,10 +18,9 @@ package io.seqera.wave.controller -import spock.lang.Shared + import spock.lang.Specification -import io.micronaut.context.ApplicationContext import io.micronaut.context.annotation.Requires import io.micronaut.http.HttpRequest import io.micronaut.http.HttpStatus @@ -67,17 +66,13 @@ class RegistryControllerPullLimitTest extends Specification implements DockerReg @Inject @Client("/") - HttpClient client; - - @Inject - @Shared - ApplicationContext applicationContext + HttpClient client @Inject ManifestCacheStore storage def setupSpec() { - initRegistryContainer(applicationContext) + initRegistryContainer() } void 'should get manifest'() { diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy index 56e1daf22..087960ae2 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy @@ -18,11 +18,14 @@ package io.seqera.wave.controller +import spock.lang.Shared import spock.lang.Specification import spock.lang.Timeout +import java.time.Duration +import java.time.Instant + import io.micronaut.context.ApplicationContext -import io.micronaut.core.io.socket.SocketUtils import io.micronaut.http.HttpRequest import io.micronaut.http.HttpResponse import io.micronaut.http.HttpStatus @@ -30,57 +33,57 @@ import io.micronaut.http.MediaType import io.micronaut.http.client.HttpClient import io.micronaut.http.client.exceptions.HttpClientResponseException import io.micronaut.runtime.server.EmbeddedServer -import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.exchange.RegistryErrorResponse import io.seqera.wave.model.ContentType +import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.builder.BuildCacheStore +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildResult +import io.seqera.wave.service.builder.BuildStoreEntry +import io.seqera.wave.service.job.JobFactory +import io.seqera.wave.service.job.JobQueue +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.token.TokenCacheStore import io.seqera.wave.storage.ManifestCacheStore import io.seqera.wave.test.DockerRegistryContainer import io.seqera.wave.test.RedisTestContainer -import redis.clients.jedis.Jedis +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User /** * * @author Jorge Aguilera */ -@MicronautTest -class RegistryControllerRedisTest extends Specification implements DockerRegistryContainer, RedisTestContainer{ +class RegistryControllerRedisTest extends Specification implements DockerRegistryContainer, RedisTestContainer { - EmbeddedServer embeddedServer + @Shared + ApplicationContext applicationContext + @Shared int port - Jedis jedis - def setup() { - port = SocketUtils.findAvailableTcpPort() - embeddedServer = ApplicationContext.run(EmbeddedServer, [ + EmbeddedServer server = ApplicationContext.run(EmbeddedServer, [ REDIS_HOST : redisHostName, REDIS_PORT : redisPort, 'wave.build.timeout':'2s', - 'wave.build.trusted-timeout':'2s', - 'micronaut.server.port': port, - 'micronaut.http.services.default.url' : "http://localhost:$port".toString(), - ], 'test', 'h2', 'redis') - - jedis = new Jedis(redisHostName, redisPort as int) - initRegistryContainer(applicationContext) + 'wave.build.trusted-timeout':'2s' + ], 'test', 'redis') + and: + port = server.port + applicationContext = server.getApplicationContext() } def cleanup(){ - jedis.flushAll() - jedis.close() - } - - ApplicationContext getApplicationContext() { - embeddedServer.applicationContext + applicationContext.close() } void 'should get manifest'() { given: - HttpClient client = applicationContext.createBean(HttpClient) + HttpClient client = applicationContext.getBean(HttpClient) ManifestCacheStore storage = applicationContext.getBean(ManifestCacheStore) when: - HttpRequest request = HttpRequest.GET("http://localhost:$port/v2/library/hello-world/manifests/sha256:53641cd209a4fecfc68e21a99871ce8c6920b2e7502df0a20671c6fccc73a7c6").headers({h-> + HttpRequest request = HttpRequest.GET("http://localhost:${port}/v2/library/hello-world/manifests/sha256:53641cd209a4fecfc68e21a99871ce8c6920b2e7502df0a20671c6fccc73a7c6").headers({h-> h.add('Accept', ContentType.DOCKER_MANIFEST_V2_TYPE) h.add('Accept', ContentType.DOCKER_MANIFEST_V1_JWS_TYPE) h.add('Accept', MediaType.APPLICATION_JSON) @@ -93,30 +96,33 @@ class RegistryControllerRedisTest extends Specification implements DockerRegistr response.getContentType().get().getName() == 'application/vnd.oci.image.index.v1+json' response.header('docker-content-digest') == 'sha256:53641cd209a4fecfc68e21a99871ce8c6920b2e7502df0a20671c6fccc73a7c6' response.getContentLength() == 10242 - - when: - storage.clear() - - and: - response = client.toBlocking().exchange(request,String) - - then: - response.status() == HttpStatus.OK - and: - response.body().indexOf('"schemaVersion":') != -1 - response.getContentType().get().getName() == 'application/vnd.oci.image.index.v1+json' - response.getContentLength() == 10242 + } - @Timeout(15) - void 'should render a timeout when build failed'() { + @Timeout(30) + void 'should return a timeout when build failed'() { given: - HttpClient client = applicationContext.createBean(HttpClient) + def client = applicationContext.createBean(HttpClient) + def buildCacheStore = applicationContext.getBean(BuildCacheStore) + def tokenCacheStore = applicationContext.getBean(TokenCacheStore) + def jobQueue = applicationContext.getBean(JobQueue) + def jobFactory = applicationContext.getBean(JobFactory) + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'library/hello-world', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofSeconds(5) + ) + def entry = new BuildStoreEntry(req, res) + def containerRequestData = new ContainerRequestData(new PlatformId(new User(id:1)), "library/hello-world") and: - jedis.set("wave-tokens/v1:1234", '{"containerImage":"library/hello-world"}') - jedis.set("wave-build/v1:library/hello-world", '{"containerImage":"library/hello-world"}') + tokenCacheStore.put("1234", containerRequestData) + buildCacheStore.put("library/hello-world", entry) + jobQueue.offer(jobFactory.build(req)) + when: - HttpRequest request = HttpRequest.GET("http://localhost:$port/v2/wt/1234/library/hello-world/manifests/latest").headers({h-> + HttpRequest request = HttpRequest.GET("http://localhost:${port}/v2/wt/1234/library/hello-world/manifests/latest").headers({h-> h.add('Accept', ContentType.DOCKER_MANIFEST_V2_TYPE) h.add('Accept', ContentType.DOCKER_MANIFEST_V1_JWS_TYPE) h.add('Accept', MediaType.APPLICATION_JSON) @@ -124,7 +130,9 @@ class RegistryControllerRedisTest extends Specification implements DockerRegistr client.toBlocking().exchange(request,String) then: final exception = thrown(HttpClientResponseException) - RegistryErrorResponse error = exception.response.getBody(RegistryErrorResponse).get() - error.errors.get(0).message.contains('Build of container \'library/hello-world\' timed out') + RegistryErrorResponse registryError = exception.response.getBody(RegistryErrorResponse).get() + def error = registryError.errors.get(0) + error.message.contains('Container image build timed out \'library/hello-world\'') + error.code == 'UNKNOWN' } } diff --git a/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy index cd60e9f4d..e1e01061f 100644 --- a/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy @@ -54,6 +54,7 @@ class ScanControllerTest extends Specification { def duration = Duration.ofMinutes(2) def scanId = '123' def buildId = "testbuildid" + def containerImage = "testcontainerimage" def scanVulnerability = new ScanVulnerability( "id1", "low", @@ -68,6 +69,7 @@ class ScanControllerTest extends Specification { new ScanResult( scanId, buildId, + containerImage, startTime, duration, 'SUCCEEDED', diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index e4447cbc1..0a3e2d040 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -37,9 +37,14 @@ import io.seqera.wave.service.logs.BuildLogServiceImpl import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord +import io.seqera.wave.service.persistence.WaveScanRecord +import io.seqera.wave.service.scan.ScanResult +import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User import jakarta.inject.Inject +import static io.seqera.wave.util.DataTimeUtils.formatDuration +import static io.seqera.wave.util.DataTimeUtils.formatTimestamp /** * * @author Paolo Di Tommaso @@ -52,7 +57,6 @@ class ViewControllerTest extends Specification { Mock(BuildLogService) } - @Inject @Client("/") HttpClient client @@ -318,4 +322,29 @@ class ViewControllerTest extends Specification { binding.build_in_progress == false binding.build_failed == true } + + def 'should render in success scan page' () { + given: + def controller = new ViewController(serverUrl: 'http://foo.com', buildLogService: buildLogService) + and: + def record = new WaveScanRecord( + id: '12345', + buildId: '12345', + containerImage: 'docker.io/some:image', + startTime: Instant.now(), + duration: Duration.ofMinutes(1), + status: ScanResult.SUCCEEDED, + vulnerabilities: [new ScanVulnerability('cve-1', 'HIGH', 'test vul', 'testpkg', '1.0.0', '1.1.0', 'http://vul/cve-1')] ) + def result = ScanResult.success(record, record.vulnerabilities) + when: + def binding = controller.makeScanViewBinding(result) + then: + binding.scan_id == '12345' + binding.scan_container_image == 'docker.io/some:image' + binding.scan_time == formatTimestamp(result.startTime) + binding.scan_duration == formatDuration(result.duration) + binding.scan_succeeded + binding.vulnerabilities == [new ScanVulnerability(id:'cve-1', severity:'HIGH', title:'test vul', pkgName:'testpkg', installedVersion:'1.0.0', fixedVersion:'1.1.0', primaryUrl:'http://vul/cve-1')] + binding.build_url == 'http://foo.com/view/builds/12345' + } } diff --git a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy index afe0fb149..9cee2c36f 100644 --- a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy @@ -32,7 +32,8 @@ import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult -import io.seqera.wave.service.job.JobId +import io.seqera.wave.service.builder.BuildStoreEntry +import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.pairing.socket.msg.PairingHeartbeat import io.seqera.wave.service.pairing.socket.msg.PairingResponse import io.seqera.wave.service.pairing.socket.msg.ProxyHttpRequest @@ -372,19 +373,58 @@ class MoshiEncodingStrategyTest extends Specification { } - def 'should encode and decode job request' () { + def 'should encode and decode build job spec' () { given: - def encoder = new MoshiEncodeStrategy() { } + def encoder = new MoshiEncodeStrategy() { } and: - def job = new JobId(JobId.Type.Transfer, '123-abc', Instant.now()) + def ts = Instant.parse('2024-08-18T19:23:33.650722Z') + and: + def build = JobSpec.build( + 'docker.io/foo:bar', + '12345', + ts, + Duration.ofMinutes(1), + Path.of('/some/path') ) when: - def json = encoder.encode(job) + def json = encoder.encode(build) + println json + and: + def copy = encoder.decode(json) + then: + copy == build + } + + def 'should encode and decode build store entry' () { + given: + def encoder = new MoshiEncodeStrategy() { } + def context = new BuildContext('http://foo.com', '12345', 100, '67890') + and: + def res = BuildResult.completed('1', 2, 'Oops', Instant.now(), null) + def req = new BuildRequest( + containerId: '12345', + containerFile: 'from foo', + condaFile: 'conda spec', + spackFile: 'spack spec', + workspace: Path.of("/some/path"), + targetImage: 'docker.io/some:image:12345', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'cacherepo', + ip: "1.2.3.4", + configJson: '{"config":"json"}', + scanId: 'scan12345', + buildContext: context ) + .withBuildId('1') + and: + def entry = new BuildStoreEntry(req, res) + when: + def json = encoder.encode(entry) and: def copy = encoder.decode(json) then: - copy.getClass() == JobId + copy.getClass() == entry.getClass() and: - copy == job + copy == entry } } diff --git a/src/test/groovy/io/seqera/wave/exchange/RegistryErrorResponseTest.groovy b/src/test/groovy/io/seqera/wave/exchange/RegistryErrorResponseTest.groovy new file mode 100644 index 000000000..d5ee03384 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/exchange/RegistryErrorResponseTest.groovy @@ -0,0 +1,47 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.exchange + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class RegistryErrorResponseTest extends Specification { + + def 'should create an error response' () { + when: + def resp = new RegistryErrorResponse('FOO', 'Oops something went wrong') + then: + resp.getErrors().size() + resp.getErrors().first() == new RegistryErrorResponse.RegistryError('FOO', 'Oops something went wrong') + } + + def 'should create from json' () { + given: + def json = '{"errors":[{"code":"DENIED","message":"Unauthenticated request. Unauthenticated requests do not have permission \\"artifactregistry.repositories.downloadArtifacts\\" on resource \\"projects/wired-height-305919/locations/us-central1/repositories/fsdx-docker-dev\\" (or it may not exist)"}]}' + when: + def resp = RegistryErrorResponse.parse(json) + then: + resp.errors.size() == 1 + resp.errors[0].code == 'DENIED' + resp.errors[0].message == 'Unauthenticated request. Unauthenticated requests do not have permission \"artifactregistry.repositories.downloadArtifacts\" on resource \"projects/wired-height-305919/locations/us-central1/repositories/fsdx-docker-dev\" (or it may not exist)' + } +} diff --git a/src/test/groovy/io/seqera/wave/memstore/range/RedisRangeProviderTest.groovy b/src/test/groovy/io/seqera/wave/memstore/range/RedisRangeProviderTest.groovy index 656659757..1f73c5fe4 100644 --- a/src/test/groovy/io/seqera/wave/memstore/range/RedisRangeProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/memstore/range/RedisRangeProviderTest.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.memstore.range +import spock.lang.Shared import spock.lang.Specification import io.micronaut.context.ApplicationContext @@ -29,8 +30,10 @@ import io.seqera.wave.test.RedisTestContainer */ class RedisRangeProviderTest extends Specification implements RedisTestContainer { + @Shared ApplicationContext applicationContext + @Shared RedisRangeProvider provider def setup() { @@ -42,6 +45,11 @@ class RedisRangeProviderTest extends Specification implements RedisTestContainer sleep(500) // workaround to wait for Redis connection } + def cleanup() { + applicationContext.close() + } + + def 'should add and get elements' () { given: provider.add('foo', 'x', 1) diff --git a/src/test/groovy/io/seqera/wave/proxy/ProxyClientWithLocalRegistryTest.groovy b/src/test/groovy/io/seqera/wave/proxy/ProxyClientWithLocalRegistryTest.groovy index d812c83f7..41d7ffd9b 100644 --- a/src/test/groovy/io/seqera/wave/proxy/ProxyClientWithLocalRegistryTest.groovy +++ b/src/test/groovy/io/seqera/wave/proxy/ProxyClientWithLocalRegistryTest.groovy @@ -18,17 +18,15 @@ package io.seqera.wave.proxy -import spock.lang.Shared + import spock.lang.Specification -import io.micronaut.context.ApplicationContext import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.auth.RegistryAuthService import io.seqera.wave.auth.RegistryCredentialsProvider import io.seqera.wave.auth.RegistryLookupService import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.http.HttpClientFactory -import io.seqera.wave.proxy.ProxyClient import io.seqera.wave.test.DockerRegistryContainer import jakarta.inject.Inject /** @@ -38,10 +36,6 @@ import jakarta.inject.Inject @MicronautTest class ProxyClientWithLocalRegistryTest extends Specification implements DockerRegistryContainer{ - @Inject - @Shared - ApplicationContext applicationContext - @Inject RegistryLookupService lookupService @Inject RegistryAuthService loginService @Inject RegistryCredentialsProvider credentialsProvider @@ -49,7 +43,7 @@ class ProxyClientWithLocalRegistryTest extends Specification implements DockerRe @Inject HttpClientConfig httpConfig def setupSpec() { - initRegistryContainer(applicationContext) + initRegistryContainer() } def 'should call target blob' () { diff --git a/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy b/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy index 2a288c150..6bff0952a 100644 --- a/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy +++ b/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy @@ -59,6 +59,9 @@ class BuildServiceRateLimitTest extends Specification { configuration = applicationContext.getBean(RateLimiterConfig) } + def cleanupSpec() { + applicationContext.close() + } def 'should not allow more auth builds than rate limit' () { given: diff --git a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayMemoryRateLimiterTest.groovy b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayMemoryRateLimiterTest.groovy index 281cf691b..c804774dc 100644 --- a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayMemoryRateLimiterTest.groovy +++ b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayMemoryRateLimiterTest.groovy @@ -42,6 +42,10 @@ class SpillwayMemoryRateLimiterTest extends Specification { rateLimiter = applicationContext.getBean(SpillwayRateLimiter) } + def cleanup() { + applicationContext.close() + } + void "can acquire 1 auth resource"() { when: rateLimiter.acquireBuild(new AcquireRequest("test", null)) diff --git a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRedisRateLimiterTest.groovy b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRedisRateLimiterTest.groovy index 2471e85b8..b3542c79f 100644 --- a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRedisRateLimiterTest.groovy +++ b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRedisRateLimiterTest.groovy @@ -18,7 +18,7 @@ package io.seqera.wave.ratelimit - +import spock.lang.Shared import spock.lang.Specification import io.micronaut.context.ApplicationContext @@ -34,10 +34,13 @@ import redis.clients.jedis.Jedis */ class SpillwayRedisRateLimiterTest extends Specification implements RedisTestContainer { + @Shared ApplicationContext applicationContext + @Shared SpillwayRateLimiter rateLimiter + @Shared Jedis jedis def setup() { @@ -47,11 +50,12 @@ class SpillwayRedisRateLimiterTest extends Specification implements RedisTestCon ], 'test', 'redis','rate-limit') rateLimiter = applicationContext.getBean(SpillwayRateLimiter) jedis = new Jedis(redisHostName, redisPort as int) + jedis.flushAll() } def cleanup(){ - jedis.flushAll() jedis.close() + applicationContext.close() } void "can acquire 1 auth resource"() { diff --git a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy index f4ece7b3b..8aff6637c 100644 --- a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy @@ -18,13 +18,12 @@ package io.seqera.wave.ratelimit -import spock.lang.Shared + import spock.lang.Specification import java.util.concurrent.atomic.AtomicInteger import groovy.json.JsonSlurper -import io.micronaut.context.ApplicationContext import io.micronaut.http.HttpRequest import io.micronaut.http.MediaType import io.micronaut.http.client.HttpClient @@ -48,10 +47,6 @@ class SpillwayRegistryControllerTest extends Specification implements DockerRegi @Client("/") HttpClient client; - @Inject - @Shared - ApplicationContext applicationContext - @Inject RateLimiterConfig configuration @@ -66,7 +61,7 @@ class SpillwayRegistryControllerTest extends Specification implements DockerRegi } def setupSpec() { - initRegistryContainer(applicationContext) + initRegistryContainer() } void 'should check rate limit in ip of anonymous manifest'() { diff --git a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy index 845f15fec..36acfb08a 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy @@ -38,6 +38,7 @@ class BlobCacheInfoTest extends Specification { blob.objectUri == 's3://foo/com' blob.headers == [:] blob.id() == 's3://foo/com' + blob.state == BlobCacheInfo.State.CREATED expect: BlobCacheInfo.create('http://foo.com', 's3://foo/com', [Foo:['alpha'], Bar:['delta', 'gamma', 'omega']], [:]) @@ -137,7 +138,7 @@ class BlobCacheInfoTest extends Specification { def cache = BlobCacheInfo.create(location, object, headers, response) when: - def result = cache.failed('Oops') + def result = cache.errored('Oops') then: result.headers == [Foo:'something'] result.locationUri == 'http://foo.com' @@ -243,6 +244,7 @@ class BlobCacheInfoTest extends Specification { def 'should validate duration' () { given: def info = new BlobCacheInfo( + BlobCacheInfo.State.CREATED, null, null, null, @@ -265,4 +267,65 @@ class BlobCacheInfoTest extends Specification { now | now.plusSeconds(10) | Duration.ofSeconds(10) now | now.plusSeconds(60) | Duration.ofSeconds(60) } + + def 'should create blob cached' () { + given: + def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + + when: + def info = blob.cached() + then: + info.state == BlobCacheInfo.State.CACHED + info.locationUri == blob.locationUri + info.objectUri == blob.objectUri + info.headers == blob.headers + info.contentLength == blob.contentLength + info.contentType == blob.contentType + info.cacheControl == blob.cacheControl + info.creationTime == blob.creationTime + info.completionTime == blob.creationTime + info.exitStatus == 0 + info.logs == null + } + + def 'should create blob completed' () { + given: + def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + + when: + def info = blob.completed(1, 'this is the log') + then: + info.state == BlobCacheInfo.State.COMPLETED + info.locationUri == blob.locationUri + info.objectUri == blob.objectUri + info.headers == blob.headers + info.contentLength == blob.contentLength + info.contentType == blob.contentType + info.cacheControl == blob.cacheControl + info.creationTime == blob.creationTime + info.completionTime <= Instant.now() + info.exitStatus == 1 + info.logs == 'this is the log' + } + + def 'should create blob failed' () { + given: + def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + + when: + def info = blob.errored('this is the log') + then: + info.state == BlobCacheInfo.State.ERRORED + info.locationUri == blob.locationUri + info.objectUri == blob.objectUri + info.headers == blob.headers + info.contentLength == blob.contentLength + info.contentType == blob.contentType + info.cacheControl == blob.cacheControl + info.creationTime == blob.creationTime + info.completionTime <= Instant.now() + info.exitStatus == null + info.logs == 'this is the log' + } + } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy index 0aeabf0fa..51c9fec89 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy @@ -21,9 +21,17 @@ package io.seqera.wave.service.blob.impl import spock.lang.Specification import spock.lang.Unroll +import java.time.Duration +import java.time.Instant + import io.micronaut.context.ApplicationContext +import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.core.RoutePath import io.seqera.wave.model.ContainerCoordinates +import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.blob.BlobStore +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState import io.seqera.wave.test.AwsS3TestContainer /** * @@ -119,4 +127,56 @@ class BlobCacheServiceImplTest2 extends Specification implements AwsS3TestContai 'ubuntu@sha256:32353' | 's3://foo' | 'https://bar.com/' | 'https' | 'bar.com' | '/docker.io/v2/library/ubuntu/manifests/sha256:32353' } + def 'handle job completion'() { + given: + def store = Mock(BlobStore) + def blob = BlobCacheInfo.create('http://some/blob','s3://some/blob', [:], [:]) + def config = new BlobCacheConfig(statusDelay: Duration.ofSeconds(2)) + def service = new BlobCacheServiceImpl(blobStore: store, blobConfig: config) + def job = JobSpec.transfer('job-id', 'foo', Instant.now(), Duration.ofMinutes(1)) + def failed = new JobState(JobState.Status.FAILED, 1, 'Oops') + def ok = new JobState(JobState.Status.SUCCEEDED, 0, 'done') + + when: + service.onJobCompletion(job, blob, failed) + then: + 1 * store.storeBlob(blob.id(), _ as BlobCacheInfo) >> { id, BlobCacheInfo info -> info.state==BlobCacheInfo.State.ERRORED } + + when: + service.onJobCompletion(job, blob, ok) + then: + 1 * store.storeBlob(blob.id(), _ as BlobCacheInfo) >> { id, BlobCacheInfo info -> info.state==BlobCacheInfo.State.COMPLETED } + + } + + def 'handle job event when job times out'() { + given: + def store = Mock(BlobStore) + def blob = BlobCacheInfo.create('http://some/blob','s3://some/blob', [:], [:]) + def config = new BlobCacheConfig(statusDelay: Duration.ofSeconds(2)) + def service = new BlobCacheServiceImpl(blobStore: store, blobConfig: config) + def job = JobSpec.transfer('job-id', 'foo', Instant.now(), Duration.ofMinutes(1)) + + when: + service.onJobTimeout(job, blob) + then: + 1 * store.storeBlob(blob.id(), _ as BlobCacheInfo) >> { id, BlobCacheInfo info -> info.state==BlobCacheInfo.State.ERRORED } + + } + + def 'handle job event when job encounters an error'() { + given: + def store = Mock(BlobStore) + def blob = BlobCacheInfo.create('http://some/blob','s3://some/blob', [:], [:]) + def config = new BlobCacheConfig(statusDelay: Duration.ofSeconds(2)) + def service = new BlobCacheServiceImpl(blobStore: store, blobConfig: config) + def job = JobSpec.transfer('job-id', 'foo', Instant.now(), Duration.ofMinutes(1)) + def error = Mock(Exception) + + when: + service.onJobException(job, blob, error) + then: + 1 * store.storeBlob(blob.id(), _ as BlobCacheInfo) >> { id, BlobCacheInfo info -> info.state==BlobCacheInfo.State.ERRORED } + } + } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheStoreImplTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheStoreImplTest.groovy new file mode 100644 index 000000000..8b06d2b03 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheStoreImplTest.groovy @@ -0,0 +1,120 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.impl + +import spock.lang.Specification + +import java.time.Duration + +import io.micronaut.context.annotation.Property +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.configuration.BlobCacheConfig +import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.cache.impl.CacheProvider +import jakarta.inject.Inject + +/** + * + * @author Paolo Di Tommaso + */ +@Property(name = 'wave.blobCache.enabled', value = 'true') +@Property(name = 'wave.blobCache.storage.bucket', value='s3://foo') +@Property(name = 'wave.blobCache.storage.region', value='eu-west-1') +@MicronautTest +class BlobCacheStoreImplTest extends Specification { + + @Inject + BlobCacheStore store + + @Inject + CacheProvider provider + + def 'should get and store an entry' () { + given: + def key = UUID.randomUUID().toString() + def info1 = new BlobCacheInfo(BlobCacheInfo.State.CREATED, 'foo') + def info2 = new BlobCacheInfo(BlobCacheInfo.State.CREATED, 'bar') + + expect: + store.get(key) == null + + when: + store.put(key, info1) + then: + store.get(key) == info1 + + when: + store.put(key, info2) + then: + store.get(key) == info2 + and: + info1 != info2 + } + + def 'should put an item only if absent' () { + given: + def key = UUID.randomUUID().toString() + def info1 = new BlobCacheInfo(BlobCacheInfo.State.CREATED, 'foo') + def info2 = new BlobCacheInfo(BlobCacheInfo.State.CREATED, 'bar') + + expect: + store.putIfAbsent(key, info1) + and: + store.get(key) == info1 + + and: + !store.putIfAbsent(key, info2) + and: + store.get(key) == info1 // <-- didn't change + } + + def 'should put an entry with conditional ttl' () { + given: + def key = UUID.randomUUID().toString() + def info_ok = new BlobCacheInfo(BlobCacheInfo.State.CREATED, 'foo') + def info_err = new BlobCacheInfo(BlobCacheInfo.State.ERRORED, 'foo') + and: + def DELAY_ONE = Duration.ofMinutes(1) + def DELAY_TWO = Duration.ofSeconds(1) + and: + def config = Mock(BlobCacheConfig) + def cache = Spy(new BlobCacheStore(provider)) + cache.@blobConfig = config + + when: + cache.storeBlob(key, info_ok) + then: + // should use 'status duration' for TTL + 1 * config.getStatusDuration() >> DELAY_ONE + 0 * config.getFailureDuration() >> null + and: + 1 * cache.put(key, info_ok, DELAY_ONE) >> null + + when: + cache.storeBlob(key, info_err) + then: + // should use 'failure duration' for TTL + 0 * config.getStatusDuration() >> null + 1 * config.getFailureDuration() >> DELAY_TWO + and: + 1 * cache.put(key, info_err, DELAY_TWO) >> null + + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerJobStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy similarity index 74% rename from src/test/groovy/io/seqera/wave/service/blob/impl/DockerJobStrategyTest.groovy rename to src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy index b245061d5..ad41c874a 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerJobStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy @@ -19,14 +19,13 @@ package io.seqera.wave.service.blob.impl import spock.lang.Specification -import spock.lang.Unroll import io.seqera.wave.configuration.BlobCacheConfig /** * * @author Paolo Di Tommaso */ -class DockerJobStrategyTest extends Specification { +class DockerTransferStrategyTest extends Specification { def 'should create transfer cli' () { given: @@ -41,16 +40,15 @@ class DockerJobStrategyTest extends Specification { def strategy = new DockerTransferStrategy(blobConfig: config) when: - def result = strategy.createProcess(['s5cmd', 'run', '--this'], "job-name", 10) + def result = strategy.createProcess(['s5cmd', 'run', '--this'], "job-name") then: result.command() == [ 'docker', 'run', + '--detach', '--name', 'job-name', - '--stop-timeout', - '10', '-e', 'AWS_ACCESS_KEY_ID', '-e', 'AWS_SECRET_ACCESS_KEY', 'cr.seqera.io/public/s5cmd:latest', @@ -65,17 +63,4 @@ class DockerJobStrategyTest extends Specification { result.redirectErrorStream() } - @Unroll - def 'should parse state string' () { - expect: - DockerTransferStrategy.State.parse(STATE) == EXPECTED - - where: - STATE | EXPECTED - 'running' | new DockerTransferStrategy.State('running') - 'exited' | new DockerTransferStrategy.State('exited') - 'exited,' | new DockerTransferStrategy.State('exited') - 'exited,0' | new DockerTransferStrategy.State('exited', 0) - 'exited,10' | new DockerTransferStrategy.State('exited', 10) - } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeJobStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeJobStrategyTest.groovy deleted file mode 100644 index 56ec8795d..000000000 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeJobStrategyTest.groovy +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.blob.impl - - -import spock.lang.Specification -import spock.lang.Unroll - -import java.time.Duration -import java.time.OffsetDateTime -import java.util.concurrent.Executors - -import io.kubernetes.client.openapi.models.V1ContainerState -import io.kubernetes.client.openapi.models.V1ContainerStateTerminated -import io.kubernetes.client.openapi.models.V1ContainerStatus -import io.kubernetes.client.openapi.models.V1Job -import io.kubernetes.client.openapi.models.V1Pod -import io.kubernetes.client.openapi.models.V1PodList -import io.kubernetes.client.openapi.models.V1PodStatus -import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.configuration.BuildConfig -import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.job.JobId -import io.seqera.wave.service.job.JobState -import io.seqera.wave.service.k8s.K8sService.JobStatus -import io.seqera.wave.service.cleanup.CleanupStrategy -import io.seqera.wave.service.k8s.K8sService -/** - * - * @author Munish Chouhan - */ -class KubeJobStrategyTest extends Specification { - - K8sService k8sService = Mock(K8sService) - BlobCacheConfig blobConfig = new BlobCacheConfig(s5Image: 's5cmd', transferTimeout: Duration.ofSeconds(10), retryAttempts: 3) - CleanupStrategy cleanup = new CleanupStrategy(buildConfig: new BuildConfig(cleanup: "OnSuccess")) - KubeTransferStrategy strategy = new KubeTransferStrategy(k8sService: k8sService, blobConfig: blobConfig, cleanup: cleanup, executor: Executors.newSingleThreadExecutor()) - - def "transfer should start a transferJob"() { - given: - def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) - def command = ["transfer", "blob"] - final jobName = "job-123" - def podName = "$jobName-abc".toString() - def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) - pod.status = new V1PodStatus(phase: "Succeeded") - def podList = new V1PodList(items: [pod]) - k8sService.launchJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) - k8sService.waitJob(_, _) >> podList - k8sService.getPod(_) >> pod - k8sService.waitPodCompletion(_, _) >> 0 - k8sService.logsPod(_) >> "transfer successful" - - when: - strategy.launchJob(podName, command) - - then: - 1 * k8sService.launchJob(podName, blobConfig.s5Image, command, blobConfig) - } - - def 'status should return correct status when job is not completed'() { - given: - def job = JobId.transfer('foo') - and: - k8sService.getJobStatus(job.schedulerId) >> K8sService.JobStatus.Running - - when: - def result = strategy.status(job) - then: - result.status == JobState.Status.RUNNING - } - - void 'status should return correct transfer status when pods are created'() { - given: - def job = JobId.transfer('foo') - and: - def status = new V1PodStatus(phase: "Succeeded", containerStatuses: [new V1ContainerStatus( state: new V1ContainerState(terminated: new V1ContainerStateTerminated(exitCode: 0)))]) - def pod = new V1Pod(metadata: [name: job.schedulerId], status: status) - and: - k8sService.getJobStatus(job.schedulerId) >> K8sService.JobStatus.Succeeded - k8sService.getLatestPodForJob(job.schedulerId) >> pod - k8sService.logsPod(pod) >> "transfer successful" - - when: - def result = strategy.status(job) - then: - result.status == JobState.Status.SUCCEEDED - result.exitCode == 0 - result.stdout == "transfer successful" - } - - def 'status should return failed transfer when no pods are created'() { - given: - def job = JobId.transfer('foo') - and: - def status = new V1PodStatus(phase: "Failed") - def pod = new V1Pod(metadata: [name: job.schedulerId], status: status) - and: - k8sService.getLatestPodForJob(job.schedulerId) >> pod - k8sService.getJobStatus(job.schedulerId) >> K8sService.JobStatus.Failed - - when: - def result = strategy.status(job) - then: - result.status == JobState.Status.FAILED - } - - def 'status should handle null job status'() { - given: - def job = JobId.transfer('foo') - and: - k8sService.getJobStatus(job.schedulerId) >> null - - when: - def result = strategy.status(job) - then: - result.status == JobState.Status.UNKNOWN - } - - @Unroll - def "mapToStatus should return correct transfer status for jobStatus #JOB_STATUS that is #TRANSFER_STATUS"() { - expect: - KubeTransferStrategy.mapToStatus(JOB_STATUS) == TRANSFER_STATUS - - where: - JOB_STATUS | TRANSFER_STATUS - JobStatus.Pending | JobState.Status.PENDING - JobStatus.Running | JobState.Status.RUNNING - JobStatus.Succeeded | JobState.Status.SUCCEEDED - JobStatus.Failed | JobState.Status.FAILED - null | JobState.Status.UNKNOWN - } -} diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy new file mode 100644 index 000000000..f4630e689 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy @@ -0,0 +1,65 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.impl + +import spock.lang.Specification + +import java.time.Duration +import java.time.OffsetDateTime + +import io.kubernetes.client.openapi.models.V1Job +import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodList +import io.kubernetes.client.openapi.models.V1PodStatus +import io.seqera.wave.configuration.BlobCacheConfig +import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.k8s.K8sService +/** + * + * @author Munish Chouhan + */ +class KubeTransferStrategyTest extends Specification { + + K8sService k8sService = Mock(K8sService) + BlobCacheConfig blobConfig = new BlobCacheConfig(s5Image: 's5cmd', transferTimeout: Duration.ofSeconds(10), retryAttempts: 3) + KubeTransferStrategy strategy = new KubeTransferStrategy(k8sService: k8sService, blobConfig: blobConfig) + + def "transfer should start a transferJob"() { + given: + def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) + def command = ["transfer", "blob"] + final jobName = "job-123" + def podName = "$jobName-abc".toString() + def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) + pod.status = new V1PodStatus(phase: "Succeeded") + def podList = new V1PodList(items: [pod]) + k8sService.launchTransferJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) + k8sService.waitJob(_, _) >> podList + k8sService.getPod(_) >> pod + k8sService.waitPodCompletion(_, _) >> 0 + k8sService.logsPod(_) >> "transfer successful" + + when: + strategy.launchJob(podName, command) + + then: + 1 * k8sService.launchTransferJob(podName, blobConfig.s5Image, command, blobConfig) + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreLocalTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreLocalTest.groovy index edde33116..59e5025fb 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreLocalTest.groovy @@ -45,10 +45,40 @@ class BuildCacheStoreLocalTest extends Specification { @Named(TaskExecutors.IO) ExecutorService ioExecutor - BuildResult zero = BuildResult.create('0') - BuildResult one = BuildResult.completed('1', 0, 'done', Instant.now(), 'abc') - BuildResult two = BuildResult.completed('2', 0, 'done', Instant.now(), 'abc') - BuildResult three = BuildResult.completed('3', 0, 'done', Instant.now(), 'abc') + BuildResult zeroResult = BuildResult.create('0') + BuildResult oneResult = BuildResult.completed('1', 0, 'done', Instant.now(), 'abc') + BuildResult twoResult = BuildResult.completed('2', 0, 'done', Instant.now(), 'abc') + BuildResult threeResult = BuildResult.completed('3', 0, 'done', Instant.now(), 'abc') + + def zeroRequest = new BuildRequest( + targetImage: 'docker.io/foo:0', + buildId: '0', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def oneRequest = new BuildRequest( + targetImage: 'docker.io/foo:1', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def twoRequest = new BuildRequest( + targetImage: 'docker.io/foo:2', + buildId: '2', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def threeRequest = new BuildRequest( + targetImage: 'docker.io/foo:3', + buildId: '3', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + + def zero = new BuildStoreEntry(zeroRequest, zeroResult) + def one = new BuildStoreEntry(oneRequest, oneResult) + def two = new BuildStoreEntry(twoRequest, twoResult) + def three = new BuildStoreEntry(threeRequest, threeResult) def 'should get and put key values' () { given: @@ -145,14 +175,13 @@ class BuildCacheStoreLocalTest extends Specification { // stops until the value is updated def result = cache.awaitBuild('foo') then: - result.get() == one + result.get() == one.result when: cache.storeBuild('foo',two) cache.storeBuild('foo',three) then: - cache.awaitBuild('foo').get()==three - + cache.awaitBuild('foo').get() == three.result } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy index 48432edb6..d5d5afab7 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy @@ -18,44 +18,56 @@ package io.seqera.wave.service.builder +import spock.lang.Shared import spock.lang.Specification import spock.lang.Timeout import java.time.Duration import java.time.Instant -import java.util.concurrent.ExecutionException import io.micronaut.context.ApplicationContext -import io.seqera.wave.exception.BuildTimeoutException +import io.seqera.wave.service.job.JobFactory +import io.seqera.wave.service.job.JobQueue import io.seqera.wave.test.RedisTestContainer -import redis.clients.jedis.Jedis /** * * @author Paolo Di Tommaso */ class BuildCacheStoreRedisTest extends Specification implements RedisTestContainer { + @Shared ApplicationContext applicationContext - Jedis jedis - def setup() { applicationContext = ApplicationContext.run([ - wave:[ build:[ timeout: '5s', 'trusted-timeout': '5s' ]], REDIS_HOST: redisHostName, REDIS_PORT: redisPort ], 'test', 'redis') - jedis = new Jedis(redisHostName, redisPort as int) + flushRedis() } def cleanup(){ - jedis.flushAll() - jedis.close() + applicationContext.close() + } + + def flushRedis() { + // The use of Redis flush removes also data structures like consumer groups + // causing unexpected exceptions in other components. Only remove entries + // creates by this tests + def cacheStore = applicationContext.getBean(BuildStore) + cacheStore.removeBuild('foo') } def 'should get and put key values' () { given: - def req1 = BuildResult.create('1') + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'docker.io/foo:0', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def entry = new BuildStoreEntry(req, res) and: def cacheStore = applicationContext.getBean(BuildStore) @@ -63,17 +75,22 @@ class BuildCacheStoreRedisTest extends Specification implements RedisTestContain cacheStore.getBuild('foo') == null when: - cacheStore.storeBuild('foo', req1) + cacheStore.storeBuild('foo', entry) then: - cacheStore.getBuild('foo') == req1 - and: - jedis.get("wave-build:foo").toString() + cacheStore.getBuild('foo') == entry } def 'should expire entry' () { given: - def req1 = BuildResult.create('1') + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'docker.io/foo:0', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def entry = new BuildStoreEntry(req, res) and: def cacheStore = applicationContext.getBean(BuildStore) @@ -81,20 +98,20 @@ class BuildCacheStoreRedisTest extends Specification implements RedisTestContain cacheStore.getBuild('foo') == null when: - cacheStore.storeBuild('foo', req1) + cacheStore.storeBuild('foo', entry) then: - cacheStore.getBuild('foo') == req1 + cacheStore.getBuild('foo') == entry and: sleep 1000 - cacheStore.getBuild('foo') == req1 + cacheStore.getBuild('foo') == entry when: - cacheStore.storeBuild('foo', req1, Duration.ofSeconds(1)) + cacheStore.storeBuild('foo', entry, Duration.ofSeconds(1)) then: - cacheStore.getBuild('foo') == req1 + cacheStore.getBuild('foo') == entry and: sleep 500 - cacheStore.getBuild('foo') == req1 + cacheStore.getBuild('foo') == entry and: sleep 1_500 cacheStore.getBuild('foo') == null @@ -102,35 +119,56 @@ class BuildCacheStoreRedisTest extends Specification implements RedisTestContain def 'should store if absent' () { given: - def req1 = BuildResult.create('1') - def req2 = BuildResult.create('2') + def res1 = BuildResult.create('1') + def req1 = new BuildRequest( + targetImage: 'docker.io/foo:1', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def entry1 = new BuildStoreEntry(req1, res1) + def res2 = BuildResult.create('2') + def req2 = new BuildRequest( + targetImage: 'docker.io/foo:2', + buildId: '2', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def entry2 = new BuildStoreEntry(req2, res2) and: def store = applicationContext.getBean(BuildStore) expect: // the value is store because the key does not exists - store.storeIfAbsent('foo', req1) + store.storeIfAbsent('foo', entry1) and: // the value is return - store.getBuild('foo') == req1 + store.getBuild('foo') == entry1 and: // storing a new value fails because the key already exist - !store.storeIfAbsent('foo', req2) + !store.storeIfAbsent('foo', entry2) and: // the previous value is returned - store.getBuild('foo') == req1 + store.getBuild('foo') == entry1 } def 'should remove a build entry' () { given: - def zero = BuildResult.create('1') + def res = BuildResult.create('0') + def req = new BuildRequest( + targetImage: 'docker.io/foo:0', + buildId: '0', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def entry = new BuildStoreEntry(req, res) def cache = applicationContext.getBean(BuildStore) when: - cache.storeBuild('foo', zero) + cache.storeBuild('foo', entry) then: - cache.getBuild('foo') == zero + cache.getBuild('foo') == entry when: cache.removeBuild('foo') @@ -142,18 +180,26 @@ class BuildCacheStoreRedisTest extends Specification implements RedisTestContain @Timeout(value=10) def 'should await for a value' () { given: - def req1 = BuildResult.create('1') + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'docker.io/foo:1', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofSeconds(10) + ) + def entry = new BuildStoreEntry(req, res) and: def cacheStore = applicationContext.getBean(BuildStore) as BuildStore when: // insert a value - cacheStore.storeBuild('foo', req1) + cacheStore.storeBuild('foo', entry) // update a value in a separate thread Thread.start { - req1 = BuildResult.completed('1', 0, '', Instant.now(), null) - cacheStore.storeBuild('foo',req1) + res = BuildResult.completed('1', 0, '', Instant.now(), null) + entry = entry.withResult(res) + cacheStore.storeBuild('foo', entry) } // wait the value is updated @@ -161,23 +207,32 @@ class BuildCacheStoreRedisTest extends Specification implements RedisTestContain def result = cacheStore.awaitBuild('foo') then: - result.get() == req1 + result.get() == entry.result } @Timeout(value=30) def 'should abort an await if build never finish' () { given: - def req1 = BuildResult.create('1') + def buildCacheStore = applicationContext.getBean(BuildCacheStore) + def jobQueue = applicationContext.getBean(JobQueue) + def jobFactory = applicationContext.getBean(JobFactory) + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'docker.io/foo:1', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofSeconds(5) + ) + def entry = new BuildStoreEntry(req, res) and: - def cacheStore = applicationContext.getBean(BuildStore) as BuildStore - // insert a value - cacheStore.storeBuild('foo', req1) + buildCacheStore.storeIfAbsent(req.targetImage, entry) + jobQueue.offer(jobFactory.build(req)) when: "wait for an update never will arrive" - cacheStore.awaitBuild('foo').get() - then: - def e = thrown(ExecutionException) - e.cause.class == BuildTimeoutException + buildCacheStore.awaitBuild(req.targetImage).get() + + then: "job will timeout and the build will be marked as failed" + buildCacheStore.getBuild(req.targetImage).result.exitStatus == -1 } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy new file mode 100644 index 000000000..eae4f8c42 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy @@ -0,0 +1,302 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Requires +import spock.lang.Specification + +import java.nio.file.Files +import java.time.Duration +import java.time.Instant +import java.util.concurrent.TimeUnit + +import groovy.util.logging.Slf4j +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.auth.RegistryCredentialsProvider +import io.seqera.wave.auth.RegistryLookupService +import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.configuration.HttpClientConfig +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.builder.store.BuildRecordStore +import io.seqera.wave.service.inspect.ContainerInspectServiceImpl +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.test.TestHelper +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.util.ContainerHelper +import io.seqera.wave.util.Packer +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@Slf4j +@MicronautTest +class ContainerBuildServiceLiveTest extends Specification { + + @Inject ContainerBuildServiceImpl service + @Inject RegistryLookupService lookupService + @Inject RegistryCredentialsProvider credentialsProvider + @Inject ContainerInspectServiceImpl dockerAuthService + @Inject HttpClientConfig httpClientConfig + @Inject BuildConfig buildConfig + @Inject BuildRecordStore buildRecordStore + @Inject BuildCacheStore buildCacheStore + @Inject PersistenceService persistenceService + @Inject JobService jobService + + @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) + def 'should build & push container to aws' () { + given: + def folder = Files.createTempDirectory('test') + def buildRepo = buildConfig.defaultBuildRepository + def cacheRepo = buildConfig.defaultCacheRepository + def duration = Duration.ofMinutes(1) + and: + def dockerFile = ''' + FROM busybox + RUN echo Hello > hello.txt + '''.stripIndent() + and: + def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, cacheRepo, Mock(PlatformId)) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now(), + maxDuration: duration + ) + .withBuildId('1') + and: + buildCacheStore.storeBuild(targetImage, new BuildStoreEntry(req, BuildResult.create(req))) + + when: + service.launch(req) + then: + service + .buildResult(targetImage) + .get(duration.toSeconds(), TimeUnit.SECONDS) + .succeeded() + + cleanup: + folder?.deleteDir() + } + + @Requires({System.getenv('DOCKER_USER') && System.getenv('DOCKER_PAT')}) + def 'should build & push container to docker.io' () { + given: + def folder = Files.createTempDirectory('test') + def buildRepo = "docker.io/pditommaso/wave-tests" + def cacheRepo = buildConfig.defaultCacheRepository + def duration = Duration.ofMinutes(1) + and: + def dockerFile = ''' + FROM busybox + RUN echo Hello > hello.txt + '''.stripIndent() + and: + def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: TestHelper.containerPlatform(), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now(), + maxDuration: duration + ) + .withBuildId('1') + and: + buildCacheStore.storeBuild(targetImage, new BuildStoreEntry(req, BuildResult.create(req))) + + when: + service.launch(req) + then: + service + .buildResult(targetImage) + .get(duration.toSeconds(), TimeUnit.SECONDS) + .succeeded() + + cleanup: + folder?.deleteDir() + } + + @Requires({System.getenv('QUAY_USER') && System.getenv('QUAY_PAT')}) + def 'should build & push container to quay.io' () { + given: + def folder = Files.createTempDirectory('test') + def cacheRepo = buildConfig.defaultCacheRepository + def duration = Duration.ofMinutes(1) + and: + def dockerFile = ''' + FROM busybox + RUN echo Hello > hello.txt + '''.stripIndent() + and: + def buildRepo = "quay.io/pditommaso/wave-tests" + def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('linux/arm64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: TestHelper.containerPlatform(), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now(), + maxDuration: duration + ) + .withBuildId('1') + and: + buildCacheStore.storeBuild(targetImage, new BuildStoreEntry(req, BuildResult.create(req))) + + when: + service.launch(req) + then: + service + .buildResult(targetImage) + .get(duration.toSeconds(), TimeUnit.SECONDS) + .succeeded() + + cleanup: + folder?.deleteDir() + } + + @Requires({System.getenv('AZURECR_USER') && System.getenv('AZURECR_PAT')}) + def 'should build & push container to azure' () { + given: + def folder = Files.createTempDirectory('test') + def buildRepo = "seqeralabs.azurecr.io/wave-tests" + def cacheRepo = buildConfig.defaultCacheRepository + and: + def dockerFile = ''' + FROM busybox + RUN echo Hello > hello.txt + '''.stripIndent() + and: + def duration = Duration.ofMinutes(1) + def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: TestHelper.containerPlatform(), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now(), + maxDuration: duration + ) + .withBuildId('1') + and: + buildCacheStore.storeBuild(targetImage, new BuildStoreEntry(req, BuildResult.create(req))) + + when: + service.launch(req) + then: + service + .buildResult(targetImage) + .get(duration.toSeconds(), TimeUnit.SECONDS) + .succeeded() + + cleanup: + folder?.deleteDir() + } + + @Requires({System.getenv('DOCKER_USER') && System.getenv('DOCKER_PAT')}) + def 'should build & push container to docker.io with local layers' () { + given: + def folder = Files.createTempDirectory('test') + def buildRepo = "docker.io/pditommaso/wave-tests" + def cacheRepo = buildConfig.defaultCacheRepository + def layer = Files.createDirectories(folder.resolve('layer')) + def file1 = layer.resolve('hola.txt'); file1.text = 'Hola\n' + def file2 = layer.resolve('ciao.txt'); file2.text = 'Ciao\n' + and: + def dockerFile = ''' + FROM busybox + RUN echo Hello > hello_docker.txt + '''.stripIndent() + and: + def l1 = new Packer().layer(layer, [file1, file2]) + def containerConfig = new ContainerConfig(cmd: ['echo', 'Hola'], layers: [l1]) + and: + def duration = Duration.ofMinutes(1) + def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: TestHelper.containerPlatform(), + cacheRepository: cacheRepo, + configJson: cfg, + containerConfig: containerConfig , + format: BuildFormat.DOCKER, + startTime: Instant.now(), + maxDuration: duration + ) + .withBuildId('1') + and: + buildCacheStore.storeBuild(targetImage, new BuildStoreEntry(req, BuildResult.create(req))) + + when: + service.launch(req) + then: + service + .buildResult(targetImage) + .get(duration.toSeconds(), TimeUnit.SECONDS) + .succeeded() + + cleanup: + folder?.deleteDir() + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index cf436f481..7e0e6d647 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.builder -import spock.lang.Requires import spock.lang.Specification import java.nio.file.Files @@ -30,6 +29,9 @@ import com.sun.net.httpserver.HttpExchange import com.sun.net.httpserver.HttpHandler import com.sun.net.httpserver.HttpServer import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Primary +import io.micronaut.context.annotation.Requires +import io.micronaut.context.event.ApplicationEventPublisher import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig @@ -40,27 +42,56 @@ import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.service.builder.store.BuildRecordStore -import io.seqera.wave.service.cleanup.CleanupStrategy import io.seqera.wave.service.inspect.ContainerInspectServiceImpl +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.test.RedisTestContainer -import io.seqera.wave.test.SurrealDBTestContainer +import io.seqera.wave.service.scan.ScanRequest +import io.seqera.wave.service.scan.ScanStrategy +import io.seqera.wave.test.TestHelper import io.seqera.wave.tower.PlatformId +import io.seqera.wave.util.ContainerHelper import io.seqera.wave.util.Packer import io.seqera.wave.util.SpackHelper import io.seqera.wave.util.TemplateRenderer import jakarta.inject.Inject -import io.seqera.wave.util.ContainerHelper +import jakarta.inject.Singleton /** * * @author Paolo Di Tommaso */ @Slf4j -@MicronautTest +@MicronautTest(environments = ['build-service-test']) +class ContainerBuildServiceTest extends Specification { + + @Primary + @Singleton + @Requires(env = 'build-service-test') + static class FakeBuildStrategy extends BuildStrategy { + + @Override + void build(String jobName, BuildRequest request) { + // do nothing + log.debug "Running fake build job=$jobName - request=$request" + } + } + + @Primary + @Singleton + @Requires(env = 'build-service-test') + static class FakeScanStrategy extends ScanStrategy { + + @Override + void scanContainer(String jobName, ScanRequest request) { + // do nothing + log.debug "Running fake scan job=$jobName - request=$request" + } + } -class ContainerBuildServiceTest extends Specification implements RedisTestContainer, SurrealDBTestContainer{ @Inject ContainerBuildServiceImpl service @Inject RegistryLookupService lookupService @@ -69,190 +100,15 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai @Inject HttpClientConfig httpClientConfig @Inject BuildConfig buildConfig @Inject BuildRecordStore buildRecordStore + @Inject BuildCacheStore buildCacheStore @Inject PersistenceService persistenceService - - - @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) - def 'should build & push container to aws' () { - given: - def folder = Files.createTempDirectory('test') - def buildRepo = buildConfig.defaultBuildRepository - def cacheRepo = buildConfig.defaultCacheRepository - and: - def dockerFile = ''' - FROM busybox - RUN echo Hello > hello.txt - '''.stripIndent() - and: - def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, cacheRepo, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = - new BuildRequest( - containerId: containerId, - containerFile: dockerFile, - workspace: folder, - targetImage: targetImage, - identity: Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), - cacheRepository: cacheRepo, - configJson: cfg, - format: BuildFormat.DOCKER, - startTime: Instant.now() - ) - .withBuildId('1') - - when: - def result = service.launch(req) - then: - result.id - result.startTime - result.duration - result.exitStatus == 0 - - cleanup: - folder?.deleteDir() - } - - @Requires({System.getenv('DOCKER_USER') && System.getenv('DOCKER_PAT')}) - def 'should build & push container to docker.io' () { - given: - def folder = Files.createTempDirectory('test') - def buildRepo = "docker.io/pditommaso/wave-tests" - def cacheRepo = buildConfig.defaultCacheRepository - and: - def dockerFile = ''' - FROM busybox - RUN echo Hello > hello.txt - '''.stripIndent() - and: - def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = - new BuildRequest( - containerId: containerId, - containerFile: dockerFile, - workspace: folder, - targetImage: targetImage, - identity: Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), - cacheRepository: cacheRepo, - configJson: cfg, - format: BuildFormat.DOCKER, - startTime: Instant.now(), - ) - .withBuildId('1') - - when: - def result = service.launch(req) - and: - println result.logs - then: - result.id - result.startTime - result.duration - result.exitStatus == 0 - - cleanup: - folder?.deleteDir() - } - - @Requires({System.getenv('QUAY_USER') && System.getenv('QUAY_PAT')}) - def 'should build & push container to quay.io' () { - given: - def folder = Files.createTempDirectory('test') - def buildRepo = buildConfig.defaultBuildRepository - def cacheRepo = buildConfig.defaultCacheRepository - and: - def dockerFile = ''' - FROM busybox - RUN echo Hello > hello.txt - '''.stripIndent() - and: - buildRepo = "quay.io/pditommaso/wave-tests" - def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = - new BuildRequest( - containerId: containerId, - containerFile: dockerFile, - workspace: folder, - targetImage: targetImage, - identity: Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), - cacheRepository: cacheRepo, - configJson: cfg, - format: BuildFormat.DOCKER, - startTime: Instant.now() - ) - .withBuildId('1') - - when: - def result = service.launch(req) - and: - println result.logs - then: - result.id - result.startTime - result.duration - result.exitStatus == 0 - - cleanup: - folder?.deleteDir() - } - - @Requires({System.getenv('AZURECR_USER') && System.getenv('AZURECR_PAT')}) - def 'should build & push container to azure' () { - given: - def folder = Files.createTempDirectory('test') - def buildRepo = "seqeralabs.azurecr.io/wave-tests" - def cacheRepo = buildConfig.defaultCacheRepository - and: - def dockerFile = ''' - FROM busybox - RUN echo Hello > hello.txt - '''.stripIndent() - and: - def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = - new BuildRequest( - containerId: containerId, - containerFile: dockerFile, - workspace: folder, - targetImage: targetImage, - identity: Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), - cacheRepository: cacheRepo, - configJson: cfg, - format: BuildFormat.DOCKER, - startTime: Instant.now() - ) - .withBuildId('1') - - when: - def result = service.launch(req) - and: - println result.logs - then: - result.id - result.startTime - result.duration - result.exitStatus == 0 - - cleanup: - folder?.deleteDir() - } + @Inject JobService jobService def 'should save build docker build file' () { given: def folder = Files.createTempDirectory('test') def buildRepo = buildConfig.defaultBuildRepository def cacheRepo = buildConfig.defaultCacheRepository - def DURATION = Duration.ofDays(1) and: def cfg = 'some credentials' def dockerFile = ''' @@ -285,29 +141,27 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai workspace: folder, targetImage: targetImage, identity: Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), + platform: TestHelper.containerPlatform(), cacheRepository: cacheRepo, format: BuildFormat.DOCKER, - startTime: Instant.now() + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) ) .withBuildId('1') and: def store = Mock(BuildStore) - def strategy = Mock(BuildStrategy) - def builder = new ContainerBuildServiceImpl(buildStrategy: strategy, buildStore: store, buildConfig: buildConfig, spackConfig:spackConfig, cleanup: new CleanupStrategy(buildConfig: buildConfig)) - def RESPONSE = Mock(BuildResult) + def jobService = Mock(JobService) + def builder = new ContainerBuildServiceImpl(buildStore: store, buildConfig: buildConfig, spackConfig:spackConfig, jobService: jobService) + def RESPONSE = Mock(JobSpec) when: - def result = builder.launch(req) + builder.launch(req) then: - 1 * strategy.build(req) >> RESPONSE - 1 * store.storeBuild(req.targetImage, RESPONSE, DURATION) >> null + 1 * jobService.launchBuild(req) >> RESPONSE and: req.workDir.resolve('Containerfile').text == new TemplateRenderer().render(dockerFile, [spack_cache_bucket:'s3://bucket/cache', spack_key_file:'/mnt/secret']) req.workDir.resolve('context/conda.yml').text == condaFile req.workDir.resolve('context/spack.yaml').text == spackFile - and: - result == RESPONSE cleanup: folder?.deleteDir() @@ -478,58 +332,6 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai '''.stripIndent() } - @Requires({System.getenv('DOCKER_USER') && System.getenv('DOCKER_PAT')}) - def 'should build & push container to docker.io with local layers' () { - given: - def folder = Files.createTempDirectory('test') - def buildRepo = "docker.io/pditommaso/wave-tests" - def cacheRepo = buildConfig.defaultCacheRepository - def context = Files.createDirectories(folder.resolve('context')) - def layer = Files.createDirectories(folder.resolve('layer')) - def file1 = layer.resolve('hola.txt'); file1.text = 'Hola\n' - def file2 = layer.resolve('ciao.txt'); file2.text = 'Ciao\n' - and: - def dockerFile = ''' - FROM busybox - RUN echo Hello > hello.txt - '''.stripIndent() - and: - def l1 = new Packer().layer(layer, [file1, file2]) - def containerConfig = new ContainerConfig(cmd: ['echo', 'Hola'], layers: [l1]) - and: - def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = - new BuildRequest( - containerId: containerId, - containerFile: dockerFile, - workspace: folder, - targetImage: targetImage, - identity: Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), - cacheRepository: cacheRepo, - configJson: cfg, - containerConfig: containerConfig , - format: BuildFormat.DOCKER, - startTime: Instant.now() - ) - .withBuildId('1') - - when: - def result = service.launch(req) - and: - println result.logs - then: - result.id - result.startTime - result.duration - result.exitStatus == 0 - - cleanup: - folder?.deleteDir() - } - def 'should untar build context' () { given: @@ -725,4 +527,84 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai 'docker.io' | 'docker.io' 'docker.io/foo/'| 'docker.io' } + + def 'should handle job completion event and update build store'() { + given: + def mockBuildStore = Mock(BuildStore) + def mockProxyService = Mock(RegistryProxyService) + def mockEventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) + def state = JobState.succeeded('logs') + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'docker.io/foo:0', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def build = new BuildStoreEntry(req, res) + + when: + service.onJobCompletion(job, build, state) + + then: + 1 * mockBuildStore.storeBuild('1', _, _) + and: + 1 * mockProxyService.getImageDigest(_, _) >> 'digest' + and: + 1 * mockEventPublisher.publishEvent(_) + } + + def 'should handle job error event and update build store'() { + given: + def mockBuildStore = Mock(BuildStore) + def mockProxyService = Mock(RegistryProxyService) + def mockEventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) + def error = new Exception('error') + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'docker.io/foo:0', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def build = new BuildStoreEntry(req, res) + + when: + service.onJobException(job, build, error) + + then: + 1 * mockBuildStore.storeBuild('1', _, _) + and: + 1 * mockEventPublisher.publishEvent(_) + } + + def 'should handle job timeout event and update build store'() { + given: + def mockBuildStore = Mock(BuildStore) + def mockProxyService = Mock(RegistryProxyService) + def mockEventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) + def res = BuildResult.create('1') + def req = new BuildRequest( + targetImage: 'docker.io/foo:0', + buildId: '1', + startTime: Instant.now(), + maxDuration: Duration.ofMinutes(1) + ) + def build = new BuildStoreEntry(req, res) + + when: + service.onJobTimeout(job, build) + + then: + 1 * mockBuildStore.storeBuild('1', _, _) + and: + 1 * mockEventPublisher.publishEvent(_) + } + } diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index 7d83bf257..3294f88a0 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -45,11 +45,13 @@ class DockerBuildStrategyTest extends Specification { and: def work = Path.of('/work/foo') when: - def cmd = service.cmdForBuildkit(work, null, null, null) + def cmd = service.cmdForBuildkit('build-job-name', work, null, null, null) then: cmd == ['docker', 'run', - '--rm', + '--detach', + '--name', + 'build-job-name', '--privileged', '-v', '/work/foo:/work/foo', '--entrypoint', @@ -57,11 +59,13 @@ class DockerBuildStrategyTest extends Specification { 'moby/buildkit:v0.14.1-rootless'] when: - cmd = service.cmdForBuildkit(work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) + cmd = service.cmdForBuildkit('build-job-name', work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) then: cmd == ['docker', 'run', - '--rm', + '--detach', + '--name', + 'build-job-name', '--privileged', '-v', '/work/foo:/work/foo', '--entrypoint', @@ -71,11 +75,13 @@ class DockerBuildStrategyTest extends Specification { 'moby/buildkit:v0.14.1-rootless'] when: - cmd = service.cmdForBuildkit(work, Path.of('/foo/creds.json'), spackConfig, null) + cmd = service.cmdForBuildkit('build-job-name', work, Path.of('/foo/creds.json'), spackConfig, null) then: cmd == ['docker', 'run', - '--rm', + '--detach', + '--name', + 'build-job-name', '--privileged', '-v', '/work/foo:/work/foo', '--entrypoint', @@ -102,11 +108,13 @@ class DockerBuildStrategyTest extends Specification { targetImage: 'repo:89fb83ce6ec8627b', cacheRepository: 'reg.io/wave/build/cache' ) when: - def cmd = service.buildCmd(req, creds) + def cmd = service.buildCmd('build-job-name', req, creds) then: cmd == ['docker', 'run', - '--rm', + '--detach', + '--name', + 'build-job-name', '--privileged', '-v', '/work/foo/89fb83ce6ec8627b:/work/foo/89fb83ce6ec8627b', '--entrypoint', @@ -156,11 +164,13 @@ class DockerBuildStrategyTest extends Specification { format: BuildFormat.SINGULARITY, isSpackBuild: true ) when: - def cmd = service.buildCmd(req, creds) + def cmd = service.buildCmd('build-job-name', req, creds) then: cmd == ['docker', 'run', - '--rm', + '--detach', + '--name', + 'build-job-name', '--privileged', '--entrypoint', '', '-v', '/work/foo/d4869cc39b8d7d55:/work/foo/d4869cc39b8d7d55', @@ -198,11 +208,13 @@ class DockerBuildStrategyTest extends Specification { format: BuildFormat.SINGULARITY, isSpackBuild: true ) when: - def cmd = service.buildCmd(req, creds) + def cmd = service.buildCmd('build-job-name', req, creds) then: cmd == ['docker', 'run', - '--rm', + '--detach', + '--name', + 'build-job-name', '--privileged', '--entrypoint', '', '-v', '/work/foo/9c68af894bb2419c:/work/foo/9c68af894bb2419c', diff --git a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy index 5c96184d0..957a32146 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy @@ -19,62 +19,80 @@ package io.seqera.wave.service.builder import spock.lang.Specification -import spock.lang.Timeout import java.nio.file.Files import java.time.Duration import java.time.Instant +import java.util.concurrent.CompletableFuture -import io.micronaut.context.annotation.Value -import io.micronaut.test.annotation.MockBean -import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId -import jakarta.inject.Inject import io.seqera.wave.util.ContainerHelper - /** * * @author Jorge Aguilera */ -@MicronautTest class FutureContainerBuildServiceTest extends Specification { - @Value('${wave.build.repo}') String buildRepo - @Value('${wave.build.cache}') String cacheRepo + String buildRepo = 'build/repo' + String cacheRepo = 'cache/repo' + + def 'should wait for successful container build' () { + given: + def folder = Files.createTempDirectory('test') + and: + def dockerfile = """ + FROM busybox + RUN echo 'hello' > hello.txt + """.stripIndent() + and: + def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def req = new BuildRequest(containerId, dockerfile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def res = new BuildResult("", 0, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') + and: + def buildStore = Mock(BuildStore) + def buildCounter = Mock(BuildCounterStore) + buildStore.getBuildResult(targetImage) >> res + buildStore.awaitBuild(targetImage) >> CompletableFuture.completedFuture(res) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, buildCounter: buildCounter) - @Inject - ContainerBuildServiceImpl service + when: + service.checkOrSubmit(req) + then: + noExceptionThrown() - int exitCode + when: + def status = service.buildResult(req.targetImage).get() + then: + status.getExitStatus() == 0 - @MockBean(BuildStrategy) - BuildStrategy fakeBuildStrategy(){ - new BuildStrategy() { - @Override - BuildResult build(BuildRequest req) { - new BuildResult("", exitCode, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') - } - } + cleanup: + folder?.deleteDir() } - @Timeout(30) - def 'should wait to build container completion' () { + def 'should wait for unsuccessful container build' () { given: def folder = Files.createTempDirectory('test') and: def dockerfile = """ FROM busybox - RUN echo $EXIT_CODE > hello.txt + RUN echo 'hello' > hello.txt """.stripIndent() and: def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) def req = new BuildRequest(containerId, dockerfile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def res = new BuildResult("", 1, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') + and: + def buildStore = Mock(BuildStore) + def buildCounter = Mock(BuildCounterStore) + buildStore.getBuildResult(targetImage) >> res + buildStore.awaitBuild(targetImage) >> CompletableFuture.completedFuture(res) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, buildCounter: buildCounter) when: - exitCode = EXIT_CODE service.checkOrSubmit(req) then: noExceptionThrown() @@ -82,15 +100,10 @@ class FutureContainerBuildServiceTest extends Specification { when: def status = service.buildResult(req.targetImage).get() then: - status.getExitStatus() == EXIT_CODE + status.getExitStatus() == 1 cleanup: folder?.deleteDir() - - where: - EXIT_CODE | _ - 0 | _ - 1 | _ } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 628ef2fe4..1b8ea51de 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -72,22 +72,18 @@ class KubeBuildStrategyTest extends Specification { def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') Files.createDirectories(req.workDir) + strategy.build('build-job-name', req) - def resp = strategy.build(req) then: - resp - and: - 1 * k8sService.buildContainer(_, _, _, _, _, _, _, [service:'wave-build']) >> null + 1 * k8sService.launchBuildJob(_, _, _, _, _, _, _, [service:'wave-build']) >> null when: def req2 = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') Files.createDirectories(req2.workDir) + strategy.build('job-name', req2) - def resp2 = strategy.build(req2) then: - resp2 - and: - 1 * k8sService.buildContainer(_, _, _, _, _, _, _, [service:'wave-build-arm64']) >> null + 1 * k8sService.launchBuildJob(_, _, _, _, _, _, _, [service:'wave-build-arm64']) >> null } @@ -120,22 +116,4 @@ class KubeBuildStrategyTest extends Specification { strategy.getBuildImage(req) == 'quay.io/singularity/singularity:v3.11.4-slim-arm64' } - def 'should get correct pod name for build' () { - given: - def USER = new PlatformId(new User(id:1, email: 'foo@user.com')) - def PATH = Files.createTempDirectory('test') - def repo = 'docker.io/wave' - def cache = 'docker.io/cache' - def dockerfile = 'from foo' - def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), repo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') - - when: - def podName = strategy.podName(req) - - then: - req.buildId == '143ee73bcdac45b1_1' - podName == 'build-143ee73bcdac45b1-1' - } } diff --git a/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy b/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy index ac2f13ca8..79c679b7e 100644 --- a/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.cache.impl +import spock.lang.Shared import spock.lang.Specification import java.time.Duration @@ -27,8 +28,10 @@ import io.seqera.wave.test.RedisTestContainer class RedisCacheProviderTest extends Specification implements RedisTestContainer { + @Shared ApplicationContext applicationContext + @Shared RedisCacheProvider redisCacheProvider def setup() { @@ -40,6 +43,10 @@ class RedisCacheProviderTest extends Specification implements RedisTestContainer sleep(500) // workaround to wait for Redis connection } + def cleanup() { + applicationContext.close() + } + def 'conditional put with current value when ke is not set'() { when: 'conditionally set a key that has no current value' def current = redisCacheProvider.putIfAbsentAndGetCurrent('key', 'new-value', Duration.ofSeconds(100)) diff --git a/src/test/groovy/io/seqera/wave/service/cleanup/CleanupConfigTest.groovy b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupConfigTest.groovy new file mode 100644 index 000000000..e34e114c4 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupConfigTest.groovy @@ -0,0 +1,42 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cleanup + +import spock.lang.Specification + +import java.time.Duration + +/** + * + * @author Paolo Di Tommaso + */ +class CleanupConfigTest extends Specification { + + def 'should get random delay' () { + when: + def d = Duration.ofSeconds(10) + def config = new CleanupConfig(cleanupStartupDelay: d) + then: + config.cleanupStartupDelay == d + and: + config.cleanupStartupDelayRandomized >= d.dividedBy(2) + config.cleanupStartupDelayRandomized < (d + d.dividedBy(2)) + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/cleanup/CleanupServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupServiceTest.groovy new file mode 100644 index 000000000..fe0cc3cbe --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupServiceTest.groovy @@ -0,0 +1,43 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cleanup + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class CleanupServiceTest extends Specification { + + def 'should validate cleanup entry' () { + given: + def service = Spy(new CleanupServiceImpl()) + + when: + service.cleanupEntry('job:foo') + then: + 1 * service.cleanupJob0('foo') >> null + + when: + service.cleanupEntry('dir:/some/data/dir') + then: + 1 * service.cleanupDir0('/some/data/dir') >> null + } +} diff --git a/src/test/groovy/io/seqera/wave/service/cleanup/CleanupStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupStrategyTest.groovy new file mode 100644 index 000000000..acffd5b8d --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupStrategyTest.groovy @@ -0,0 +1,59 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cleanup + +import spock.lang.Specification +import spock.lang.Unroll +/** + * + * @author Paolo Di Tommaso + */ +class CleanupStrategyTest extends Specification { + + @Unroll + def 'should validate cleanup rule' () { + given: + def strategy = new CleanupStrategy(config: new CleanupConfig(strategy: CONFIG), debugMode: DEBUG) + + expect: + strategy.shouldCleanup(STATUS) == EXPECTED + + where: + STATUS | CONFIG | DEBUG | EXPECTED + 0 | null | false | true + 1 | null | false | true + and: + 0 | null | true | false + 1 | null | true | false + and: + 0 | 'always' | false | true + 0 | 'never' | false | false + 0 | 'onsuccess' | false | true + 1 | 'always' | false | true + 1 | 'never' | false | false + 1 | 'onsuccess' | false | false + and: + 0 | 'always' | true | true + 0 | 'never' | true | false + 0 | 'onsuccess' | true | true + 1 | 'always' | true | true + 1 | 'never' | true | false + 1 | 'onsuccess' | true | false + } +} diff --git a/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy b/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy index d592171a9..ef58d6bec 100644 --- a/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.counter.impl +import spock.lang.Shared import spock.lang.Specification import io.micronaut.context.ApplicationContext @@ -28,8 +29,10 @@ import io.seqera.wave.test.RedisTestContainer */ class RedisCounterProviderTest extends Specification implements RedisTestContainer { + @Shared ApplicationContext applicationContext + @Shared RedisCounterProvider redisCounterProvider def setup() { @@ -41,6 +44,10 @@ class RedisCounterProviderTest extends Specification implements RedisTestContain sleep(500) // workaround to wait for Redis connection } + def cleanup() { + applicationContext.close() + } + def 'should increment a counter value' () { expect: diff --git a/src/test/groovy/io/seqera/wave/service/data/future/impl/RedisFutureHashTest.groovy b/src/test/groovy/io/seqera/wave/service/data/future/impl/RedisFutureHashTest.groovy index 4af0038c4..7ba7ddaf5 100644 --- a/src/test/groovy/io/seqera/wave/service/data/future/impl/RedisFutureHashTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/future/impl/RedisFutureHashTest.groovy @@ -33,19 +33,22 @@ import io.seqera.wave.test.RedisTestContainer class RedisFutureHashTest extends Specification implements RedisTestContainer { @Shared - ApplicationContext applicationContext + ApplicationContext context def setup() { - applicationContext = ApplicationContext.run([ + context = ApplicationContext.run([ REDIS_HOST: redisHostName, REDIS_PORT: redisPort ], 'test', 'redis') + } + def cleanup() { + context.stop() } def 'should set and get a value' () { given: - def queue = applicationContext.getBean(RedisFutureHash) + def queue = context.getBean(RedisFutureHash) expect: queue.take('xyz') == null diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy index 480c9013d..8923f6cf3 100644 --- a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy @@ -40,7 +40,6 @@ class AbstractMessageQueueLocalTest extends Specification { @Inject private MessageQueue broker - def 'should send and consume a request'() { given: def queue = new PairingOutboundQueue(broker, Duration.ofMillis(100)) diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy index 4bfc76ca5..ad77e1d11 100644 --- a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy @@ -39,20 +39,22 @@ import io.seqera.wave.test.RedisTestContainer class AbstractMessageQueueRedisTest extends Specification implements RedisTestContainer { @Shared - ApplicationContext applicationContext + ApplicationContext context def setup() { - applicationContext = ApplicationContext.run([ + context = ApplicationContext.run([ REDIS_HOST: redisHostName, REDIS_PORT: redisPort ], 'test', 'redis') - } + def cleanup() { + context.stop() + } def 'should send and consume a request'() { given: - def broker = applicationContext.getBean(RedisMessageQueue) + def broker = context.getBean(RedisMessageQueue) def queue = new PairingOutboundQueue(broker, Duration.ofMillis(100)) and: def result = new CompletableFuture() @@ -70,10 +72,10 @@ class AbstractMessageQueueRedisTest extends Specification implements RedisTestCo def 'should send and consume a request across instances'() { given: - def broker1 = applicationContext.getBean(RedisMessageQueue) + def broker1 = context.getBean(RedisMessageQueue) def queue1 = new PairingOutboundQueue(broker1, Duration.ofMillis(100)) and: - def broker2 = applicationContext.getBean(RedisMessageQueue) + def broker2 = context.getBean(RedisMessageQueue) def queue2 = new PairingOutboundQueue(broker2, Duration.ofMillis(100)) and: def result = new CompletableFuture() diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy index 80286d78a..e670a2ca7 100644 --- a/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy @@ -24,14 +24,12 @@ import spock.lang.Specification import java.time.Duration import io.micronaut.context.ApplicationContext -import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.service.data.queue.impl.RedisMessageQueue import io.seqera.wave.test.RedisTestContainer /** * * @author Paolo Di Tommaso */ -@MicronautTest(environments = ['test']) class RedisMessageQueueTest extends Specification implements RedisTestContainer { @Shared @@ -42,7 +40,10 @@ class RedisMessageQueueTest extends Specification implements RedisTestContainer REDIS_HOST: redisHostName, REDIS_PORT: redisPort ], 'test', 'redis') + } + def cleanup() { + context.stop() } def 'should return null if empty' () { diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy index 8531af73f..f93f73759 100644 --- a/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy @@ -42,13 +42,13 @@ class AbstractMessageStreamLocalTest extends Specification { and: def stream = new TestStream(target) def queue = new ArrayBlockingQueue(10) + and: + stream.addConsumer(id1, { it-> queue.add(it) }) when: stream.offer(id1, new TestMessage('one','two')) stream.offer(id1, new TestMessage('alpha','omega')) then: - stream.consume(id1, { it-> queue.add(it) }) - and: queue.take()==new TestMessage('one','two') queue.take()==new TestMessage('alpha','omega') diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy index 185754655..b6f81becb 100644 --- a/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy @@ -44,6 +44,10 @@ class AbstractMessageStreamRedisTest extends Specification implements RedisTestC ], 'test', 'redis') } + def cleanup() { + context.stop() + } + def 'should offer and consume some messages' () { given: def id1 = "stream-${LongRndKey.rndHex()}" @@ -51,13 +55,13 @@ class AbstractMessageStreamRedisTest extends Specification implements RedisTestC def target = context.getBean(RedisMessageStream) def stream = new TestStream(target) def queue = new ArrayBlockingQueue(10) + and: + stream.addConsumer(id1, { it-> queue.add(it) }) when: stream.offer(id1, new TestMessage('one','two')) stream.offer(id1, new TestMessage('alpha','omega')) then: - stream.consume(id1, { it-> queue.add(it) }) - and: queue.take()==new TestMessage('one','two') queue.take()==new TestMessage('alpha','omega') diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy index eb2a25969..c04d51d1f 100644 --- a/src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy @@ -36,6 +36,9 @@ class LocalMessageStreamTest extends Specification { def id2 = "stream-${LongRndKey.rndHex()}" and: def stream = new LocalMessageStream() + and: + stream.init(id1) + stream.init(id2) when: stream.offer(id1, 'one') and: @@ -57,6 +60,7 @@ class LocalMessageStreamTest extends Specification { given: def id1 = "stream-${LongRndKey.rndHex()}" def stream = new LocalMessageStream() + stream.init(id1) when: stream.offer(id1, 'alpha') stream.offer(id1, 'delta') diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy index b680b0929..f1372f888 100644 --- a/src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy @@ -22,16 +22,13 @@ import spock.lang.Shared import spock.lang.Specification import io.micronaut.context.ApplicationContext -import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.service.data.stream.impl.RedisMessageStream import io.seqera.wave.test.RedisTestContainer import io.seqera.wave.util.LongRndKey - /** * * @author Paolo Di Tommaso */ -@MicronautTest(environments = ['test']) class RedisMessageStreamTest extends Specification implements RedisTestContainer { @Shared @@ -45,12 +42,19 @@ class RedisMessageStreamTest extends Specification implements RedisTestContainer ], 'test', 'redis') } + def cleanup() { + context.stop() + } + def 'should offer and consume a value' () { given: def id1 = "stream-${LongRndKey.rndHex()}" def id2 = "stream-${LongRndKey.rndHex()}" and: def stream = context.getBean(RedisMessageStream) + and: + stream.init(id1) + stream.init(id2) when: stream.offer(id1, 'one') and: @@ -72,6 +76,7 @@ class RedisMessageStreamTest extends Specification implements RedisTestContainer given: def id1 = "stream-${LongRndKey.rndHex()}" def stream = context.getBean(RedisMessageStream) + stream.init(id1) when: stream.offer(id1, 'alpha') stream.offer(id1, 'delta') diff --git a/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy new file mode 100644 index 000000000..2fa094560 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy @@ -0,0 +1,103 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +import io.seqera.wave.configuration.BlobCacheConfig +import io.seqera.wave.configuration.ScanConfig +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.scan.ScanRequest + +/** + * + * @author Paolo Di Tommaso + */ +class JobFactoryTest extends Specification { + + def 'should create job id' () { + given: + def ts = Instant.parse('2024-08-18T19:23:33.650722Z') + def factory = new JobFactory() + and: + def request = new BuildRequest( + targetImage: 'docker.io/foo:bar', + buildId: '12345_9', + startTime: ts, + maxDuration: Duration.ofMinutes(1), + workDir: Path.of('/some/work/dir') + ) + + when: + def job = factory.build(request) + then: + job.recordId == 'docker.io/foo:bar' + job.operationName == 'build-12345-9' + job.creationTime == ts + job.type == JobSpec.Type.Build + job.maxDuration == Duration.ofMinutes(1) + job.workDir == Path.of('/some/work/dir') + } + + def 'should create transfer job' () { + given: + def duration = Duration.ofMinutes(1) + def config = new BlobCacheConfig(transferTimeout: duration) + def factory = new JobFactory(blobConfig:config) + + when: + def job = factory.transfer('foo-123') + then: + job.recordId == 'foo-123' + job.operationName =~ /transfer-.+/ + job.type == JobSpec.Type.Transfer + job.maxDuration == duration + } + + def 'should create scan job' () { + given: + def workdir = Path.of('/some/work/dir') + def duration = Duration.ofMinutes(1) + def config = new ScanConfig(timeout: duration) + def factory = new JobFactory(scanConfig: config) + def request = new ScanRequest( + '12345', + 'build-123', + '{ jsonConfig }', + 'docker.io/foo:bar', + ContainerPlatform.of('linux/amd64'), + workdir + ) + + when: + def job = factory.scan(request) + then: + job.recordId == '12345' + job.operationName == 'scan-12345' + job.type == JobSpec.Type.Scan + job.maxDuration == duration + job.creationTime == request.creationTime + job.workDir == workdir + } +} diff --git a/src/test/groovy/io/seqera/wave/service/job/JobIdTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobIdTest.groovy deleted file mode 100644 index e0dc8a3fd..000000000 --- a/src/test/groovy/io/seqera/wave/service/job/JobIdTest.groovy +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.job - -import spock.lang.Specification - -import java.time.Instant - -/** - * - * @author Paolo Di Tommaso - */ -class JobIdTest extends Specification { - - def 'should create job id' () { - given: - def ts = Instant.parse('2024-08-18T19:23:33.650722Z') - - when: - def job = new JobId(JobId.Type.Transfer, 'foo', ts) - then: - job.id == 'foo' - job.schedulerId == 'transfer-8e5e0d3b81e48cac' - job.creationTime == ts - job.type == JobId.Type.Transfer - - } - - def 'should create transfer job' () { - when: - def job = JobId.transfer('abc-123') - then: - job.id == 'abc-123' - job.schedulerId =~ /transfer-.+/ - job.type == JobId.Type.Transfer - } - - def 'should create build job' () { - when: - def job = JobId.build('abc-123') - then: - job.id == 'abc-123' - job.schedulerId =~ /build-.+/ - job.type == JobId.Type.Build - } - - def 'should create scan job' () { - when: - def job = JobId.scan('abc-123') - then: - job.id == 'abc-123' - job.schedulerId =~ /scan-.+/ - job.type == JobId.Type.Scan - } - -} diff --git a/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy index 2ad51b6c8..f58513a13 100644 --- a/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy @@ -20,109 +20,158 @@ package io.seqera.wave.service.job import spock.lang.Specification +import java.nio.file.Path import java.time.Duration +import java.time.Instant + +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.Caffeine + /** * * @author Munish Chouhan */ class JobManagerTest extends Specification { - def "handle should process valid transferId"() { + def 'processJob should handle valid TransferJobSpec'() { given: - def jobStrategy = Mock(JobStrategy) + def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) - def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + def config = new JobConfig(graceInterval: Duration.ofMillis(1)) + def cache = Caffeine.newBuilder().build() + def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: - def job = JobId.transfer('foo') + def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now(), Duration.ofMinutes(10)) when: - def done = manager.processJob(job) + def result = manager.processJob0(jobSpec) + then: - 1 * jobStrategy.status(job) >> JobState.completed(0, 'My job logs') - and: - 1 * jobDispatcher.onJobCompletion(job, _) >> { JobId id, JobState state -> - assert state.exitCode == 0 - assert state.stdout == 'My job logs' - } - and: - 1 * jobStrategy.cleanup(job,0) - and: - done + 1 * jobService.status(jobSpec) >> JobState.completed(0, 'My job logs') + 1 * jobDispatcher.notifyJobCompletion(jobSpec, _) + result } - def "handle should log error for unknown transferId"() { + def 'processJob should handle exception for TransferJobSpec'() { given: - def jobStrategy = Mock(JobStrategy) + def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) - def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + def config = new JobConfig(graceInterval: Duration.ofMillis(1)) + def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config) and: - def job = JobId.transfer('unknown') + def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now(), Duration.ofMinutes(10)) when: - def done = manager.processJob(job) + def result = manager.processJob(jobSpec) + then: - 1 * jobStrategy.status(job) >> null - and: - 1 * jobDispatcher.onJobException(job,_) >> null - and: - done + 1 * jobService.status(jobSpec) >> { throw new RuntimeException('Error') } + 1 * jobDispatcher.notifyJobException(jobSpec, _) + result } - def "handle0 should fail transfer when status is unknown and duration exceeds grace period"() { + def 'processJob0 should timeout TransferJobSpec when duration exceeds max limit'() { given: - def jobStrategy = Mock(JobStrategy) + def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) - def config = new JobConfig(graceInterval: Duration.ofMillis(500)) - def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher, config:config) + def config = new JobConfig(graceInterval: Duration.ofMillis(1)) + def cache = Caffeine.newBuilder().build() + def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config:config, debounceCache: cache) and: - def job = JobId.transfer('foo') + def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now() - Duration.ofMinutes(5), Duration.ofMinutes(2)) when: - sleep 1_000 //sleep longer than grace period - def done = manager.processJob(job) + def result = manager.processJob0(jobSpec) then: - 1 * jobStrategy.status(job) >> JobState.unknown('logs') - 1 * jobDispatcher.onJobCompletion(job, _) - 1 * jobStrategy.cleanup(job, _) - and: - done + 1 * jobService.status(jobSpec) >> JobState.running() + 1 * jobDispatcher.notifyJobTimeout(jobSpec) + result } - def "should requeue transfer when duration is within limits"() { + def 'processJob0 should requeue TransferJobSpec when duration is within limits'() { given: - def jobStrategy = Mock(JobStrategy) + def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) - def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + def config = new JobConfig(graceInterval: Duration.ofMillis(1)) + def cache = Caffeine.newBuilder().build() + def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: - def job = JobId.transfer('foo') + def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now().minus(Duration.ofMillis(500)), Duration.ofMinutes(10)) when: - def done = manager.processJob(job) + def result = manager.processJob0(jobSpec) + then: - 1 * jobStrategy.status(job) >> JobState.running() - 1 * jobDispatcher.jobMaxDuration(job) >> Duration.ofSeconds(10) - and: - !done + 1 * jobService.status(jobSpec) >> JobState.running() + !result } - def "handle0 should timeout transfer when duration exceeds max limit"() { + def 'should validate unknown state cache' () { given: - def jobStrategy = Mock(JobStrategy) - def jobDispatcher = Mock(JobDispatcher) - def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + Cache cache = Caffeine + .newBuilder() + .expireAfterWrite(Duration.ofMinutes(1)) + .build() and: - def job = JobId.transfer('foo') + def jobService = Mock(JobService) + def manager = new JobManager(jobService: jobService) + and: + def job1 = new JobSpec('1', JobSpec.Type.Build, '1', '1', Instant.now(), Duration.ofMinutes(1), Mock(Path)) + and: + def PENDING = new JobState(JobState.Status.PENDING, null, null) + def UNKNOWN = new JobState(JobState.Status.UNKNOWN, null, null) + def FAILED = new JobState(JobState.Status.FAILED, null, null) + def _100_ms = Duration.ofMillis(100) + def _1_sec = Duration.ofSeconds(1) + and: + JobState result when: - sleep 1_000 //await timeout - def done = manager.processJob(job) + result = manager.state0(job1, _1_sec, cache) then: - 1 * jobStrategy.status(job) >> JobState.running() - 1 * jobDispatcher.jobMaxDuration(job) >> Duration.ofMillis(100) + jobService.status(job1) >> PENDING and: - 1 * jobDispatcher.onJobTimeout(job) + result == PENDING + cache.getIfPresent('1') == null + + // now return an unknown status + when: + result = manager.state0(job1, _100_ms, cache) + then: + jobService.status(job1) >> UNKNOWN + and: + result == UNKNOWN + cache.getIfPresent('1') != null + + // the following state is pending, so unknown is cleared + when: + sleep 150 + result = manager.state0(job1, _100_ms, cache) + then: + jobService.status(job1) >> PENDING + and: + result == PENDING + cache.getIfPresent('1') == null + + // now two unknown for longer than the grace period + when: + result = manager.state0(job1, _100_ms, cache) + then: + jobService.status(job1) >> UNKNOWN + and: + result == UNKNOWN + cache.getIfPresent('1') != null + + when: + sleep 150 + and: + result = manager.state0(job1, _100_ms, cache) + then: + jobService.status(job1) >> UNKNOWN and: - done + result == FAILED + cache.getIfPresent('1') == null + } } diff --git a/src/test/groovy/io/seqera/wave/service/job/JobSpecTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobSpecTest.groovy new file mode 100644 index 000000000..507ca9712 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/job/JobSpecTest.groovy @@ -0,0 +1,96 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +/** + * + * @author Paolo Di Tommaso + */ +class JobSpecTest extends Specification { + + def 'should validate constructor' () { + given: + def ts = Instant.now() + when: + def job = new JobSpec( + '1234', + JobSpec.Type.Build, + 'record-123', + 'oper-123', + ts, + Duration.ofMinutes(1), + Path.of('/some/path') + ) + then: + job.id == '1234' + job.recordId == 'record-123' + job.operationName == 'oper-123' + job.creationTime == ts + job.maxDuration == Duration.ofMinutes(1) + job.workDir == Path.of('/some/path') + + } + + def 'should create transfer job' () { + given: + def now = Instant.now() + def job = JobSpec.transfer('12345','xyz', now, Duration.ofMinutes(1)) + expect: + job.id + job.recordId == '12345' + job.type == JobSpec.Type.Transfer + job.creationTime == now + job.maxDuration == Duration.ofMinutes(1) + job.operationName == 'xyz' + } + + def 'should create build job' () { + given: + def now = Instant.now() + def job = JobSpec.build('12345','xyz', now, Duration.ofMinutes(1), Path.of('/some/path')) + expect: + job.id + job.recordId == '12345' + job.type == JobSpec.Type.Build + job.creationTime == now + job.maxDuration == Duration.ofMinutes(1) + job.operationName == 'xyz' + job.workDir == Path.of('/some/path') + } + + def 'should create scan job' () { + given: + def now = Instant.now() + def job = JobSpec.scan('12345','xyz', now, Duration.ofMinutes(1), Path.of('/some/path')) + expect: + job.id + job.recordId == '12345' + job.type == JobSpec.Type.Scan + job.creationTime == now + job.maxDuration == Duration.ofMinutes(1) + job.operationName == 'xyz' + job.workDir == Path.of('/some/path') + } +} diff --git a/src/test/groovy/io/seqera/wave/service/job/TransferTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobStateTest.groovy similarity index 82% rename from src/test/groovy/io/seqera/wave/service/job/TransferTest.groovy rename to src/test/groovy/io/seqera/wave/service/job/JobStateTest.groovy index 76a29cf80..a18a7e4be 100644 --- a/src/test/groovy/io/seqera/wave/service/job/TransferTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobStateTest.groovy @@ -26,7 +26,7 @@ import spock.lang.Unroll * * @author Paolo Di Tommaso */ -class TransferTest extends Specification { +class JobStateTest extends Specification { @Unroll def 'should validate completed status' () { @@ -50,12 +50,12 @@ class TransferTest extends Specification { where: STATUS | EXIT | EXPECTED - JobState.Status.PENDING | null | false - JobState.Status.RUNNING | null | false - JobState.Status.UNKNOWN | null | false - JobState.Status.FAILED | null | false - JobState.Status.SUCCEEDED | 1 | false - JobState.Status.SUCCEEDED | 0 | true + JobState.Status.PENDING | null | false + JobState.Status.RUNNING | null | false + JobState.Status.UNKNOWN | null | false + JobState.Status.FAILED | null | false + JobState.Status.SUCCEEDED | 1 | false + JobState.Status.SUCCEEDED | 0 | true } diff --git a/src/test/groovy/io/seqera/wave/service/job/impl/DockerJobOperationTest.groovy b/src/test/groovy/io/seqera/wave/service/job/impl/DockerJobOperationTest.groovy new file mode 100644 index 000000000..121054d50 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/job/impl/DockerJobOperationTest.groovy @@ -0,0 +1,43 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job.impl + +import spock.lang.Specification +import spock.lang.Unroll +/** + * + * @author Paolo Di Tommaso + */ +class DockerJobOperationTest extends Specification { + + @Unroll + def 'should parse state string' () { + expect: + DockerJobOperation.State.parse(STATE) == EXPECTED + + where: + STATE | EXPECTED + 'running' | new DockerJobOperation.State('running') + 'exited' | new DockerJobOperation.State('exited') + 'exited,' | new DockerJobOperation.State('exited') + 'exited,0' | new DockerJobOperation.State('exited', 0) + 'exited,10' | new DockerJobOperation.State('exited', 10) + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/job/impl/K8SJobOperationTest.groovy b/src/test/groovy/io/seqera/wave/service/job/impl/K8SJobOperationTest.groovy new file mode 100644 index 000000000..ae388eba5 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/job/impl/K8SJobOperationTest.groovy @@ -0,0 +1,115 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job.impl + + +import spock.lang.Specification +import spock.lang.Unroll + +import io.kubernetes.client.openapi.models.V1ContainerState +import io.kubernetes.client.openapi.models.V1ContainerStateTerminated +import io.kubernetes.client.openapi.models.V1ContainerStatus +import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodStatus +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.k8s.K8sService +/** + * + * @author Paolo Di Tommaso + */ +class K8SJobOperationTest extends Specification { + + K8sService k8sService = Mock(K8sService) + K8sJobOperation strategy = new K8sJobOperation(k8sService: k8sService) + + def 'status should return correct status when job is not completed'() { + given: + def job = Mock(JobSpec) + and: + k8sService.getJobStatus(job.operationName) >> K8sService.JobStatus.Running + + when: + def result = strategy.status(job) + then: + result.status == JobState.Status.RUNNING + } + + void 'status should return correct transfer status when pods are created'() { + given: + def job = Mock(JobSpec) + and: + def status = new V1PodStatus(phase: "Succeeded", containerStatuses: [new V1ContainerStatus( state: new V1ContainerState(terminated: new V1ContainerStateTerminated(exitCode: 0)))]) + def pod = new V1Pod(metadata: [name: job.operationName], status: status) + and: + k8sService.getJobStatus(job.operationName) >> K8sService.JobStatus.Succeeded + k8sService.getLatestPodForJob(job.operationName) >> pod + k8sService.logsPod(pod) >> "transfer successful" + + when: + def result = strategy.status(job) + then: + result.status == JobState.Status.SUCCEEDED + result.exitCode == 0 + result.stdout == "transfer successful" + } + + def 'status should return failed transfer when no pods are created'() { + given: + def job = Mock(JobSpec) + and: + def status = new V1PodStatus(phase: "Failed") + def pod = new V1Pod(metadata: [name: job.operationName], status: status) + and: + k8sService.getLatestPodForJob(job.operationName) >> pod + k8sService.getJobStatus(job.operationName) >> K8sService.JobStatus.Failed + + when: + def result = strategy.status(job) + then: + result.status == JobState.Status.FAILED + } + + def 'status should handle null job status'() { + given: + def job = Mock(JobSpec) + and: + k8sService.getJobStatus(job.operationName) >> null + + when: + def result = strategy.status(job) + then: + result.status == JobState.Status.UNKNOWN + } + + @Unroll + def "mapToStatus should return correct transfer status for jobStatus #JOB_STATUS that is #TRANSFER_STATUS"() { + expect: + K8sJobOperation.mapToStatus(JOB_STATUS) == TRANSFER_STATUS + + where: + JOB_STATUS | TRANSFER_STATUS + K8sService.JobStatus.Pending | JobState.Status.PENDING + K8sService.JobStatus.Running | JobState.Status.RUNNING + K8sService.JobStatus.Succeeded | JobState.Status.SUCCEEDED + K8sService.JobStatus.Failed | JobState.Status.FAILED + null | JobState.Status.UNKNOWN + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 2d7f89dd3..3f85ebb63 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.service.k8s import spock.lang.Specification +import spock.lang.Unroll import java.nio.file.Path import java.time.Duration @@ -26,13 +27,18 @@ import java.time.OffsetDateTime import io.kubernetes.client.custom.Quantity import io.kubernetes.client.openapi.ApiClient +import io.kubernetes.client.openapi.apis.BatchV1Api import io.kubernetes.client.openapi.apis.CoreV1Api import io.kubernetes.client.openapi.models.V1EnvVar +import io.kubernetes.client.openapi.models.V1Job +import io.kubernetes.client.openapi.models.V1JobSpec +import io.kubernetes.client.openapi.models.V1JobStatus import io.kubernetes.client.openapi.models.V1ObjectMeta import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1PodStatus import io.micronaut.context.ApplicationContext +import io.micronaut.context.annotation.Replaces import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig @@ -44,6 +50,14 @@ import io.seqera.wave.configuration.SpackConfig @MicronautTest class K8sServiceImplTest extends Specification { + @Replaces(ScanConfig.class) + static class MockScanConfig extends ScanConfig { + @Override + Path getCacheDirectory() { + return Path.of('/build/scan/cache') + } + } + def 'should validate context OK ' () { when: def PROPS = [ @@ -51,7 +65,8 @@ class K8sServiceImplTest extends Specification { 'wave.build.k8s.namespace': 'foo', 'wave.build.k8s.configPath': '/home/kube.config', 'wave.build.k8s.storage.claimName': 'bar', - 'wave.build.k8s.storage.mountPath': '/build' ] + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.scan.enabled': 'true'] and: def ctx = ApplicationContext.run(PROPS) ctx.getBean(K8sServiceImpl) @@ -507,10 +522,8 @@ class K8sServiceImplTest extends Specification { def ctx = ApplicationContext.run(PROPS) def k8sService = ctx.getBean(K8sServiceImpl) def config = Mock(BlobCacheConfig) { - getTransferTimeout() >> Duration.ofSeconds(20) getEnvironment() >> [:] getRetryAttempts() >> 5 - getDeleteAfterFinished() >> Duration.ofDays(10) } when: @@ -521,10 +534,8 @@ class K8sServiceImplTest extends Specification { result.metadata.namespace == 'my-ns' and: result.spec.backoffLimit == 5 - result.spec.ttlSecondsAfterFinished == Duration.ofDays(10).seconds as Integer and: verifyAll(result.spec.template.spec) { - activeDeadlineSeconds == 20 serviceAccount == null containers.get(0).name == 'foo' containers.get(0).image == 'my-image:latest' @@ -547,12 +558,10 @@ class K8sServiceImplTest extends Specification { def ctx = ApplicationContext.run(PROPS) def k8sService = ctx.getBean(K8sServiceImpl) def config = Mock(BlobCacheConfig) { - getTransferTimeout() >> Duration.ofSeconds(20) getEnvironment() >> ['FOO':'one', 'BAR':'two'] getRequestsCpu() >> '2' getRequestsMemory() >> '8Gi' getRetryAttempts() >> 3 - getDeleteAfterFinished() >> Duration.ofDays(1) } when: @@ -562,10 +571,8 @@ class K8sServiceImplTest extends Specification { result.metadata.namespace == 'my-ns' and: result.spec.backoffLimit == 3 - result.spec.ttlSecondsAfterFinished == Duration.ofDays(1).seconds as Integer and: verifyAll(result.spec.template.spec) { - activeDeadlineSeconds == 20 serviceAccount == 'foo-sa' containers.get(0).name == 'foo' containers.get(0).image == 'my-image:latest' @@ -684,4 +691,354 @@ class K8sServiceImplTest extends Specification { latestPod == null } + def 'buildJobSpec should create job with singularity image'() { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.k8s.service-account': 'theAdminAccount', + 'wave.build.deleteAfterFinished': '1d', + 'wave.build.retry-attempts': 3 + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def name = 'test-job' + def containerImage = 'singularity://test-image' + def args = ['arg1', 'arg2'] + def workDir = Path.of('/work/dir') + def credsFile = Path.of('/creds/file') + def timeout = Duration.ofMinutes(10) + def spackConfig = new SpackConfig(secretKeyFile: Path.of('/build/secret/key'), secretMountPath: '/secret/mount') + def nodeSelector = [key: 'value'] + + when: + def job = k8sService.buildJobSpec(name, containerImage, args, workDir, credsFile, timeout, spackConfig, nodeSelector) + + then: + job.spec.backoffLimit == 3 + job.spec.template.spec.containers[0].image == containerImage + job.spec.template.spec.containers[0].command == args + job.spec.template.spec.containers[0].securityContext.privileged + + cleanup: + ctx.close() + } + + def 'buildJobSpec should create job with docker image'() { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.k8s.service-account': 'theAdminAccount' + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def name = 'test-job' + def containerImage = 'docker://test-image' + def args = ['arg1', 'arg2'] + def workDir = Path.of('/work/dir') + def credsFile = Path.of('/creds/file') + def timeout = Duration.ofMinutes(10) + def spackConfig = new SpackConfig(secretKeyFile: Path.of('/build/secret/key'), secretMountPath: '/secret/mount') + def nodeSelector = [key: 'value'] + + when: + def job = k8sService.buildJobSpec(name, containerImage, args, workDir, credsFile, timeout, spackConfig, nodeSelector) + + then: + job.spec.template.spec.containers[0].image == containerImage + job.spec.template.spec.containers[0].env.find { it.name == 'BUILDKITD_FLAGS' } + job.spec.template.spec.containers[0].command == ['buildctl-daemonless.sh'] + job.spec.template.spec.containers[0].args == args + + cleanup: + ctx.close() + } + + def 'should create scan job spec with valid inputs'() { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.k8s.service-account': 'theAdminAccount' + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def name = 'scan-job' + def containerImage = 'scan-image:latest' + def args = ['arg1', 'arg2'] + def workDir = Path.of('/work/dir') + def credsFile = Path.of('/creds/file') + def scanConfig = Mock(ScanConfig) { + getCacheDirectory() >> Path.of('/build/cache/dir') + getRequestsCpu() >> '2' + getRequestsMemory() >> '4Gi' + } + def nodeSelector = [key: 'value'] + + when: + def job = k8sService.scanJobSpec(name, containerImage, args, workDir, credsFile, scanConfig, nodeSelector) + + then: + job.metadata.name == name + job.metadata.namespace == 'foo' + job.spec.template.spec.containers[0].image == containerImage + job.spec.template.spec.containers[0].args == args + job.spec.template.spec.containers[0].resources.requests.get('cpu') == new Quantity('2') + job.spec.template.spec.containers[0].resources.requests.get('memory') == new Quantity('4Gi') + job.spec.template.spec.volumes.size() == 1 + job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar' + job.spec.template.spec.nodeSelector == nodeSelector + job.spec.template.spec.restartPolicy == 'Never' + + cleanup: + ctx.close() + } + + def 'should create scan job spec without creds file'() { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.k8s.service-account': 'theAdminAccount' + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def name = 'scan-job' + def containerImage = 'scan-image:latest' + def args = ['arg1', 'arg2'] + def workDir = Path.of('/work/dir') + def credsFile = null + def scanConfig = Mock(ScanConfig) { + getCacheDirectory() >> Path.of('/build/cache/dir') + getRequestsCpu() >> '2' + getRequestsMemory() >> '4Gi' + } + def nodeSelector = [key: 'value'] + + when: + def job = k8sService.scanJobSpec(name, containerImage, args, workDir, credsFile, scanConfig, nodeSelector) + + then: + job.metadata.name == name + job.metadata.namespace == 'foo' + job.spec.template.spec.containers[0].image == containerImage + job.spec.template.spec.containers[0].args == args + job.spec.template.spec.containers[0].resources.requests.get('cpu') == new Quantity('2') + job.spec.template.spec.containers[0].resources.requests.get('memory') == new Quantity('4Gi') + job.spec.template.spec.volumes.size() == 1 + job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar' + job.spec.template.spec.nodeSelector == nodeSelector + job.spec.template.spec.restartPolicy == 'Never' + + cleanup: + ctx.close() + } + + def 'should create scan job spec without node selector'() { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.k8s.service-account': 'theAdminAccount', + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def name = 'scan-job' + def containerImage = 'scan-image:latest' + def args = ['arg1', 'arg2'] + def workDir = Path.of('/work/dir') + def credsFile = Path.of('/creds/file') + def scanConfig = Mock(ScanConfig) { + getCacheDirectory() >> Path.of('/build/cache/dir') + getRequestsCpu() >> '2' + getRequestsMemory() >> '4Gi' + getRetryAttempts() >> 3 + } + def nodeSelector = null + + when: + def job = k8sService.scanJobSpec(name, containerImage, args, workDir, credsFile, scanConfig, nodeSelector) + + then: + job.metadata.name == name + job.metadata.namespace == 'foo' + job.spec.backoffLimit == 3 + job.spec.template.spec.containers[0].image == containerImage + job.spec.template.spec.containers[0].args == args + job.spec.template.spec.containers[0].resources.requests.get('cpu') == new Quantity('2') + job.spec.template.spec.containers[0].resources.requests.get('memory') == new Quantity('4Gi') + job.spec.template.spec.volumes.size() == 1 + job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar' + job.spec.template.spec.nodeSelector == null + job.spec.template.spec.restartPolicy == 'Never' + + cleanup: + ctx.close() + } + + def 'should create scan job spec without resource requests'() { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.k8s.service-account': 'theAdminAccount', + 'wave.scan.retry-attempts': 3 + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def name = 'scan-job' + def containerImage = 'scan-image:latest' + def args = ['arg1', 'arg2'] + def workDir = Path.of('/work/dir') + def credsFile = Path.of('/creds/file') + def scanConfig = Mock(ScanConfig) { + getCacheDirectory() >> Path.of('/build/cache/dir') + getRequestsCpu() >> null + getRequestsMemory() >> null + getRetryAttempts() >> 3 + } + def nodeSelector = [key: 'value'] + + when: + def job = k8sService.scanJobSpec(name, containerImage, args, workDir, credsFile, scanConfig, nodeSelector) + + then: + job.metadata.name == name + job.metadata.namespace == 'foo' + job.spec.backoffLimit == 3 + job.spec.template.spec.containers[0].image == containerImage + job.spec.template.spec.containers[0].args == args + job.spec.template.spec.containers[0].resources.requests == null + job.spec.template.spec.volumes.size() == 1 + job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar' + job.spec.template.spec.nodeSelector == nodeSelector + job.spec.template.spec.restartPolicy == 'Never' + + cleanup: + ctx.close() + } + + + private V1Job jobActive() { + def status = new V1JobStatus() + status.startTime(OffsetDateTime.now()) + status.setActive(1) + status.setFailed(2) + def result = new V1Job() + result.setStatus(status) + return result + } + + private V1Job jobSucceeded() { + def status = new V1JobStatus() + status.startTime(OffsetDateTime.now()) + status.setSucceeded(1) + status.setFailed(2) + + def result = new V1Job() + result.setStatus(status) + return result + } + + private V1Job jobFailed() { + def status = new V1JobStatus(); + status.startTime(OffsetDateTime.now()) + status.setFailed(3) // <-- failed 3 times + def spec = new V1JobSpec() + spec.setBackoffLimit(2) // <-- max 2 retries + def result = new V1Job() + result.setStatus(status) + result.setSpec(spec) + return result + } + + private V1Job jobFailedWitMoreRetries() { + def status = new V1JobStatus(); + status.startTime(OffsetDateTime.now()) + status.setFailed(1) // <-- failed 1 time + def spec = new V1JobSpec() + spec.setBackoffLimit(2) // <-- max 2 retries + def result = new V1Job() + result.setStatus(status) + result.setSpec(spec) + return result + } + + private V1Job jobCompleted() { + def status = new V1JobStatus(); + status.startTime(OffsetDateTime.now()) + status.setFailed(1) // <-- failed 1 time + status.setCompletionTime(OffsetDateTime.now()) + def result = new V1Job() + result.setStatus(status) + return result + } + + private V1Job jobStarted() { + def status = new V1JobStatus(); + status.startTime(OffsetDateTime.now()) + def result = new V1Job() + result.setStatus(status) + return result + } + + private V1Job jobUnknown() { + def status = new V1JobStatus(); + def result = new V1Job() + result.setStatus(status) + return result + } + + @Unroll + def 'should validate get status' () { + given: + def NS = 'foo' + def NAME = 'bar' + def api = Mock(BatchV1Api) + def client = Mock(K8sClient) { batchV1Api()>>api } + def service = Spy(new K8sServiceImpl(namespace:NS, k8sClient: client)) + + when: + def status = service.getJobStatus(NAME) + then: + 1 * api.readNamespacedJob(NAME, NS, null) >> JOB + and: + status == EXPECTED + + where: + JOB | EXPECTED + null | null + jobActive() | K8sService.JobStatus.Pending + jobSucceeded() | K8sService.JobStatus.Succeeded + jobFailed() | K8sService.JobStatus.Failed + jobFailedWitMoreRetries() | K8sService.JobStatus.Pending + jobCompleted() | K8sService.JobStatus.Failed + jobStarted() | K8sService.JobStatus.Pending + jobUnknown() | K8sService.JobStatus.Pending + } } diff --git a/src/test/groovy/io/seqera/wave/service/metric/MetricsCounterStoreRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/metric/MetricsCounterStoreRedisTest.groovy index fb2c179d3..81d53eb4d 100644 --- a/src/test/groovy/io/seqera/wave/service/metric/MetricsCounterStoreRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/metric/MetricsCounterStoreRedisTest.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.metric +import spock.lang.Shared import spock.lang.Specification import io.micronaut.context.ApplicationContext @@ -29,8 +30,11 @@ import redis.clients.jedis.Jedis * @author Munish Chouhan */ class MetricsCounterStoreRedisTest extends Specification implements RedisTestContainer { + + @Shared ApplicationContext applicationContext + @Shared Jedis jedis def setup() { @@ -45,6 +49,7 @@ class MetricsCounterStoreRedisTest extends Specification implements RedisTestCo def cleanup(){ jedis.flushAll() jedis.close() + applicationContext.close() } def 'should get correct count value' () { diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy index 88b6720fe..dddf9b58f 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.service.persistence import spock.lang.Specification +import spock.lang.Unroll import java.time.Duration import java.time.Instant @@ -38,6 +39,7 @@ class WaveScanRecordTest extends Specification { def duration = Duration.ofMinutes(2) def scanId = '12345' def buildId = "testbuildid" + def containerImage = "testcontainerimage" def scanVulnerability = new ScanVulnerability( "id1", "low", @@ -52,6 +54,7 @@ class WaveScanRecordTest extends Specification { def scanResult= new ScanResult( scanId, buildId, + containerImage, startTime, duration, 'SUCCEEDED', @@ -59,6 +62,7 @@ class WaveScanRecordTest extends Specification { then: scanResult.id == scanId scanResult.buildId == buildId + scanResult.containerImage == containerImage scanResult.isCompleted() scanResult.isSucceeded() @@ -70,4 +74,15 @@ class WaveScanRecordTest extends Specification { scanRecord.buildId == buildId scanRecord.vulnerabilities[0] == scanVulnerability } + + @Unroll + def 'should validate done' () { + expect: + RECORD.done() == EXPECTED + + where: + EXPECTED| RECORD + false | new WaveScanRecord('123', 'scan-123', 'testcontainerimage', Instant.now()) + true | new WaveScanRecord('123', 'scan-123', 'testcontainerimage', Instant.now(), Duration.ofMinutes(1), 'OK', []) + } } diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index fe8050177..b21215e5f 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -265,12 +265,13 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def NOW = Instant.now() def SCAN_ID = 'a1' def BUILD_ID = '100' + def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') - def scanRecord = new WaveScanRecord(SCAN_ID, BUILD_ID, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3]) + def scanRecord = new WaveScanRecord(SCAN_ID, BUILD_ID, CONTAINER_IMAGE, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3]) when: - persistence.createScanRecord(new WaveScanRecord(SCAN_ID, BUILD_ID, NOW)) + persistence.createScanRecord(new WaveScanRecord(SCAN_ID, BUILD_ID, CONTAINER_IMAGE, NOW)) persistence.updateScanRecord(scanRecord) then: def result = persistence.loadScanRecord(SCAN_ID) @@ -285,9 +286,9 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe when: def SCAN_ID2 = 'b2' def BUILD_ID2 = '102' - def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1]) + def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, CONTAINER_IMAGE, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1]) and: - persistence.createScanRecord(new WaveScanRecord(SCAN_ID2, BUILD_ID2, NOW)) + persistence.createScanRecord(new WaveScanRecord(SCAN_ID2, BUILD_ID2, CONTAINER_IMAGE, NOW)) // should save the same CVE into another build persistence.updateScanRecord(scanRecord2) then: diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy new file mode 100644 index 000000000..1fdad79a5 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -0,0 +1,183 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveScanRecord +import jakarta.inject.Inject +/** + * Tests for ContainerScanServiceImpl + * + * @author Munish Chouhan + */ +@MicronautTest +class ContainerScanServiceImplTest extends Specification { + + @Inject ContainerScanServiceImpl containerScanService + + @Inject PersistenceService persistenceService + + def 'should start scan successfully'() { + given: + def workDir = Files.createTempDirectory('test') + def scanRequest = new ScanRequest('scan-1', 'build-1', null, 'ubuntu:latest', ContainerPlatform.of('linux/amd64'), workDir, Instant.now()) + + when: + containerScanService.scan(scanRequest) + sleep 500 // wait for the scan record to be stored in db + + then: + def scanRecord = persistenceService.loadScanRecord(scanRequest.id) + scanRecord.id == scanRequest.id + scanRecord.buildId == scanRequest.buildId + + cleanup: + workDir?.deleteDir() + } + + def 'should handle job completion event and update scan record'() { + given: + def trivyDockerResulJson = """ + {"Results": [ + { + "Target": "redis (debian 12.0)", + "Class": "os-pkgs", + "Type": "debian", + "Vulnerabilities": [ + + { + "VulnerabilityID": "CVE-2010-4756", + "PkgID": "libc-bin@2.36-9", + "PkgName": "libc-bin", + "InstalledVersion": "2.36-9", + "FixedVersion": "1.1.1n-0+deb11u5", + "Layer": { + "Digest": "sha256:faef57eae888cbe4a5613eca6741b5e48d768b83f6088858aee9a5a2834f8151", + "DiffID": "sha256:24839d45ca455f36659219281e0f2304520b92347eb536ad5cc7b4dbb8163588" + }, + "SeveritySource": "debian", + "PrimaryURL": "https://avd.aquasec.com/nvd/cve-2010-4756", + "DataSource": { + "ID": "debian", + "Name": "Debian Security Tracker", + "URL": "https://salsa.debian.org/security-tracker-team/security-tracker" + }, + "Title": "glibc: glob implementation can cause excessive CPU and memory consumption due to crafted glob expressions", + "Description": "The glob implementation in the GNU C Library (aka glibc or libc6) allows remote authenticated users to cause a denial of service (CPU and memory consumption) via crafted glob expressions that do not match any pathnames, as demonstrated by glob expressions in STAT commands to an FTP daemon, a different vulnerability than CVE-2010-2632.", + "Severity": "LOW", + "CweIDs": [ + "CWE-399" + ], + "CVSS": { + "nvd": { + "V2Vector": "AV:N/AC:L/Au:S/C:N/I:N/A:P", + "V2Score": 4 + }, + "redhat": { + "V2Vector": "AV:N/AC:L/Au:N/C:N/I:N/A:P", + "V2Score": 5 + } + }, + "References": [ + "http://cxib.net/stuff/glob-0day.c", + "http://securityreason.com/achievement_securityalert/89", + "http://securityreason.com/exploitalert/9223", + "https://access.redhat.com/security/cve/CVE-2010-4756", + "https://bugzilla.redhat.com/show_bug.cgi?id=681681", + "https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2010-4756", + "https://nvd.nist.gov/vuln/detail/CVE-2010-4756", + "https://www.cve.org/CVERecord?id=CVE-2010-4756" + ], + "PublishedDate": "2011-03-02T20:00:00Z", + "LastModifiedDate": "2021-09-01T12:15:00Z" + }]} + ] + } + """ + def workDir = Files.createTempDirectory('test') + def reportFile = workDir.resolve('report.json') + Files.write(reportFile, trivyDockerResulJson.bytes) + def mockPersistenceService = Mock(PersistenceService) + def jobService = Mock(JobService) + def service = new ContainerScanServiceImpl(persistenceService: mockPersistenceService, jobService: jobService) + def job = JobSpec.scan('scan-1', 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), workDir) + def state = Mock(JobState) + def scan = new WaveScanRecord('scan-1', 'build-1', 'ubuntu:latest', Instant.now()) + + when: + service.onJobCompletion(job, scan, state) + then: + 1 * state.completed() >> true + 1 * mockPersistenceService.updateScanRecord(_ as WaveScanRecord) >> { WaveScanRecord scanRecord -> assert scanRecord.status=='SUCCEEDED' } + + when: + service.onJobCompletion(job, scan, state) + then: + 1 * state.completed() >> false + 1 * mockPersistenceService.updateScanRecord(_ as WaveScanRecord) >> { WaveScanRecord scanRecord -> assert scanRecord.status=='FAILED' } + + cleanup: + workDir?.deleteDir() + } + + def 'should handle job error event and update scan record'() { + given: + def mockPersistenceService = Mock(PersistenceService) + def jobService = Mock(JobService) + def service = new ContainerScanServiceImpl(persistenceService: mockPersistenceService, jobService: jobService) + def job = JobSpec.scan('scan-1', 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) + def error = new Exception('error') + def scan = new WaveScanRecord('scan-1', 'build-1', 'ubuntu:latest', Instant.now()) + + when: + service.onJobException(job, scan, error) + then: + 1 * mockPersistenceService.updateScanRecord(_ as WaveScanRecord) >> { WaveScanRecord scanRecord -> assert scanRecord.status=='FAILED' } + + } + + def 'should handle job timeout event and update scan record'() { + given: + def mockPersistenceService = Mock(PersistenceService) + def jobService = Mock(JobService) + def service = new ContainerScanServiceImpl(persistenceService: mockPersistenceService, jobService: jobService) + def job = JobSpec.scan('scan-1', 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) + def scan = new WaveScanRecord('scan-1', 'build-1', 'ubuntu:latest', Instant.now()) + + when: + service.onJobTimeout(job, scan) + + then: + 1 * mockPersistenceService.updateScanRecord(_ as WaveScanRecord) >> { WaveScanRecord scanRecord -> assert scanRecord.status=='FAILED' } + + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy index 8def3fee6..af9bf3c0a 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy @@ -42,19 +42,21 @@ class DockerScanStrategyTest extends Specification { when: def scanDir = Path.of('/some/scan/dir') def config = Path.of("/user/test/build-workspace/config.json") - def command = dockerContainerStrategy.dockerWrapper(scanDir, config) + def command = dockerContainerStrategy.dockerWrapper('foo-123', scanDir, config) then: command == [ 'docker', 'run', - '--rm', + '--detach', + '--name', + 'foo-123', '-w', '/some/scan/dir', '-v', '/some/scan/dir:/some/scan/dir:rw', '-v', - "$workspace/.trivy-cache:/root/.cache/:rw", + "/build/scan/cache:/root/.cache/:rw", '-v', '/user/test/build-workspace/config.json:/root/.docker/config.json:ro' ] diff --git a/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy index 1e01e876e..ef64a6505 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy @@ -66,21 +66,18 @@ class KubeScanStrategyTest extends Specification { def request = new ScanRequest('100', 'abc', null, 'ubuntu', ContainerPlatform.of('amd64'), folder.resolve('foo')) Files.createDirectories(request.workDir) - def scanResult = strategy.scanContainer(request) + strategy.scanContainer('job-name', request) then: - scanResult - and: - 1 * k8sService.scanContainer(_, _, _, _, _, _, [service:'wave-scan']) >> null + 1 * k8sService.launchScanJob(_, _, _, _, _, _, [service:'wave-scan']) >> null when: def request2 = new ScanRequest('100', 'abc', null, 'ubuntu', ContainerPlatform.of('arm64'), folder.resolve('bar')) Files.createDirectories(request.workDir) - def scanResult2 = strategy.scanContainer(request2) + strategy.scanContainer('job-name', request2) + then: - scanResult2 - and: - 1 * k8sService.scanContainer(_, _, _, _, _, _, [service:'wave-scan-arm64']) >> null + 1 * k8sService.launchScanJob(_, _, _, _, _, _, [service:'wave-scan-arm64']) >> null cleanup: folder?.deleteDir() diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanResultTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanResultTest.groovy new file mode 100644 index 000000000..081278143 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanResultTest.groovy @@ -0,0 +1,201 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import spock.lang.Specification +import spock.lang.Unroll + +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.persistence.WaveScanRecord + +/** + * + * @author Paolo Di Tommaso + */ +class ScanResultTest extends Specification { + + boolean nearly(Duration given, Duration expected) { + given >= expected + given < expected.plusSeconds(5) + } + + def 'should create a result' () { + given: + def elapsed = Duration.ofMinutes(1) + def ts = Instant.now().minus(elapsed) + def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') + when: + def result = new ScanResult( + '123', + 'build-123', + 'docker.io/foo/bar:latest', + ts, + elapsed, + 'DONE', + [ CVE1 ] + ) + then: + result.id == '123' + result.buildId == 'build-123' + result.containerImage == 'docker.io/foo/bar:latest' + result.startTime == ts + result.duration == Duration.ofMinutes(1) + result.status == 'DONE' + result.vulnerabilities == [ CVE1 ] + } + + @Unroll + def 'should validate completed' () { + when: + def result = new ScanResult( + '123', + 'build-123', + 'docker.io/foo/bar:latest', + Instant.now(), + DURATION, + 'DONE', + [] + ) + then: + result.isCompleted() == EXPECTED + + where: + DURATION | EXPECTED + null | false + Duration.ofMinutes(1) | true + } + + @Unroll + def 'should validate completed' () { + when: + def result = new ScanResult( + '123', + 'build-123', + 'docker.io/foo/bar:latest', + Instant.now(), + null, + STATUS, + [] + ) + then: + result.isSucceeded() == EXPECTED + + where: + STATUS | EXPECTED + 'SOMETHING' | false + 'SUCCEEDED' | true + } + + def 'should create result object' () { + given: + def cve1 = new ScanVulnerability('cve-1', 'HIGH', 'test vul', 'testpkg', '1.0.0', '1.1.0', 'http://vul/cve-1') + def elapsed = Duration.ofMinutes(1) + def ts = Instant.now().minus(elapsed) + when: + def result = ScanResult.create('scan-123', 'build-123', 'ubuntu:latest', ts, Duration.ofMinutes(1), 'XYZ', [cve1]) + then: + result.id == 'scan-123' + result.buildId == 'build-123' + result.containerImage == 'ubuntu:latest' + result.startTime == ts + result.duration == elapsed + result.status == 'XYZ' + result.vulnerabilities == [cve1] + } + + def 'should create succeed result' () { + given: + def cve1 = new ScanVulnerability('cve-1', 'HIGH', 'test vul', 'testpkg', '1.0.0', '1.1.0', 'http://vul/cve-1') + def elapsed = Duration.ofMinutes(1) + def ts = Instant.now().minus(elapsed) + and: + def record = new WaveScanRecord( + id: '12345', + buildId: 'build-12345', + containerImage: 'docker.io/some:image', + startTime: ts, + duration: elapsed, + status: ScanResult.SUCCEEDED, + vulnerabilities: [cve1] ) + when: + def result = ScanResult.success(record, record.vulnerabilities) + then: + result.id == '12345' + result.buildId == 'build-12345' + result.containerImage == 'docker.io/some:image' + result.startTime == ts + nearly(result.duration, elapsed) + result.status == ScanResult.SUCCEEDED + result.vulnerabilities == [cve1] + } + + def 'should create failed result from record' () { + given: + def elapsed = Duration.ofMinutes(1) + def ts = Instant.now().minus(elapsed) + and: + def record = new WaveScanRecord( + id: '12345', + buildId: 'build-12345', + containerImage: 'docker.io/some:image', + startTime: ts, + duration: elapsed, + status: ScanResult.FAILED, + vulnerabilities: [] ) + when: + def result = ScanResult.failure(record) + then: + result.id == '12345' + result.buildId == 'build-12345' + result.containerImage == 'docker.io/some:image' + result.startTime == ts + nearly(result.duration, elapsed) + result.status == ScanResult.FAILED + result.vulnerabilities == [] + } + + def 'should create failed result from request' () { + given: + def elapsed = Duration.ofMinutes(1) + def ts = Instant.now().minus(elapsed) + and: + def request = new ScanRequest( + 'scan-123', + 'build-345', + 'config', + 'docker.io/foo/bar', + ContainerPlatform.DEFAULT, + Path.of('/some/path'), + ts ) + when: + def result = ScanResult.failure(request) + then: + result.id == 'scan-123' + result.buildId == 'build-345' + result.containerImage == 'docker.io/foo/bar' + result.startTime == ts + nearly(result.duration, elapsed) + result.status == ScanResult.FAILED + result.vulnerabilities == [] + } +} diff --git a/src/test/groovy/io/seqera/wave/test/DockerRegistryContainer.groovy b/src/test/groovy/io/seqera/wave/test/DockerRegistryContainer.groovy index c1f0e934d..a799dcafb 100644 --- a/src/test/groovy/io/seqera/wave/test/DockerRegistryContainer.groovy +++ b/src/test/groovy/io/seqera/wave/test/DockerRegistryContainer.groovy @@ -20,7 +20,6 @@ package io.seqera.wave.test import spock.lang.Shared -import io.micronaut.context.ApplicationContext import org.testcontainers.containers.GenericContainer import org.testcontainers.containers.wait.strategy.Wait import org.testcontainers.utility.DockerImageName @@ -44,7 +43,7 @@ trait DockerRegistryContainer extends BaseTestContainerRegistry { @Override GenericContainer getTestcontainers() { testcontainers0 } - void initRegistryContainer(ApplicationContext applicationContext){ + void initRegistryContainer(){ testcontainers0.start() assert testcontainers0.execInContainer("apk","add", "docker","bash").exitCode==0 assert testcontainers0.execInContainer("sh","-c","dockerd &").exitCode==0 diff --git a/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy b/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy index f38dc516e..33494be32 100644 --- a/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy +++ b/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy @@ -34,7 +34,6 @@ trait RedisTestContainer { static GenericContainer redisContainer - String getRedisHostName(){ redisContainer.getHost() } @@ -53,6 +52,7 @@ trait RedisTestContainer { } def cleanupSpec(){ + log.debug "Stopping Redis test container" redisContainer.stop() } } diff --git a/src/test/groovy/io/seqera/wave/test/TestHelper.groovy b/src/test/groovy/io/seqera/wave/test/TestHelper.groovy index f1eb2b547..76c491fd3 100644 --- a/src/test/groovy/io/seqera/wave/test/TestHelper.groovy +++ b/src/test/groovy/io/seqera/wave/test/TestHelper.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.test + +import io.seqera.wave.core.ContainerPlatform /** * * @author Paolo Di Tommaso @@ -41,4 +43,11 @@ class TestHelper { return sb.toString(); } + static ContainerPlatform containerPlatform() { + final arm = System.getProperty("os.arch") in ['arm', 'aarch64'] + return arm + ? ContainerPlatform.of('linux/arm64') + : ContainerPlatform.of('linux/amd64') + } + } diff --git a/src/test/groovy/io/seqera/wave/tower/auth/JwtConfigTest.groovy b/src/test/groovy/io/seqera/wave/tower/auth/JwtConfigTest.groovy new file mode 100644 index 000000000..d74646444 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/tower/auth/JwtConfigTest.groovy @@ -0,0 +1,41 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.tower.auth + +import spock.lang.Specification + +import java.time.Duration +/** + * + * @author Paolo Di Tommaso + */ +class JwtConfigTest extends Specification { + + def 'should get random delay' () { + when: + def d = Duration.ofSeconds(10) + def config = new JwtConfig(monitorDelay: d) + then: + config.monitorDelay == d + and: + config.monitorDelayRandomized >= d.dividedBy(2) + config.monitorDelayRandomized < (d + d.dividedBy(2)) + } + +} diff --git a/src/test/groovy/io/seqera/wave/tower/auth/JwtTimeRedisTest.groovy b/src/test/groovy/io/seqera/wave/tower/auth/JwtTimeRedisTest.groovy index 4a8a03536..26cfeb92d 100644 --- a/src/test/groovy/io/seqera/wave/tower/auth/JwtTimeRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/tower/auth/JwtTimeRedisTest.groovy @@ -35,6 +35,7 @@ class JwtTimeRedisTest extends Specification implements RedisTestContainer{ @Shared ApplicationContext applicationContext + @Shared JwtTimeStore timer def setup() { @@ -46,6 +47,11 @@ class JwtTimeRedisTest extends Specification implements RedisTestContainer{ timer = applicationContext.getBean(JwtTimeStore) } + def cleanup() { + applicationContext.close() + } + + def 'should add and get token timers' () { given: def now = Instant.now().epochSecond diff --git a/src/test/groovy/io/seqera/wave/util/DurationUtilsTest.groovy b/src/test/groovy/io/seqera/wave/util/DurationUtilsTest.groovy new file mode 100644 index 000000000..596252923 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/util/DurationUtilsTest.groovy @@ -0,0 +1,139 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.util + +import spock.lang.Specification +import spock.lang.Unroll + +import java.time.Duration + +class DurationUtilsTest extends Specification { + + @Unroll + def "randomDuration returns a duration between #min and #max"() { + given: + def minDuration = Duration.ofSeconds(min) + def maxDuration = Duration.ofSeconds(max) + + when: + def result = DurationUtils.randomDuration(minDuration, maxDuration) + + then: + result >= minDuration + result <= maxDuration + + where: + min | max + 1 | 10 + 0 | 100 + 60 | 3600 + 3600| 7200 + } + + def "randomDuration returns min or max when they are equal"() { + given: + def duration = Duration.ofSeconds(10) + + when: + def result = DurationUtils.randomDuration(duration, duration) + + then: + result == duration + } + + + def "randomDuration generates different values over multiple calls"() { + given: + def minDuration = Duration.ofSeconds(1) + def maxDuration = Duration.ofSeconds(1000) + def iterations = 100 + def results = [] + + when: + iterations.times { + results << DurationUtils.randomDuration(minDuration, maxDuration) + } + + then: + results.unique().size() > 1 + } + + @Unroll + def "randomDuration returns a duration within #intervalPercentage of #reference"() { + given: + def referenceDuration = Duration.ofSeconds(reference) + + when: + def result = DurationUtils.randomDuration(referenceDuration, intervalPercentage) + + then: + def minAllowed = referenceDuration.multipliedBy((long)((1 - intervalPercentage) * 100)).dividedBy(100) + def maxAllowed = referenceDuration.multipliedBy((long)((1 + intervalPercentage) * 100)).dividedBy(100) + result >= minAllowed + result <= maxAllowed + + where: + reference | intervalPercentage + 100 | 0.2f + 60 | 0.5f + 3600 | 0.1f + 10 | 0.0f + } + + def "randomDuration throws IllegalArgumentException for invalid interval percentage"() { + given: + def referenceDuration = Duration.ofSeconds(100) + + when: + DurationUtils.randomDuration(referenceDuration, invalidPercentage) + + then: + thrown(IllegalArgumentException) + + where: + invalidPercentage << [-0.1f, 1.1f] + } + + def "randomDuration handles zero reference duration"() { + given: + def referenceDuration = Duration.ZERO + + when: + def result = DurationUtils.randomDuration(referenceDuration, 0.2f) + + then: + result == Duration.ZERO + } + + def "randomDuration generates different values over multiple calls"() { + given: + def referenceDuration = Duration.ofSeconds(100) + def intervalPercentage = 0.2f + def iterations = 100 + def results = [] + + when: + iterations.times { + results << DurationUtils.randomDuration(referenceDuration, intervalPercentage) + } + + then: + results.unique().size() > 1 + } +} diff --git a/src/test/groovy/io/seqera/wave/util/RetryableTest.groovy b/src/test/groovy/io/seqera/wave/util/RetryableTest.groovy index 419a26c53..70c2fd6d7 100644 --- a/src/test/groovy/io/seqera/wave/util/RetryableTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/RetryableTest.groovy @@ -69,4 +69,24 @@ class RetryableTest extends Specification { e.cause instanceof IOException } + def 'should validate config' () { + given: + def config = Mock(Retryable.Config){ + getDelay() >> Duration.ofSeconds(1) + getMaxDelay() >> Duration.ofSeconds(10) + getMaxAttempts() >> 10 + getJitter() >> 0.25 + getMultiplier() >> 1.5 + } + + when: + def retry = Retryable.of(config).retryPolicy() + then: + retry.getConfig().getDelay() == Duration.ofSeconds(1) + retry.getConfig().getMaxDelay() == Duration.ofSeconds(10) + retry.getConfig().getMaxAttempts() == 10 + retry.getConfig().getJitterFactor() == 0.25d + retry.getConfig().getDelayFactor() == 1.5d + } + } diff --git a/typespec/models/WaveScanRecord.tsp b/typespec/models/WaveScanRecord.tsp index e74e56cf9..f5f4d9158 100644 --- a/typespec/models/WaveScanRecord.tsp +++ b/typespec/models/WaveScanRecord.tsp @@ -4,8 +4,9 @@ import "./Vulnerability.tsp"; model WaveScanRecord { buildId: string; duration: int64; + containerImage: string; id: string; startTime: string; status: string; vulnerabilities: Vulnerability[]; - } \ No newline at end of file + }