diff --git a/README.md b/README.md index cbbe3e5f6..10d853924 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,9 @@ images. * Augment container images i.e. dynamically add one or more container layers to existing images; * Build container images on-demand for a given container file (aka Dockerfile); * Build container images on-demand based on one or more Conda packages; -* Build container images on-demand based on one or more Spack packages, Spack support will be removed in future releases; * Build container images for a specified target platform (currently linux/amd64 and linux/arm64); * Push and cache built containers to a user-provided container repository; -* Build Singularity native containers both using a Singularity spec file, Conda package(s) and Spack package(s); +* Build Singularity native containers both using a Singularity spec file, Conda package(s); * Push Singularity native container images to OCI-compliant registries; diff --git a/VERSION b/VERSION index 6b89d58f8..81f363239 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.2 +1.12.3 diff --git a/build.gradle b/build.gradle index d1e3293bf..951f39628 100644 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,7 @@ dependencies { compileOnly("io.micronaut:micronaut-http-validation") implementation("jakarta.persistence:jakarta.persistence-api:3.0.0") api 'io.seqera:lib-mail:1.0.0' - api 'io.seqera:wave-api:0.10.0' + api 'io.seqera:wave-api:0.12.0' api 'io.seqera:wave-utils:0.13.1' implementation("io.micronaut:micronaut-http-client") diff --git a/changelog.txt b/changelog.txt index 5c865f7eb..5eaee4da4 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,8 @@ # Wave changelog +1.12.3 - 22 Sep 2024 +- Fix build status completion of submit exception [3c3af360] +- Fix singularity build mounts [3b338b29] + 1.12.2 - 18 Sep 2024 - Fix Remove entries permanently from stream once consumed [adfad9d6] - Refactor container build service [1a858c12] @@ -13,7 +17,7 @@ - Do not retry on build failure (#632) [e6568d1e] - Fix Blob cache failure duration (#643) [ebf65adc] - Fix K8s job status detection (#630) [d5b45d8d] [7a9046ed] [e26811dd] - - Fix Retry policy delay multipler (#629) [80037565] + - Fix Retry policy delay multiplier (#629) [80037565] - Improve blob cache info (#644) [8b96173a] - Improve blob cache logging [e4c75671] - Improve blob cache reliability (#596) [dfb64bad] @@ -33,7 +37,7 @@ - Add /v1alpha2/container/{token} in typespec (#618) [5cbd67a8] - Fix failing type checks [bd704bea] - Fix too many requests error code (#610) [ec43fa0d] -- Increase blob cache timeout to 10m and decrese status to 1h [cf4b7588] +- Increase blob cache timeout to 10m and decrease status to 1h [cf4b7588] - Improve container view page (#615) [d9b8cab8] - Improve registry auth error handling (#628) [c9185730] - Increase cache-tower-client to 1min (#641) [df32b305] diff --git a/configuration.md b/configuration.md index b7523b17c..d162a98b5 100644 --- a/configuration.md +++ b/configuration.md @@ -107,20 +107,6 @@ Below are the standard format for known registries, but you can change registry - **`wave.build.force-compression`**: determines whether to force the compression for each cache layers produced by the build process. The default is `false`, enabling compression for more efficient storage. *Optional*. -### Spack configuration for wave build process - -**Note**: Spack support will be removed in future releases. - -Spack configuration consists of the path of its secret file, the mount path for the secret file in the spack container, and the optional S3 bucket name for the spack binary cache. - -**Note**: these configuration are mandatory to support Spack in a wave installation. - -- **`wave.build.spack.secretKeyFile`**: the path to the file containing the PGP private key used to [sign Spack packages built by Wave](https://spack.readthedocs.io/en/latest/binary_caches.html#build-cache-signing). For example, `/efs/wave/spack/key`. *Mandatory*. - -- **`wave.build.spack.secretMountPath`**: sets the mount path inside the Spack Docker image for the PGP private key specified by `wave.build.spack.secretKeyFile`. For instance `/var/seqera/spack/key`. Indicating where the PGP private key should be mounted inside the Spack Docker image. *Mandatory*. - -- **`wave.build.spack.cacheBucket`**: specifies the S3 bucket for the Spack binary cache, for example, `s3://spack-binarycache`. *Optional*. - ### Build process logs configuration This configuration specifies attributes for the persistence of the logs fetched from containers or k8s pods used for building requested images, which can be accessed later and also attached to the build completion email. diff --git a/docs/api.mdx b/docs/api.mdx index 0d646c0b3..ca52843f3 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -52,7 +52,6 @@ This API endpoint is deprecated in current versions of Wave. ] }, condaFile: string, - spackFile: string, containerPlatform: string, buildRepository: string, cacheRepository: string, @@ -81,7 +80,6 @@ This API endpoint is deprecated in current versions of Wave. | `containerConfig.layers.gzipSize` | The size in bytes of the the provided layer tar gzip file. | | `containerFile` | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted. | | `condaFile` | Conda environment file encoded as base64 string. | -| `spackFile` | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases. | | `containerPlatform` | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64. | | `buildRepository` | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional). | | `cacheRepository` | Container repository used to cache build layers `docker.io/user/my-cache` (optional). | @@ -136,7 +134,6 @@ The endpoint returns the name of the container request made available by Wave. ] }, condaFile: string, - spackFile: string, containerPlatform: string, buildRepository: string, cacheRepository: string, @@ -157,10 +154,6 @@ The endpoint returns the name of the container request made available by Wave. commands: string[], basePackages: string } - spackOpts:{ - commands: string[], - basePackages: string - } }, nameStrategy: string @@ -182,7 +175,6 @@ The endpoint returns the name of the container request made available by Wave. | `containerConfig.layers.gzipSize` | The size in bytes of the the provided layer tar gzip file. | | `containerFile` | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted. | | `condaFile` | Conda environment file encoded as base64 string. | -| `spackFile` | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases. | | `containerPlatform` | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64. | | `buildRepository` | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional). | | `cacheRepository` | Container repository used to cache build layers `docker.io/user/my-cache` (optional). | diff --git a/src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy deleted file mode 100644 index 0b81d6299..000000000 --- a/src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.configuration - -import java.nio.file.Path - -import groovy.transform.CompileStatic -import groovy.transform.EqualsAndHashCode -import groovy.transform.ToString -import io.micronaut.context.annotation.Value -import io.micronaut.core.annotation.Nullable -import jakarta.inject.Singleton -/** - * Model Spack configuration - * - * @author Paolo Di Tommaso - */ -@ToString -@EqualsAndHashCode -@Singleton -@CompileStatic -@Deprecated -class SpackConfig { - - /** - * The s3 bucket where Spack cached binaries are stored - */ - @Nullable - @Value('${wave.build.spack.cacheBucket}') - private String cacheBucket - - /** - * The host path where the GPG key required by the Spack "buildcache" is located - */ - @Nullable - @Value('${wave.build.spack.secretKeyFile}') - private String secretKeyFile - - /** - * The container path where the GPG key required by the Spack "buildcache" is located - */ - @Nullable - @Value('${wave.build.spack.secretMountPath}') - private String secretMountPath - - /** - * The container image used for Spack builds - */ - @Value('${wave.build.spack.builderImage:`spack/ubuntu-jammy:v0.20.0`}') - private String builderImage - - /** - * The container image used for Spack container - */ - @Value('${wave.build.spack.runnerImage:`ubuntu:22.04`}') - private String runnerImage - - String getCacheBucket() { - if( !cacheBucket ) - throw new IllegalStateException("Missing Spack 'cacheBucket' configuration setting") - return cacheBucket - } - - Path getSecretKeyFile() { - if( !secretKeyFile ) - throw new IllegalStateException("Missing Spack 'secretKeyFile' configuration setting") - return Path.of(secretKeyFile).toAbsolutePath().normalize() - } - - String getSecretMountPath() { - if( !secretMountPath ) - throw new IllegalStateException("Missing Spack 'secretMountPath' configuration setting") - return secretMountPath - } - - String getBuilderImage() { - return builderImage - } - - String getRunnerImage() { - return runnerImage - } -} diff --git a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy index 0036232ce..f7e016d6d 100644 --- a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy @@ -31,9 +31,12 @@ import io.micronaut.http.server.types.files.StreamedFile import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.conda.CondaLockService import io.seqera.wave.service.logs.BuildLogService +import io.seqera.wave.service.mirror.ContainerMirrorService +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.persistence.WaveBuildRecord import jakarta.inject.Inject /** @@ -50,6 +53,9 @@ class BuildController { @Inject private ContainerBuildService buildService + @Inject + private ContainerMirrorService mirrorService + @Inject @Nullable BuildLogService logService @@ -58,7 +64,7 @@ class BuildController { CondaLockService condaLockService @Get("/v1alpha1/builds/{buildId}") - HttpResponse getBuildRecord(String buildId){ + HttpResponse getBuildRecord(String buildId) { final record = buildService.getBuildRecord(buildId) return record ? HttpResponse.ok(record) @@ -77,10 +83,10 @@ class BuildController { } @Get("/v1alpha1/builds/{buildId}/status") - HttpResponse getBuildStatus(String buildId){ - final build = buildService.getBuildRecord(buildId) - build != null - ? HttpResponse.ok(build.toStatusResponse()) + HttpResponse getBuildStatus(String buildId) { + final resp = buildResponse0(buildId) + resp != null + ? HttpResponse.ok(resp) : HttpResponse.notFound() } @@ -96,4 +102,20 @@ class BuildController { : HttpResponse.notFound() } + protected BuildStatusResponse buildResponse0(String buildId) { + if( !buildId ) + throw new BadRequestException("Missing 'buildId' parameter") + // build IDs starting with the `mr-` prefix are interpreted as mirror requests + if( buildId.startsWith(MirrorRequest.ID_PREFIX) ) { + return mirrorService + .getMirrorState(buildId) + ?.toStatusResponse() + } + else { + return buildService + .getBuildRecord(buildId) + ?.toStatusResponse() + } + } + } diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 2499bf925..5478d44dd 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -59,6 +59,8 @@ import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.builder.FreezeService import io.seqera.wave.service.inclusion.ContainerInclusionService import io.seqera.wave.service.inspect.ContainerInspectService +import io.seqera.wave.service.mirror.ContainerMirrorService +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.service.pairing.socket.PairingChannel import io.seqera.wave.service.persistence.PersistenceService @@ -86,8 +88,6 @@ import static io.seqera.wave.util.ContainerHelper.makeResponseV1 import static io.seqera.wave.util.ContainerHelper.makeResponseV2 import static io.seqera.wave.util.ContainerHelper.makeTargetImage import static io.seqera.wave.util.ContainerHelper.patchPlatformEndpoint -import static io.seqera.wave.util.ContainerHelper.spackFileFromRequest -import static io.seqera.wave.util.SpackHelper.prependBuilderTemplate import static java.util.concurrent.CompletableFuture.completedFuture /** * Implement a controller to receive container token requests @@ -127,7 +127,7 @@ class ContainerController { ContainerBuildService buildService @Inject - ContainerInspectService dockerAuthService + ContainerInspectService inspectService @Inject RegistryProxyService registryProxyService @@ -154,6 +154,9 @@ class ContainerController { @Nullable RateLimiterService rateLimiterService + @Inject + private ContainerMirrorService mirrorService + @PostConstruct private void init() { log.info "Wave server url: $serverUrl; allowAnonymous: $allowAnonymous; tower-endpoint-url: $towerEndpointUrl; default-build-repo: $buildConfig.defaultBuildRepository; default-cache-repo: $buildConfig.defaultCacheRepository; default-public-repo: $buildConfig.defaultPublicRepository" @@ -178,6 +181,7 @@ class ContainerController { // validate request validateContainerRequest(req) + validateMirrorRequest(req, v2) // this is needed for backward compatibility with old clients if( !req.towerEndpoint ) { @@ -237,6 +241,10 @@ class ContainerController { req = req.copyWith(containerFile: generated.bytes.encodeBase64().toString()) } + if( req.spackFile ) { + throw new BadRequestException("Spack packages are not supported any more") + } + final ip = addressResolver.resolve(httpRequest) // check the rate limit before continuing if( rateLimiterService ) @@ -317,18 +325,16 @@ class ContainerController { final containerSpec = decodeBase64OrFail(req.containerFile, 'containerFile') final condaContent = condaFileFromRequest(req) - final spackContent = spackFileFromRequest(req) final format = req.formatSingularity() ? SINGULARITY : DOCKER final platform = ContainerPlatform.of(req.containerPlatform) final buildRepository = targetRepo( req.buildRepository ?: (req.freeze && buildConfig.defaultPublicRepository ? buildConfig.defaultPublicRepository : buildConfig.defaultBuildRepository), req.nameStrategy) final cacheRepository = req.cacheRepository ?: buildConfig.defaultCacheRepository - final configJson = dockerAuthService.credentialsConfigJson(containerSpec, buildRepository, cacheRepository, identity) + final configJson = inspectService.credentialsConfigJson(containerSpec, buildRepository, cacheRepository, identity) final containerConfig = req.freeze ? req.containerConfig : null final offset = DataTimeUtils.offsetId(req.timestamp) final scanId = scanEnabled && format==DOCKER ? LongRndKey.rndHex() : null - final containerFile = spackContent ? prependBuilderTemplate(containerSpec,format) : containerSpec // use 'imageSuffix' strategy by default for public repo images final nameStrategy = req.nameStrategy==null && buildRepository @@ -338,14 +344,13 @@ class ContainerController { checkContainerSpec(containerSpec) // create a unique digest to identify the build request - final containerId = makeContainerId(containerFile, condaContent, spackContent, platform, buildRepository, req.buildContext) - final targetImage = makeTargetImage(format, buildRepository, containerId, condaContent, spackContent, nameStrategy) + final containerId = makeContainerId(containerSpec, condaContent, platform, buildRepository, req.buildContext) + final targetImage = makeTargetImage(format, buildRepository, containerId, condaContent, nameStrategy) final maxDuration = buildConfig.buildMaxDuration(req) return new BuildRequest( containerId, - containerFile, + containerSpec, condaContent, - spackContent, Path.of(buildConfig.buildWorkspace), targetImage, identity, @@ -415,6 +420,15 @@ class ContainerController { buildId = track.id buildNew = !track.cached } + else if( req.mirrorRegistry ) { + final mirror = makeMirrorRequest(req, identity) + final track = checkMirror(mirror, identity, req.dryRun) + targetImage = track.targetImage + targetContent = null + condaContent = null + buildId = track.id + buildNew = !track.cached + } else if( req.containerImage ) { // normalize container image final coords = ContainerCoordinates.parse(req.containerImage) @@ -436,7 +450,51 @@ class ContainerController { ContainerPlatform.of(req.containerPlatform), buildId, buildNew, - req.freeze ) + req.freeze, + req.mirrorRegistry!=null + ) + } + + protected MirrorRequest makeMirrorRequest(SubmitContainerTokenRequest request, PlatformId identity) { + final coords = ContainerCoordinates.parse(request.containerImage) + if( coords.registry == request.mirrorRegistry ) + throw new BadRequestException("Source and target mirror registry as the same - offending value '${request.mirrorRegistry}'") + final targetImage = request.mirrorRegistry + '/' + coords.imageAndTag + final configJson = inspectService.credentialsConfigJson(null, request.containerImage, targetImage, identity) + final platform = request.containerPlatform + ? ContainerPlatform.of(request.containerPlatform) + : ContainerPlatform.DEFAULT + final digest = registryProxyService.getImageDigest(request.containerImage, identity) + if( !digest ) + throw new BadRequestException("Container image '$request.containerImage' does not exist") + return MirrorRequest.create( + request.containerImage, + targetImage, + digest, + platform, + Path.of(buildConfig.buildWorkspace).toAbsolutePath(), + configJson ) + } + + protected BuildTrack checkMirror(MirrorRequest request, PlatformId identity, boolean dryRun) { + final targetDigest = registryProxyService.getImageDigest(request.targetImage, identity) + log.debug "== Mirror target digest: $targetDigest" + final cached = request.digest==targetDigest + // check for dry-run execution + if( dryRun ) { + log.debug "== Dry-run request request: $request" + final dryId = request.id + BuildRequest.SEP + '0' + return new BuildTrack(dryId, request.targetImage, cached) + } + // check for existing image + if( request.digest==targetDigest ) { + log.debug "== Found cached request for request: $request" + final cache = persistenceService.loadMirrorState(request.targetImage, targetDigest) + return new BuildTrack(cache?.mirrorId, request.targetImage, true) + } + else { + return mirrorService.mirrorImage(request) + } } protected String targetImage(String token, ContainerCoordinates container) { @@ -462,7 +520,7 @@ class ContainerController { return HttpResponse.ok() } - void validateContainerRequest(SubmitContainerTokenRequest req) throws BadRequestException{ + void validateContainerRequest(SubmitContainerTokenRequest req) throws BadRequestException { String msg // check valid image name msg = validationService.checkContainerName(req.containerImage) @@ -475,6 +533,32 @@ class ContainerController { if( msg ) throw new BadRequestException(msg) } + void validateMirrorRequest(SubmitContainerTokenRequest req, boolean v2) throws BadRequestException { + if( !req.mirrorRegistry ) + return + // container mirror validation + if( !v2 ) + throw new BadRequestException("Container mirroring requires the use of v2 API") + if( !req.containerImage ) + throw new BadRequestException("Attribute `containerImage` is required when specifying `mirrorRegistry`") + if( !req.towerAccessToken ) + throw new BadRequestException("Container mirroring requires an authenticated request - specify the tower token attribute") + if( req.freeze ) + throw new BadRequestException("Attribute `mirrorRegistry` and `freeze` conflict each other") + if( req.containerFile ) + throw new BadRequestException("Attribute `mirrorRegistry` and `containerFile` conflict each other") + if( req.containerIncludes ) + throw new BadRequestException("Attribute `mirrorRegistry` and `containerIncludes` conflict each other") + if( req.containerConfig ) + throw new BadRequestException("Attribute `mirrorRegistry` and `containerConfig` conflict each other") + final coords = ContainerCoordinates.parse(req.containerImage) + if( coords.registry == req.mirrorRegistry ) + throw new BadRequestException("Source and target mirror registry as the same - offending value '${req.mirrorRegistry}'") + def msg = validationService.checkMirrorRegistry(req.mirrorRegistry) + if( msg ) + throw new BadRequestException(msg) + } + @Error(exception = AuthorizationException.class) HttpResponse handleAuthorizationException() { return HttpResponse.unauthorized() diff --git a/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy new file mode 100644 index 000000000..bb1626505 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy @@ -0,0 +1,53 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.controller + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.http.HttpResponse +import io.micronaut.http.annotation.Controller +import io.micronaut.http.annotation.Get +import io.micronaut.scheduling.TaskExecutors +import io.micronaut.scheduling.annotation.ExecuteOn +import io.seqera.wave.service.mirror.ContainerMirrorService +import io.seqera.wave.service.mirror.MirrorState +import jakarta.inject.Inject +/** + * Implements a controller for container mirror apis + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +@Controller("/") +@ExecuteOn(TaskExecutors.IO) +class MirrorController { + + @Inject + private ContainerMirrorService mirrorService + + @Get("/v1alpha1/mirrors/{mirrorId}") + HttpResponse getMirrorRecord(String mirrorId) { + final result = mirrorService.getMirrorState(mirrorId) + return result + ? HttpResponse.ok(result) + : HttpResponse.notFound() + } + +} diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 02735f372..4df9ba1be 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -18,11 +18,12 @@ package io.seqera.wave.controller -import groovy.json.JsonOutput -import io.micronaut.core.annotation.Nullable +import java.util.regex.Pattern import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpResponse import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Get @@ -47,6 +48,7 @@ import static io.seqera.wave.util.DataTimeUtils.formatTimestamp * * @author Paolo Di Tommaso */ +@Slf4j @CompileStatic @Controller("/view") @ExecuteOn(TaskExecutors.IO) @@ -74,13 +76,55 @@ class ViewController { @View("build-view") @Get('/builds/{buildId}') - HttpResponse> viewBuild(String buildId) { + HttpResponse viewBuild(String buildId) { + // check redirection for invalid suffix in the form `-nn` + final r1 = shouldRedirect1(buildId) + if( r1 ) { + log.debug "Redirect to build page [1]: $r1" + return HttpResponse.redirect(URI.create(r1)) + } + // check redirection when missing the suffix `_nn` + final r2 = shouldRedirect2(buildId) + if( r2 ) { + log.debug "Redirect to build page [2]: $r2" + return HttpResponse.redirect(URI.create(r2)) + } + // go ahead with proper handling final record = buildService.getBuildRecord(buildId) if( !record ) throw new NotFoundException("Unknown build id '$buildId'") return HttpResponse.ok(renderBuildView(record)) } + static final private Pattern DASH_SUFFIX = ~/([0-9a-zA-Z\-]+)-(\d+)$/ + + static final private Pattern MISSING_SUFFIX = ~/([0-9a-zA-Z\-]+)(? renderBuildView(WaveBuildRecord result) { // create template binding final binding = new HashMap(20) @@ -97,7 +141,6 @@ class ViewController { binding.build_platform = result.platform binding.build_containerfile = result.dockerFile ?: '-' binding.build_condafile = result.condaFile - binding.build_spackfile = result.spackFile binding.build_digest = result.digest ?: '-' binding.put('server_url', serverUrl) binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null diff --git a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy index ba9bb553f..e8a8fae6a 100644 --- a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy +++ b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy @@ -188,11 +188,15 @@ class RegistryProxyService { } String getImageDigest(BuildRequest request, boolean retryOnNotFound=false) { + return getImageDigest(request.targetImage, request.identity, retryOnNotFound) + } + + String getImageDigest(String containerImage, PlatformId identity, boolean retryOnNotFound=false) { try { - return getImageDigest0(request.targetImage, request.identity, retryOnNotFound) + return getImageDigest0(containerImage, identity, retryOnNotFound) } catch(Exception e) { - log.warn "Unable to retrieve digest for image '${request.targetImage}' -- cause: ${e.message}" + log.warn "Unable to retrieve digest for image '${containerImage}' -- cause: ${e.message}" return null } } diff --git a/src/main/groovy/io/seqera/wave/service/ContainerRequestData.groovy b/src/main/groovy/io/seqera/wave/service/ContainerRequestData.groovy index 959b0f798..2a6020947 100644 --- a/src/main/groovy/io/seqera/wave/service/ContainerRequestData.groovy +++ b/src/main/groovy/io/seqera/wave/service/ContainerRequestData.groovy @@ -43,6 +43,11 @@ class ContainerRequestData { final String buildId final Boolean buildNew final Boolean freeze + final Boolean mirror + + boolean durable() { + return freeze || mirror + } PlatformId getIdentity() { return identity diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy index cdd914598..dbc4085c7 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy @@ -60,7 +60,7 @@ class BuildCacheStore extends AbstractCacheStore implements Bui protected Duration getDuration() { return buildConfig.statusDuration } - + @Override BuildStoreEntry getBuild(String imageName) { return get(imageName) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 545a52ca7..9682fb536 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -41,7 +41,7 @@ import static io.seqera.wave.util.StringUtils.trunc @CompileStatic class BuildRequest { - static final String SEP = '_' + static final public String SEP = '_' /** * Unique request Id. This is computed as a consistent hash generated from @@ -60,11 +60,6 @@ class BuildRequest { */ final String condaFile - /** - * The spock file recipe associated with this request - */ - final String spackFile - /** * The build context work directory */ @@ -116,11 +111,6 @@ class BuildRequest { final ContainerConfig containerConfig /** - * Whenever is a spack build - */ - final boolean isSpackBuild - - /** * The ID of the security scan triggered by this build */ final String scanId @@ -147,7 +137,6 @@ class BuildRequest { BuildRequest(String containerId, String containerFile, String condaFile, - String spackFile, Path workspace, String targetImage, PlatformId identity, @@ -166,7 +155,6 @@ class BuildRequest { this.containerId = containerId this.containerFile = containerFile this.condaFile = condaFile - this.spackFile = spackFile this.workspace = workspace this.targetImage = targetImage this.identity = identity @@ -177,7 +165,6 @@ class BuildRequest { this.configJson = configJson this.offsetId = offsetId ?: OffsetDateTime.now().offset.id this.containerConfig = containerConfig - this.isSpackBuild = spackFile this.scanId = scanId this.buildContext = buildContext this.format = format @@ -188,7 +175,6 @@ class BuildRequest { this.containerId = opts.containerId this.containerFile = opts.containerFile this.condaFile = opts.condaFile - this.spackFile = opts.spackFile this.workspace = opts.workspace as Path this.targetImage = opts.targetImage this.identity = opts.identity as PlatformId @@ -199,7 +185,6 @@ class BuildRequest { this.configJson = opts.configJson this.offsetId = opts.offesetId this.containerConfig = opts.containerConfig as ContainerConfig - this.isSpackBuild = opts.isSpackBuild this.scanId = opts.scanId this.buildContext = opts.buildContext as BuildContext this.format = opts.format as BuildFormat @@ -210,7 +195,7 @@ class BuildRequest { @Override String toString() { - return "BuildRequest[containerId=$containerId; targetImage=$targetImage; identity=$identity; dockerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; spackFile=${trunc(spackFile)}; buildId=$buildId, maxDuration=$maxDuration]" + return "BuildRequest[containerId=$containerId; targetImage=$targetImage; identity=$identity; dockerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; buildId=$buildId, maxDuration=$maxDuration]" } String getContainerId() { @@ -230,10 +215,6 @@ class BuildRequest { return condaFile } - String getSpackFile() { - return spackFile - } - Path getWorkDir() { return workDir } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index fb5a25cb4..c061c30e5 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -84,19 +84,6 @@ abstract class BuildStrategy { result << "type=registry,ref=$req.cacheRepository:$req.containerId".toString() } - if(req.spackFile){ - result << '--opt' - result << 'build-arg:AWS_STS_REGIONAL_ENDPOINTS=$(AWS_STS_REGIONAL_ENDPOINTS)' - result << '--opt' - result << 'build-arg:AWS_REGION=$(AWS_REGION)' - result << '--opt' - result << 'build-arg:AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION)' - result << '--opt' - result << 'build-arg:AWS_ROLE_ARN=$(AWS_ROLE_ARN)' - result << '--opt' - result << 'build-arg:AWS_WEB_IDENTITY_TOKEN_FILE=$(AWS_WEB_IDENTITY_TOKEN_FILE)' - } - return result } diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy index 8d90dd7ee..2e540ff0f 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildService.groovy @@ -84,4 +84,13 @@ interface ContainerBuildService { * @return The {@link WaveBuildRecord} associated with the corresponding Id, or {@code null} if it cannot be found */ WaveBuildRecord getBuildRecord(String buildId) + + /** + * Retrieve the latest build record available for the specified container id. + * + * @param containerId The ID of the container for which the build record needs to be retrieve + * @return The {@link WaveBuildRecord} associated with the corresponding Id, or {@code null} if it cannot be found + */ + WaveBuildRecord getLatestBuild(String containerId) + } diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 5fb518627..f2a26302e 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -36,7 +36,6 @@ import io.seqera.wave.auth.RegistryCredentialsProvider import io.seqera.wave.auth.RegistryLookupService import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.HttpClientConfig -import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.exception.HttpServerRetryableErrorException import io.seqera.wave.ratelimit.AcquireRequest @@ -52,9 +51,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.stream.StreamService import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.Retryable -import io.seqera.wave.util.SpackHelper import io.seqera.wave.util.TarUtils -import io.seqera.wave.util.TemplateRenderer import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton @@ -100,9 +97,6 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler buildCmd(String jobName, BuildRequest req, Path credsFile) { - final spack = req.isSpackBuild ? spackConfig : null final dockerCmd = req.formatDocker() - ? cmdForBuildkit(jobName, req.workDir, credsFile, spack, req.platform) - : cmdForSingularity(jobName, req.workDir, credsFile, spack, req.platform) + ? cmdForBuildkit(jobName, req.workDir, credsFile, req.platform) + : cmdForSingularity(jobName, req.workDir, credsFile, req.platform) return dockerCmd + launchCmd(req) } - protected List cmdForBuildkit(String name, Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { + protected List cmdForBuildkit(String name, Path workDir, Path credsFile, ContainerPlatform platform ) { //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', @@ -125,12 +120,6 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add("$credsFile:/home/user/.docker/config.json:ro".toString()) } - if( spackConfig ) { - // secret file - wrapper.add('-v') - wrapper.add("${spackConfig.secretKeyFile}:${spackConfig.secretMountPath}:ro".toString()) - } - if( platform ) { wrapper.add('--platform') wrapper.add(platform.toString()) @@ -142,7 +131,7 @@ class DockerBuildStrategy extends BuildStrategy { return wrapper } - protected List cmdForSingularity(String name, Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform) { + protected List cmdForSingularity(String name, Path workDir, Path credsFile, ContainerPlatform platform) { final wrapper = ['docker', 'run', '--detach', @@ -159,12 +148,6 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add("${credsFile.resolveSibling('singularity-remote.yaml')}:/root/.singularity/remote.yaml:ro".toString()) } - if( spackConfig ) { - // secret file - wrapper.add('-v') - wrapper.add("${spackConfig.secretKeyFile}:${spackConfig.secretMountPath}:ro".toString()) - } - if( platform ) { wrapper.add('--platform') wrapper.add(platform.toString()) diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index 4d4e9fa06..f6ba6e041 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -30,7 +30,6 @@ import io.micronaut.context.annotation.Property import io.micronaut.context.annotation.Requires import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.BuildConfig -import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService @@ -65,9 +64,6 @@ class KubeBuildStrategy extends BuildStrategy { @Inject private BuildConfig buildConfig - @Inject - private SpackConfig spackConfig - @Inject private RegistryProxyService proxyService @@ -95,8 +91,7 @@ class KubeBuildStrategy extends BuildStrategy { final buildCmd = launchCmd(req) final timeout = req.maxDuration ?: buildConfig.defaultTimeout final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final spackCfg0 = req.isSpackBuild ? spackConfig : null - k8sService.launchBuildJob(jobName, buildImage, buildCmd, req.workDir, configFile, timeout, spackCfg0, selector) + k8sService.launchBuildJob(jobName, buildImage, buildCmd, req.workDir, configFile, timeout, selector) } catch (ApiException e) { throw new BadRequestException("Unexpected build failure - ${e.responseBody}", e) diff --git a/src/main/groovy/io/seqera/wave/service/builder/store/BuildRecordCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/store/BuildRecordCacheStore.groovy index 0f9884cfd..f820f080e 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/store/BuildRecordCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/store/BuildRecordCacheStore.groovy @@ -23,6 +23,7 @@ import java.time.Duration import groovy.transform.CompileStatic import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.service.builder.BuildStoreEntry import io.seqera.wave.service.cache.AbstractCacheStore import io.seqera.wave.service.cache.impl.CacheProvider import io.seqera.wave.service.persistence.WaveBuildRecord diff --git a/src/main/groovy/io/seqera/wave/service/cache/AbstractCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/cache/AbstractCacheStore.groovy index b78e4f535..91e707701 100644 --- a/src/main/groovy/io/seqera/wave/service/cache/AbstractCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/cache/AbstractCacheStore.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.cache import java.time.Duration +import groovy.transform.CompileStatic import io.seqera.wave.encoder.EncodingStrategy import io.seqera.wave.service.cache.impl.CacheProvider @@ -28,7 +29,8 @@ import io.seqera.wave.service.cache.impl.CacheProvider * * @author Paolo Di Tommaso */ -abstract class AbstractCacheStore implements CacheStore, BiCacheStore { +@CompileStatic +abstract class AbstractCacheStore implements CacheStore { private EncodingStrategy encodingStrategy @@ -45,6 +47,10 @@ abstract class AbstractCacheStore implements CacheStore, BiCacheSto protected String key0(String k) { return getPrefix() + k } + protected String recordId0(String recordId) { + return getPrefix() + 'state-id/' + recordId + } + protected V deserialize(String encoded) { return encodingStrategy.decode(encoded) } @@ -63,32 +69,34 @@ abstract class AbstractCacheStore implements CacheStore, BiCacheSto return result ? deserialize(result) : null } + V getByRecordId(String recordId) { + final key = delegate.get(recordId0(recordId)) + return get(key) + } + void put(String key, V value) { - delegate.put(key0(key), serialize(value), getDuration()) + put(key, value, getDuration()) } @Override void put(String key, V value, Duration ttl) { delegate.put(key0(key), serialize(value), ttl) + if( value instanceof StateRecord ) { + delegate.put(recordId0(value.getRecordId()), key, ttl) + } } @Override boolean putIfAbsent(String key, V value, Duration ttl) { - delegate.putIfAbsent(key0(key), serialize(value), ttl) + final result = delegate.putIfAbsent(key0(key), serialize(value), ttl) + if( result && value instanceof StateRecord ) { + delegate.put(recordId0(value.getRecordId()), key, ttl) + } + return result } boolean putIfAbsent(String key, V value) { - delegate.putIfAbsent(key0(key), serialize(value), getDuration()) - } - - @Override - V putIfAbsentAndGetCurrent(String key, V value, Duration ttl) { - final result = delegate.putIfAbsentAndGetCurrent(key0(key), serialize(value), ttl) - return result? deserialize(result) : null - } - - V putIfAbsentAndGetCurrent(String key, V value) { - return putIfAbsentAndGetCurrent(key, value, getDuration()) + return putIfAbsent(key, value, getDuration()) } @Override @@ -101,25 +109,4 @@ abstract class AbstractCacheStore implements CacheStore, BiCacheSto delegate.clear() } - @Override - void biPut(String key, V value, Duration ttl) { - delegate.biPut(key0(key), serialize(value), ttl) - } - - @Override - void biRemove(String key) { - delegate.biRemove(key0(key)) - } - - @Override - Set biKeysFor(V value) { - final keys = delegate.biKeysFor(serialize(value)) - return keys.collect( (it) -> it.replace(getPrefix(),'') ) - } - - @Override - String biKeyFind(V value, boolean sorted) { - final result = delegate.biKeyFind(serialize(value), sorted) - result ? result.replace(getPrefix(),'') : null - } } diff --git a/src/main/groovy/io/seqera/wave/service/cache/BiCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/cache/BiCacheStore.groovy deleted file mode 100644 index 546c2166d..000000000 --- a/src/main/groovy/io/seqera/wave/service/cache/BiCacheStore.groovy +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.cache - -import java.time.Duration - -/** - * A cache store implementing a bi-direction key-value access, - * it allows retrieving all keys for a given value in time O(2) - * - * @author Paolo Di Tommaso - */ -interface BiCacheStore { - - /** - * Add a bi-directional key-value - * - * @param key The entry key - * @param value The entry value - * @param ttl The entry time-to-live. - */ - void biPut(K key, V value, Duration ttl) - - /** - * Remove an entry for the given value. - * - * @param key The key of entry to be removed - */ - void biRemove(K key) - - /** - * Get all key for the given value. - * - * @param value The value for which find corresponding keys - * @return A set of keys associated with the specified value or an empty set otherwise. - */ - Set biKeysFor(V value) - - /** - * Find a key in the cache for the given value. - * - * @param value The value for which find corresponding key - * @param sorted When true, the list of keys is sorted before getting the first value - * @return A key associated with the specified value or null if not key is found - */ - K biKeyFind(V value, boolean sorted) - -} diff --git a/src/main/groovy/io/seqera/wave/service/cache/CacheStore.groovy b/src/main/groovy/io/seqera/wave/service/cache/CacheStore.groovy index 9bd6f33ae..822260639 100644 --- a/src/main/groovy/io/seqera/wave/service/cache/CacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/cache/CacheStore.groovy @@ -37,6 +37,14 @@ interface CacheStore { */ V get(K key) + /** + * Store a the specified key-value pair in the underlying cache + * + * @param key The key to retrieve the associated value + * @param value The value to be store in the cache + */ + void put(K key, V value) + /** * Store a the specified key-value pair in the underlying cache * @@ -50,23 +58,18 @@ interface CacheStore { * Store a value in the cache only if does not exist yet * @param key The unique associated with this object * @param value The object to store - * @param ttl The max time-to-live of the stored entry * @return {@code true} if the value was stored, {@code false} otherwise */ - boolean putIfAbsent(K key, V value, Duration ttl) + boolean putIfAbsent(K key, V value) /** * Store a value in the cache only if does not exist yet - * and returns the value that is in the cache after the - * operation. - * - * @param key The unique key associated with this object + * @param key The unique associated with this object * @param value The object to store * @param ttl The max time-to-live of the stored entry - * @return The current value in the cache, that is the existing one - * if already present or the new value otherwise + * @return {@code true} if the value was stored, {@code false} otherwise */ - V putIfAbsentAndGetCurrent(K key, V value, Duration ttl) + boolean putIfAbsent(K key, V value, Duration ttl) /** * Remove the entry with the specified key from the cache diff --git a/src/main/groovy/io/seqera/wave/service/cache/StateRecord.groovy b/src/main/groovy/io/seqera/wave/service/cache/StateRecord.groovy new file mode 100644 index 000000000..3e3896957 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/cache/StateRecord.groovy @@ -0,0 +1,30 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cache + +/** + * Marker interface for record object that model long running operation state + * + * @author Paolo Di Tommaso + */ +interface StateRecord { + + String getRecordId() + +} diff --git a/src/main/groovy/io/seqera/wave/service/cache/impl/CacheProvider.groovy b/src/main/groovy/io/seqera/wave/service/cache/impl/CacheProvider.groovy index 3b8dcb80a..0e458c75b 100644 --- a/src/main/groovy/io/seqera/wave/service/cache/impl/CacheProvider.groovy +++ b/src/main/groovy/io/seqera/wave/service/cache/impl/CacheProvider.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.cache.impl -import io.seqera.wave.service.cache.BiCacheStore import io.seqera.wave.service.cache.CacheStore /** @@ -26,5 +25,5 @@ import io.seqera.wave.service.cache.CacheStore * * @author Paolo Di Tommaso */ -interface CacheProvider extends CacheStore, BiCacheStore { +interface CacheProvider extends CacheStore { } diff --git a/src/main/groovy/io/seqera/wave/service/cache/impl/LocalCacheProvider.groovy b/src/main/groovy/io/seqera/wave/service/cache/impl/LocalCacheProvider.groovy index 09f0c0cfa..696461895 100644 --- a/src/main/groovy/io/seqera/wave/service/cache/impl/LocalCacheProvider.groovy +++ b/src/main/groovy/io/seqera/wave/service/cache/impl/LocalCacheProvider.groovy @@ -67,19 +67,24 @@ class LocalCacheProvider implements CacheProvider { return entry.value } + @Override + void put(String key, String value) { + store.put(key, new Entry<>(value,null)) + } + + @Override void put(String key, String value, Duration ttl) { store.put(key, new Entry<>(value,ttl)) } @Override - boolean putIfAbsent(String key, String value, Duration ttl) { - return putIfAbsent0(key, value, ttl) == null + boolean putIfAbsent(String key, String value) { + return putIfAbsent0(key, value, null) == null } @Override - String putIfAbsentAndGetCurrent(String key, String value, Duration ttl) { - final ret = putIfAbsent0(key, value, ttl) - return ret!=null ? ret : value + boolean putIfAbsent(String key, String value, Duration ttl) { + return putIfAbsent0(key, value, ttl) == null } private String putIfAbsent0(String key, String value, Duration ttl) { @@ -99,57 +104,4 @@ class LocalCacheProvider implements CacheProvider { store.clear() } - // =============== bi-cache store implementation =============== - - private Map> index = new HashMap<>() - - @Override - void biPut(String key, String value, Duration ttl) { - synchronized (this) { - this.put(key, value, ttl) - final id = value.hashCode() - def set = index.get(id) - if( set==null ) { - set=new HashSet() - index.put(id, set) - } - set.add(key) - } - } - - @Override - void biRemove(String key) { - synchronized (this) { - final entry = store.remove(key) - if( !entry ) - return - final id = entry.value.hashCode() - final set = index.get(id) - if( set ) { - set.remove(key) - } - } - } - - @Override - Set biKeysFor(String value) { - final id = value.hashCode() - return index.get(id) ?: Set.of() - } - - String biKeyFind(String value, boolean sorted) { - final id = value.hashCode() - final list = biKeysFor(value).toList() - final keys = sorted ? list.toSorted() : list.shuffled() - final itr = keys.iterator() - while( itr.hasNext() ) { - final result = itr.next() - // verify the key still exists - if( get(result)!=null ) - return result - // if not exist, remove it from the set - index.get(id)?.remove(result) - } - return null - } } diff --git a/src/main/groovy/io/seqera/wave/service/cache/impl/RedisCacheProvider.groovy b/src/main/groovy/io/seqera/wave/service/cache/impl/RedisCacheProvider.groovy index e597a96f8..95b006152 100644 --- a/src/main/groovy/io/seqera/wave/service/cache/impl/RedisCacheProvider.groovy +++ b/src/main/groovy/io/seqera/wave/service/cache/impl/RedisCacheProvider.groovy @@ -24,7 +24,6 @@ import groovy.transform.CompileStatic import io.micronaut.context.annotation.Requires import jakarta.inject.Inject import jakarta.inject.Singleton -import org.apache.commons.codec.digest.DigestUtils import redis.clients.jedis.Jedis import redis.clients.jedis.JedisPool import redis.clients.jedis.params.SetParams @@ -48,31 +47,34 @@ class RedisCacheProvider implements CacheProvider { } } + @Override + void put(String key, String value) { + put(key, value, null) + } + + @Override void put(String key, String value, Duration ttl) { try( Jedis conn=pool.getResource() ) { - final params = new SetParams().ex(ttl.toSeconds()) + final params = new SetParams() + if( ttl ) + params.px(ttl.toMillis()) conn.set(key, value, params) } } @Override - boolean putIfAbsent(String key, String value, Duration duration) { - try( Jedis conn=pool.getResource() ) { - final params = new SetParams().nx().ex(duration.toSeconds()) - final result = conn.set(key, value, params) - return result == 'OK' - } + boolean putIfAbsent(String key, String value) { + putIfAbsent(key, value, null) } @Override - String putIfAbsentAndGetCurrent(String key, String value, Duration ttl) { - try (Jedis conn = pool.getResource()){ - final params = new SetParams().nx().ex(ttl.toSeconds()) - final tx = conn.multi() - tx.set(key,value,params) - tx.get(key) - final result = tx.exec() - return result[1].toString() + boolean putIfAbsent(String key, String value, Duration ttl) { + try( Jedis conn=pool.getResource() ) { + final params = new SetParams().nx() + if( ttl ) + params.px(ttl.toMillis()) + final result = conn.set(key, value, params) + return result == 'OK' } } @@ -90,58 +92,4 @@ class RedisCacheProvider implements CacheProvider { } } - // =============== bi-cache store implementation =============== - - @Override - void biPut(String key, String value, Duration ttl) { - final id = DigestUtils.sha256Hex(value) - try( Jedis conn=pool.getResource() ) { - final params = new SetParams().nx().ex(ttl.toSeconds()) - final tx = conn.multi() - tx.set(key, value, params) - tx.sadd(id, key) - tx.exec() - } - } - - @Override - void biRemove(String key) { - try( Jedis conn=pool.getResource() ) { - final value = conn.get(key) - final tx = conn.multi() - tx.del(key) - if( value ) { - final id = DigestUtils.sha256Hex(value) - tx.srem(id, key) - } - tx.exec() - } - } - - @Override - Set biKeysFor(String value) { - final id = DigestUtils.sha256Hex(value) - try( Jedis conn=pool.getResource() ) { - return conn.smembers(id) - } - } - - @Override - String biKeyFind(String value, boolean sorted) { - final id = DigestUtils.sha256Hex(value) - final list = biKeysFor(value).toList() - final keys = sorted ? list.toSorted() : list.shuffled() - final itr = keys.iterator() - while( itr.hasNext() ) { - final key = itr.next() - // verify the key still exists - if( get(key)!=null ) - return key - // if the key is not found, remove it from the set - try( Jedis conn=pool.getResource() ) { - conn.srem(id, key) - } - } - return null - } } diff --git a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy index 03b4e5093..90f7355ca 100644 --- a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy @@ -87,7 +87,8 @@ class ContainerInspectServiceImpl implements ContainerInspectService { @Override String credentialsConfigJson(String containerFile, String buildRepo, String cacheRepo, PlatformId identity) { final repos = new HashSet(10) - repos.addAll(findRepositories(containerFile)) + if( containerFile ) + repos.addAll(findRepositories(containerFile)) if( buildRepo ) repos.add(buildRepo) if( cacheRepo ) diff --git a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy index c01158145..132861ebe 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy @@ -26,6 +26,8 @@ import groovy.transform.CompileStatic import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest import jakarta.inject.Inject import jakarta.inject.Singleton @@ -46,6 +48,10 @@ class JobFactory { @Nullable private ScanConfig scanConfig + @Inject + @Nullable + private MirrorConfig mirrorConfig + JobSpec transfer(String stateId) { JobSpec.transfer( stateId, @@ -75,6 +81,16 @@ class JobFactory { ) } + JobSpec mirror(MirrorRequest request) { + JobSpec.mirror( + request.targetImage, + "mirror-${request.id.substring(MirrorRequest.ID_PREFIX.length())}", + request.creationTime, + mirrorConfig.maxDuration, + request.workDir + ) + } + static private String generate(String type, String id, Instant creationTime) { final prefix = type.toLowerCase() return prefix + '-' + Hashing diff --git a/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy b/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy index 1630c0f26..493df35ec 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy @@ -24,12 +24,48 @@ package io.seqera.wave.service.job */ interface JobHandler { + /** + * Retrieve the {@link JobRecord} instance associated with the specified {@link JobSpec} object + * + * @param job + * The {@link JobSpec} object for which the corresponding record is needed + * @return + * The associated job record or {@link null} otherwise + */ R getJobRecord(JobSpec job) + /** + * Event invoked when a job complete either successfully or with a failure + * + * @param job + * The {@link JobSpec} object + * @param jobRecord + * The associate state record + * @param state + * The job execution state + */ void onJobCompletion(JobSpec job, R jobRecord, JobState state) + /** + * Event invoked when a job execution reports an exception + * + * @param job + * The {@link JobSpec} object + * @param jobRecord + * The associate state record + * @param error + * The job job exception + */ void onJobException(JobSpec job, R jobRecord, Throwable error) + /** + * Event invoked when a job exceed the expected max execution duration + * + * @param job + * The {@link JobSpec} object + * @param jobRecord + * The associate state record + */ void onJobTimeout(JobSpec job, R jobRecord) } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy index 91bd19b77..20439c17f 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.job import io.seqera.wave.service.blob.BlobCacheInfo import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest /** @@ -35,6 +36,8 @@ interface JobService { JobSpec launchScan(ScanRequest request) + JobSpec launchMirror(MirrorRequest request) + JobState status(JobSpec jobSpec) void cleanup(JobSpec jobSpec, Integer exitStatus) diff --git a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy index 8e093af1d..8d4ee4a4f 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy @@ -27,6 +27,8 @@ import io.seqera.wave.service.blob.TransferStrategy import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildStrategy import io.seqera.wave.service.cleanup.CleanupService +import io.seqera.wave.service.mirror.MirrorRequest +import io.seqera.wave.service.mirror.strategy.MirrorStrategy import io.seqera.wave.service.scan.ScanRequest import io.seqera.wave.service.scan.ScanStrategy import jakarta.inject.Inject @@ -64,6 +66,9 @@ class JobServiceImpl implements JobService { @Inject private JobFactory jobFactory + @Inject + private MirrorStrategy mirrorStrategy + @Override JobSpec launchTransfer(BlobCacheInfo blob, List command) { if( !transferStrategy ) @@ -102,6 +107,17 @@ class JobServiceImpl implements JobService { return job } + @Override + JobSpec launchMirror(MirrorRequest request) { + // create the unique job id for the build + final job = jobFactory.mirror(request) + // launch the scan job + mirrorStrategy.mirrorJob(job.operationName, request) + // signal the build has been submitted + jobQueue.offer(job) + return job + } + @Override JobState status(JobSpec job) { try { diff --git a/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy index 90bae336c..14e75d1f3 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy @@ -37,7 +37,7 @@ import io.seqera.wave.util.LongRndKey @CompileStatic class JobSpec { - enum Type { Transfer, Build, Scan } + enum Type { Transfer, Build, Scan, Mirror } /** * The job unique identifier @@ -86,11 +86,11 @@ class JobSpec { this.workDir = dir } - static JobSpec transfer(String stateId, String operationName, Instant creationTime, Duration maxDuration) { + static JobSpec transfer(String recordId, String operationName, Instant creationTime, Duration maxDuration) { new JobSpec( LongRndKey.rndHex(), Type.Transfer, - stateId, + recordId, operationName, creationTime, maxDuration, @@ -98,11 +98,11 @@ class JobSpec { ) } - static JobSpec scan(String stateId, String operationName, Instant creationTime, Duration maxDuration, Path dir) { + static JobSpec scan(String recordId, String operationName, Instant creationTime, Duration maxDuration, Path dir) { new JobSpec( LongRndKey.rndHex(), Type.Scan, - stateId, + recordId, operationName, creationTime, maxDuration, @@ -110,15 +110,27 @@ class JobSpec { ) } - static JobSpec build(String stateId, String operationName, Instant creationTime, Duration maxDuration, Path dir) { + static JobSpec build(String recordId, String operationName, Instant creationTime, Duration maxDuration, Path dir) { new JobSpec( LongRndKey.rndHex(), Type.Build, - stateId, + recordId, operationName, creationTime, maxDuration, dir ) } + + static JobSpec mirror(String recordId, String operationName, Instant creationTime, Duration maxDuration, Path workDir) { + new JobSpec( + LongRndKey.rndHex(), + Type.Mirror, + recordId, + operationName, + creationTime, + maxDuration, + workDir + ) + } } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 6817caffd..6fa0a18b2 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -26,7 +26,8 @@ import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.configuration.SpackConfig +import io.seqera.wave.service.mirror.MirrorConfig + /** * Defines Kubernetes operations * @@ -43,7 +44,7 @@ interface K8sService { void deletePod(String name) @Deprecated - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) @Deprecated V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) @@ -65,10 +66,12 @@ interface K8sService { V1Job launchTransferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) - V1Job launchBuildJob(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) + V1Job launchBuildJob(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) V1Job launchScanJob(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) + V1Job launchMirrorJob(String name, String containerImage, List args, Path workDir, Path creds, MirrorConfig config) + @Deprecated V1PodList waitJob(V1Job job, Long timeout) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 77fe61b6b..1a91e9a39 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -47,8 +47,8 @@ import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.mirror.MirrorConfig import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton @@ -98,9 +98,6 @@ class K8sServiceImpl implements K8sService { @Nullable private String requestsMemory - @Inject - private SpackConfig spackConfig - @Inject private K8sClient k8sClient @@ -300,27 +297,6 @@ class K8sServiceImpl implements K8sService { .readOnly(true) } - protected V1VolumeMount mountSpackCacheDir(Path spackCacheDir, String storageMountPath, String containerPath) { - final rel = Path.of(storageMountPath).relativize(spackCacheDir).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Spack cacheDirectory '$spackCacheDir' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .mountPath(containerPath) - .subPath(rel) - } - - protected V1VolumeMount mountSpackSecretFile(Path secretFile, String storageMountPath, String containerPath) { - final rel = Path.of(storageMountPath).relativize(secretFile).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Spack secretKeyFile '$secretFile' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .readOnly(true) - .mountPath(containerPath) - .subPath(rel) - } - protected V1VolumeMount mountScanCacheDir(Path scanCacheDir, String storageMountPath) { final rel = Path.of(storageMountPath).relativize(scanCacheDir).toString() if( !rel || rel.startsWith('../') ) @@ -349,14 +325,14 @@ class K8sServiceImpl implements K8sService { */ @Override @Deprecated - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { - final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, spackConfig, nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) { + final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, nodeSelector) return k8sClient .coreV1Api() .createNamespacedPod(namespace, spec, null, null, null,null) } - V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { + V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, Map nodeSelector) { // dirty dependency to avoid introducing another parameter final singularity = containerImage.contains('singularity') @@ -379,10 +355,6 @@ class K8sServiceImpl implements K8sService { } } - if( spackConfig ) { - mounts.add(mountSpackSecretFile(spackConfig.secretKeyFile, storageMountPath, spackConfig.secretMountPath)) - } - V1PodBuilder builder = new V1PodBuilder() //metadata section @@ -670,14 +642,14 @@ class K8sServiceImpl implements K8sService { * The {@link V1Pod} description the submitted pod */ @Override - V1Job launchBuildJob(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { - final spec = buildJobSpec(name, containerImage, args, workDir, creds, timeout, spackConfig, nodeSelector) + V1Job launchBuildJob(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) { + final spec = buildJobSpec(name, containerImage, args, workDir, creds, timeout, nodeSelector) return k8sClient .batchV1Api() .createNamespacedJob(namespace, spec, null, null, null,null) } - V1Job buildJobSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { + V1Job buildJobSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, Map nodeSelector) { // dirty dependency to avoid introducing another parameter final singularity = containerImage.contains('singularity') @@ -690,11 +662,14 @@ class K8sServiceImpl implements K8sService { volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) if( credsFile ){ - mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json')) - } - - if( spackConfig ) { - mounts.add(mountSpackSecretFile(spackConfig.secretKeyFile, storageMountPath, spackConfig.secretMountPath)) + if( !singularity ) { + mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json')) + } + else { + final remoteFile = credsFile.resolveSibling('singularity-remote.yaml') + mounts.add(0, mountHostPath(credsFile, storageMountPath, '/root/.singularity/docker-config.json')) + mounts.add(1, mountHostPath(remoteFile, storageMountPath, '/root/.singularity/remote.yaml')) + } } V1JobBuilder builder = new V1JobBuilder() @@ -733,6 +708,7 @@ class K8sServiceImpl implements K8sService { .withImage(containerImage) .withVolumeMounts(mounts) .withResources(requests) + .withWorkingDir('/tmp') if( singularity ) { container @@ -815,6 +791,67 @@ class K8sServiceImpl implements K8sService { return builder.build() } + @Override + V1Job launchMirrorJob(String name, String containerImage, List args, Path workDir, Path creds, MirrorConfig config) { + final spec = mirrorJobSpec(name, containerImage, args, workDir, creds, config) + return k8sClient + .batchV1Api() + .createNamespacedJob(namespace, spec, null, null, null,null) + } + + V1Job mirrorJobSpec(String name, String containerImage, List args, Path workDir, Path credsFile, MirrorConfig config) { + + // required volumes + final mounts = new ArrayList(5) + mounts.add(mountBuildStorage(workDir, storageMountPath, true)) + + final volumes = new ArrayList(5) + volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) + + if( credsFile ){ + mounts.add(0, mountHostPath(credsFile, storageMountPath, '/tmp/config.json')) + } + + V1JobBuilder builder = new V1JobBuilder() + + //metadata section + builder.withNewMetadata() + .withNamespace(namespace) + .withName(name) + .addToLabels(labels) + .endMetadata() + + //spec section + def spec = builder + .withNewSpec() + .withBackoffLimit(config.retryAttempts) + .withNewTemplate() + .editOrNewSpec() + .withServiceAccount(serviceAccount) + .withRestartPolicy("Never") + .addAllToVolumes(volumes) + + final requests = new V1ResourceRequirements() + if( config.requestsCpu ) + requests.putRequestsItem('cpu', new Quantity(config.requestsCpu)) + if( config.requestsMemory ) + requests.putRequestsItem('memory', new Quantity(config.requestsMemory)) + + // container section + final container = new V1ContainerBuilder() + .withName(name) + .withImage(containerImage) + .withArgs(args) + .withVolumeMounts(mounts) + .withResources(requests) + .withEnv(new V1EnvVar().name("REGISTRY_AUTH_FILE").value("/tmp/config.json")) + + // spec section + spec.withContainers(container.build()).endSpec().endTemplate().endSpec() + + return builder.build() + } + protected List toEnvList(Map env) { final result = new ArrayList(env.size()) for( Map.Entry it : env ) diff --git a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy index 6e73804aa..259183d25 100644 --- a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy @@ -97,7 +97,6 @@ class MailServiceImpl implements MailService { binding.build_platform = req.platform binding.build_containerfile = req.containerFile ?: '-' binding.build_condafile = req.condaFile - binding.build_spackfile = req.spackFile binding.build_digest = result.digest ?: '-' binding.put('build_log_data', result.logs) binding.build_url = "$serverUrl/view/builds/${result.id}" diff --git a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorService.groovy b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorService.groovy new file mode 100644 index 000000000..6ef528507 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorService.groovy @@ -0,0 +1,63 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import java.util.concurrent.CompletableFuture + +import io.seqera.wave.service.builder.BuildTrack + +/** + * Define the contract for container images mirroring service + * + * @author Paolo Di Tommaso + */ +interface ContainerMirrorService { + + /** + * Submit a container mirror request + * + * @param request + * The {@link MirrorRequest} modelling the container mirror request + * @return + * A {@link BuildTrack} object representing the state of the request + */ + BuildTrack mirrorImage(MirrorRequest request) + + /** + * Await the completion for the specified target container image + * + * @param targetImage + * The container image of the mirror operation to be awaited + * @return + * A future holding the {@link MirrorState} when the mirror operation complete + */ + CompletableFuture awaitCompletion(String targetImage) + + /** + * Retrieve the current state of the mirror operation + * + * @param id + * The id of the mirror state record + * @return + * The {@link MirrorState} object modelling the current state of the mirror operation, + * or {@link null} otherwise + */ + MirrorState getMirrorState(String id) + +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy new file mode 100644 index 000000000..5e7adc8a1 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy @@ -0,0 +1,139 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ExecutorService + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.scheduling.TaskExecutors +import io.seqera.wave.service.builder.BuildTrack +import io.seqera.wave.service.job.JobHandler +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.persistence.PersistenceService +import jakarta.inject.Inject +import jakarta.inject.Named +import jakarta.inject.Singleton + +/** + * Implement a service to mirror a container image to a repository specified by the user + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@Named('Mirror') +@CompileStatic +class ContainerMirrorServiceImpl implements ContainerMirrorService, JobHandler { + + @Inject + private MirrorStateStore store + + @Inject + private JobService jobService + + @Inject + @Named(TaskExecutors.IO) + private ExecutorService ioExecutor + + @Inject + private PersistenceService persistence + + /** + * {@inheritDoc} + */ + @Override + BuildTrack mirrorImage(MirrorRequest request) { + if( store.putIfAbsent(request.targetImage, MirrorState.from(request))) { + log.info "== Container mirror submitted - request=$request" + jobService.launchMirror(request) + return new BuildTrack(request.id, request.targetImage, false) + } + final ret = store.get(request.targetImage) + if( ret ) { + log.info "== Container mirror hit cache - request=$request" + // note: mark as cached only if the build result is 'done' + // if the build is still in progress it should be marked as not cached + // so that the client will wait for the container completion + return new BuildTrack(ret.mirrorId, ret.targetImage, ret.done()) + } + // invalid state + throw new IllegalStateException("Unable to determine mirror status for '$request.targetImage'") + } + + /** + * {@inheritDoc} + */ + @Override + CompletableFuture awaitCompletion(String targetImage) { + return CompletableFuture.supplyAsync(()-> store.awaitCompletion(targetImage), ioExecutor) + } + + /** + * {@inheritDoc} + */ + @Override + MirrorState getMirrorState(String mirrorId) { + store.getByRecordId(mirrorId) ?: persistence.loadMirrorState(mirrorId) + } + + /** + * {@inheritDoc} + */ + @Override + MirrorState getJobRecord(JobSpec jobSpec) { + store.get(jobSpec.recordId) + } + + /** + * {@inheritDoc} + */ + @Override + void onJobCompletion(JobSpec jobSpec, MirrorState mirror, JobState jobState) { + final result = mirror.complete(jobState.exitCode, jobState.stdout) + store.put(mirror.targetImage, result) + persistence.saveMirrorState(result) + log.debug "Mirror container completed - job=${jobSpec.operationName}; result=${result}; state=${jobState}" + } + + /** + * {@inheritDoc} + */ + @Override + void onJobTimeout(JobSpec jobSpec, MirrorState mirror) { + final result = mirror.complete(null, "Container mirror timed out") + store.put(mirror.targetImage, result) + persistence.saveMirrorState(result) + log.warn "Mirror container timed out - job=${jobSpec.operationName}; result=${result}" + } + + /** + * {@inheritDoc} + */ + @Override + void onJobException(JobSpec jobSpec, MirrorState mirror, Throwable error) { + final result = mirror.complete(null, error.message) + store.put(mirror.targetImage, result) + persistence.saveMirrorState(result) + log.error("Mirror container errored - job=${jobSpec.operationName}; result=${result}", error) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy new file mode 100644 index 000000000..52be93819 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy @@ -0,0 +1,59 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import java.time.Duration + +import groovy.transform.CompileStatic +import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable +import jakarta.inject.Singleton +/** + * Model mirror service config options + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +class MirrorConfig { + + @Value('${wave.mirror.max-duration:15m}') + Duration maxDuration + + @Value('${wave.mirror.status.delay:4s}') + Duration statusDelay + + @Value('${wave.mirror.status.duration:1h}') + Duration statusDuration + + @Value('${wave.mirror.skopeoImage:`quay.io/skopeo/stable`}') + String skopeoImage + + @Value('${wave.mirror.retry-attempts:3}') + Integer retryAttempts + + @Nullable + @Value('${wave.mirror.requestsCpu}') + String requestsCpu + + @Nullable + @Value('${wave.mirror.requestsMemory}') + String requestsMemory + +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorRequest.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorRequest.groovy new file mode 100644 index 000000000..f1157cc72 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorRequest.groovy @@ -0,0 +1,99 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import java.nio.file.Path +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.util.LongRndKey + +/** + * Model a container mirror request + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@Canonical +@CompileStatic +class MirrorRequest { + + static final String ID_PREFIX = 'mr-' + + /** + * Unique id of the request + */ + final String id + + /** + * The container image to be mirrored + */ + final String sourceImage + + /** + * The target image name where the container should be mirrored + */ + final String targetImage + + /** + * The (SHA256) digest of the container to be mirrored + */ + final String digest + + /** + * The container platform to be copied + */ + final ContainerPlatform platform + + /** + * The work directory used by the mirror operation + */ + final Path workDir + + /** + * Docker config json to authorise the mirror (pull & push) operation + */ + final String authJson + + /** + * The timestamp when the request has been submitted + */ + final Instant creationTime + + static MirrorRequest create(String sourceImage, String targetImage, String digest, ContainerPlatform platform, Path workspace, String authJson, Instant ts=Instant.now()) { + assert sourceImage, "Argument 'sourceImage' cannot be null" + assert targetImage, "Argument 'targetImage' cannot be empty" + assert workspace, "Argument 'workspace' cannot be null" + assert digest, "Argument 'digest' cannot be empty" + + final id = LongRndKey.rndHex() + return new MirrorRequest( + ID_PREFIX + id, + sourceImage, + targetImage, + digest, + platform, + workspace.resolve("mirror-${id}"), + authJson, + ts ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorState.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorState.groovy new file mode 100644 index 000000000..39f313d51 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorState.groovy @@ -0,0 +1,111 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import java.time.Duration +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.cache.StateRecord +import io.seqera.wave.service.job.JobRecord +import jakarta.inject.Singleton + +/** + * Model a container mirror result object + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@Singleton +@CompileStatic +@Canonical +class MirrorState implements JobRecord, StateRecord { + enum Status { PENDING, COMPLETED } + + final String mirrorId + final String digest + final String sourceImage + final String targetImage + final ContainerPlatform platform + final Instant creationTime + final Status status + final Duration duration + final Integer exitCode + final String logs + + @Override + String getRecordId() { + return mirrorId + } + + @Override + boolean done() { + status==Status.COMPLETED + } + + boolean succeeded() { + status==Status.COMPLETED && exitCode==0 + } + + MirrorState complete(Integer exitCode, String logs ) { + new MirrorState( + this.mirrorId, + this.digest, + this.sourceImage, + this.targetImage, + this.platform, + this.creationTime, + Status.COMPLETED, + Duration.between(this.creationTime, Instant.now()), + exitCode, + logs + ) + } + + static MirrorState from(MirrorRequest request) { + new MirrorState( + request.id, + request.digest, + request.sourceImage, + request.targetImage, + request.platform, + request.creationTime, + Status.PENDING + ) + } + + BuildStatusResponse toStatusResponse() { + final status = status == Status.COMPLETED + ? BuildStatusResponse.Status.COMPLETED + : BuildStatusResponse.Status.PENDING + final succeeded = exitCode!=null + ? exitCode==0 + : null + return new BuildStatusResponse( + mirrorId, + status, + creationTime, + duration, + succeeded ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy new file mode 100644 index 000000000..85338ded2 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy @@ -0,0 +1,72 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import java.time.Duration + +import groovy.transform.CompileStatic +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.service.cache.AbstractCacheStore +import io.seqera.wave.service.cache.impl.CacheProvider +import jakarta.inject.Inject +import jakarta.inject.Singleton + +/** + * Implement a {@link io.seqera.wave.service.cache.CacheStore} for {@link MirrorState} objects + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +class MirrorStateStore extends AbstractCacheStore { + + @Inject + private MirrorConfig config + + MirrorStateStore(CacheProvider provider) { + super(provider, new MoshiEncodeStrategy() {}) + } + + @Override + protected String getPrefix() { + return 'wave-mirror/v1:' + } + + @Override + protected Duration getDuration() { + return config.statusDuration + } + + MirrorState awaitCompletion(String targetImage) { + final beg = System.currentTimeMillis() + while( true ) { + final result = get(targetImage) + // missing record + if( !result ) + throw new IllegalStateException("Unknown mirror container $targetImage") + // ok done + if( result.done() ) + return result + Thread.sleep(config.statusDelay) + // timeout the request + if( System.currentTimeMillis()-beg > config.statusDuration.toMillis() ) + throw new IllegalStateException("Timeout mirror container $targetImage") + } + } +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategy.groovy b/src/main/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategy.groovy new file mode 100644 index 000000000..9fa346d66 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategy.groovy @@ -0,0 +1,106 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror.strategy + + +import java.nio.file.Files +import java.nio.file.Path + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value +import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.service.mirror.MirrorRequest +import jakarta.inject.Inject +import jakarta.inject.Singleton +import static java.nio.file.StandardOpenOption.CREATE +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING +import static java.nio.file.StandardOpenOption.WRITE + +/** + * Implements a container mirror runner based on Docker + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +@Slf4j +class DockerMirrorStrategy extends MirrorStrategy { + + @Value('${wave.debug:false}') + private Boolean debug + + @Inject + private MirrorConfig mirrorConfig + + @Override + void mirrorJob(String jobName, MirrorRequest request) { + Path configFile = null + // create the work directory + Files.createDirectories(request.workDir) + // save docker config for creds + if( request.authJson ) { + configFile = request.workDir.resolve('config.json') + Files.write(configFile, JsonOutput.prettyPrint(request.authJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + } + + // command the docker build command + final buildCmd = mirrorCmd(jobName, request.workDir, configFile) + buildCmd.addAll( copyCommand(request) ) + log.debug "Container mirror command: ${buildCmd.join(' ')}" + // save docker cli for debugging purpose + if( debug ) { + Files.write(request.workDir.resolve('docker.sh'), + buildCmd.join(' ').bytes, + CREATE, WRITE, TRUNCATE_EXISTING) + } + + final process = new ProcessBuilder() + .command(buildCmd) + .directory(request.workDir.toFile()) + .redirectErrorStream(true) + .start() + if( process.waitFor()!=0 ) { + throw new IllegalStateException("Unable to launch mirror container job - exitCode=${process.exitValue()}; output=${process.text}") + } + } + + protected List mirrorCmd(String name, Path workDir, Path credsFile ) { + //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker + final wrapper = ['docker', + 'run', + '--detach', + '--name', name, + '-v', "$workDir:$workDir".toString() ] + + if( credsFile ) { + wrapper.add('-v') + wrapper.add("$credsFile:/tmp/config.json:ro".toString()) + + wrapper.add("-e") + wrapper.add("REGISTRY_AUTH_FILE=/tmp/config.json") + } + + // the container image to be used to build + wrapper.add( mirrorConfig.skopeoImage ) + // return it + return wrapper + } +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy b/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy new file mode 100644 index 000000000..40fa62ed2 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy @@ -0,0 +1,83 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror.strategy + +import java.nio.file.Files +import java.nio.file.Path + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.kubernetes.client.openapi.ApiException +import io.micronaut.context.annotation.Primary +import io.micronaut.context.annotation.Requires +import io.seqera.wave.exception.BadRequestException +import io.seqera.wave.service.k8s.K8sService +import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.service.mirror.MirrorRequest +import jakarta.inject.Inject +import jakarta.inject.Singleton +import static java.nio.file.StandardOpenOption.CREATE +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING +import static java.nio.file.StandardOpenOption.WRITE + +/** + * Implements a container mirror runner based on Kubernetes + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Primary +@Requires(property = 'wave.build.k8s') +@Singleton +@CompileStatic +class KubeMirrorStrategy extends MirrorStrategy { + + @Inject + private MirrorConfig config + + @Inject + private K8sService k8sService + + @Override + void mirrorJob(String jobName, MirrorRequest request) { + Path configFile = null + // create the work directory + Files.createDirectories(request.workDir) + // save docker config for creds + if( request.authJson ) { + configFile = request.workDir.resolve('config.json') + Files.write(configFile, JsonOutput.prettyPrint(request.authJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + } + + try { + k8sService.launchMirrorJob( + jobName, + config.skopeoImage, + copyCommand(request), + request.workDir, + configFile, + config) + } + catch (ApiException e) { + throw new BadRequestException("Unexpected build failure - ${e.responseBody}", e) + } + + } +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategy.groovy b/src/main/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategy.groovy new file mode 100644 index 000000000..a69acfe6e --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategy.groovy @@ -0,0 +1,57 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror.strategy + + +import groovy.transform.CompileStatic +import io.seqera.wave.service.mirror.MirrorRequest +/** + * Implement the common strategy to handle container mirror + * via Skopeo + * + * @author Paolo Di Tommaso + */ +@CompileStatic +abstract class MirrorStrategy { + + abstract void mirrorJob(String jobName, MirrorRequest request) + + protected List copyCommand(MirrorRequest request) { + final result = new ArrayList(20) + if( request.platform ) { + result.add("--override-os") + result.add(request.platform.os) + result.add("--override-arch") + result.add(request.platform.arch) + if( request.platform.variant ) { + result.add("--override-variant") + result.add(request.platform.variant) + } + } + + result.add("copy") + result.add("--preserve-digests") + result.add("--multi-arch") + result.add( request.platform ? 'system' : 'all') + result.add("docker://${request.sourceImage}".toString()) + result.add("docker://${request.targetImage}".toString()) + + return result + } +} diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index 3db2b6793..b1701652f 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -21,6 +21,7 @@ package io.seqera.wave.service.persistence import groovy.transform.CompileStatic import io.seqera.wave.core.ContainerDigestPair import io.seqera.wave.exception.NotFoundException +import io.seqera.wave.service.mirror.MirrorState import io.seqera.wave.service.scan.ScanResult /** * A storage for statistic data @@ -58,6 +59,14 @@ interface PersistenceService { */ WaveBuildRecord loadBuild(String targetImage, String digest) + /** + * Retrieve the latest {@link WaveBuildRecord} object for the given container id + * + * @param containerId The container id for which the latest build record should be retrieved + * @return The corresponding {@link WaveBuildRecord} object or {@code null} if no record is found + */ + WaveBuildRecord latestBuild(String containerId) + /** * Store a {@link WaveContainerRecord} object in the Surreal wave_request table. * @@ -141,4 +150,27 @@ interface PersistenceService { * @return The corresponding condaLock file as a string */ WaveCondaLockRecord loadCondaLock(String buildId) + * Load a mirror state record + * + * @param mirrorId The ID of the mirror record + * @return The corresponding {@link MirrorState} object or null if it cannot be found + */ + MirrorState loadMirrorState(String mirrorId) + + /** + * Load a mirror state record given the target image name and the image digest + * + * @param targetImage The target mirrored image name + * @param digest The image content SHA256 digest + * @return The corresponding {@link MirrorState} object or null if it cannot be found + */ + MirrorState loadMirrorState(String targetImage, String digest) + + /** + * Persists a {@link MirrorState} state record + * + * @param mirror {@link MirrorState} object + */ + void saveMirrorState(MirrorState mirror) + } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index 10107f5d0..ba5225b34 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -42,7 +42,6 @@ class WaveBuildRecord { String buildId String dockerFile String condaFile - String spackFile String targetImage String userName String userEmail @@ -66,7 +65,6 @@ class WaveBuildRecord { buildId: event.request.buildId, dockerFile: event.request.containerFile, condaFile: event.request.condaFile, - spackFile: event.request.spackFile, targetImage: event.request.targetImage, userName: event.request.identity.user?.userName, userEmail: event.request.identity.user?.email, diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy index 797ffaf2e..aed66f85d 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.persistence.impl import groovy.transform.CompileStatic import io.seqera.wave.core.ContainerDigestPair +import io.seqera.wave.service.mirror.MirrorState import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveCondaLockRecord @@ -40,6 +41,7 @@ class LocalPersistenceService implements PersistenceService { private Map requestStore = new HashMap<>() private Map scanStore = new HashMap<>() + private Map mirrorStore = new HashMap<>() private Map condaLockStore = new HashMap<>() @@ -53,6 +55,15 @@ class LocalPersistenceService implements PersistenceService { return buildStore.get(buildId) } + @Override + WaveBuildRecord latestBuild(String containerId) { + buildStore + .values() + .findAll( it-> it.buildId.startsWith(containerId) ) + .sort( it-> it.startTime ) + .reverse() [0] + } + @Override WaveBuildRecord loadBuild(String targetImage, String digest) { buildStore.values().find( (build) -> build.targetImage==targetImage && build.digest==digest ) @@ -100,4 +111,17 @@ class LocalPersistenceService implements PersistenceService { WaveCondaLockRecord loadCondaLock(String buildId) { return condaLockStore.get(buildId) } + + MirrorState loadMirrorState(String mirrorId) { + mirrorStore.get(mirrorId) + } + + MirrorState loadMirrorState(String targetImage, String digest) { + mirrorStore.values().find( (MirrorState mirror) -> mirror.targetImage==targetImage && mirror.digest==digest ) + } + + void saveMirrorState(MirrorState mirror) { + mirrorStore.put(mirror.mirrorId, mirror) + } + } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy index fd5f582d6..14fee8927 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy @@ -27,6 +27,7 @@ import io.micronaut.http.annotation.Post import io.micronaut.http.annotation.Put import io.micronaut.http.client.annotation.Client import io.micronaut.retry.annotation.Retryable +import io.seqera.wave.service.mirror.MirrorState import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.service.persistence.WaveBuildRecord @@ -85,4 +86,6 @@ interface SurrealClient { @Post('/key/wave_scan_vuln') Map insertScanVulnerability(@Header String authorization, @Body ScanVulnerability scanVulnerability) + @Post('/key/wave_mirror') + Flux> insertMirrorAsync(@Header String authorization, @Body MirrorState body) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index 043fe55ab..90b74c5e0 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -30,6 +30,7 @@ import io.micronaut.runtime.event.ApplicationStartupEvent import io.micronaut.runtime.event.annotation.EventListener import io.seqera.wave.core.ContainerDigestPair import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.mirror.MirrorState import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveCondaLockRecord @@ -101,9 +102,13 @@ class SurrealPersistenceService implements PersistenceService { final ret8 = surrealDb.sqlAsMap(authorization, "DEFINE INDEX idx_buildId ON TABLE wave_conda_lock COLUMNS buildId UNIQUE;") if( ret8.status != "OK") throw new IllegalStateException("Unable to define SurrealDB index idx_buildId on table wave_conda_lock - cause: $ret8") + // create wave_mirror table + final ret5 = surrealDb.sqlAsMap(authorization, "define table wave_mirror SCHEMALESS") + if( ret5.status != "OK") + throw new IllegalStateException("Unable to define SurrealDB table wave_mirror - cause: $ret5") } - private String getAuthorization() { + protected String getAuthorization() { "Basic "+"$user:$password".bytes.encodeBase64() } @@ -160,6 +165,21 @@ class SurrealPersistenceService implements PersistenceService { return result } + @Override + WaveBuildRecord latestBuild(String containerId) { + final query = """ + select * + from wave_build + where buildId ~ '${containerId}${BuildRequest.SEP}' + order by startTime desc limit 1 + """.stripIndent() + final json = surrealDb.sqlAsString(getAuthorization(), query) + final type = new TypeReference>>() {} + final data= json ? JacksonHelper.fromJson(json, type) : null + final result = data && data[0].result ? data[0].result[0] : null + return result + } + @Override void saveContainerRequest(String token, WaveContainerRecord data) { surrealDb.insertContainerRequestAsync(authorization, token, data).subscribe({ result-> @@ -274,8 +294,56 @@ class SurrealPersistenceService implements PersistenceService { final query = "select * from wave_conda_lock where buildId = '$buildId'" final json = surrealDb.sqlAsString(getAuthorization(), query) final type = new TypeReference>>() {} + + // === mirror operations + + /** + * Load a mirror state record + * + * @param mirrorId The ID of the mirror record + * @return The corresponding {@link MirrorState} object or null if it cannot be found + */ + MirrorState loadMirrorState(String mirrorId) { + final query = "select * from wave_mirror where mirrorId = '$mirrorId'" + final json = surrealDb.sqlAsString(getAuthorization(), query) + final type = new TypeReference>>() {} final data= json ? JacksonHelper.fromJson(json, type) : null final result = data && data[0].result ? data[0].result[0] : null return result } + + /** + * Load a mirror state record given the target image name and the image digest + * + * @param targetImage The target mirrored image name + * @param digest The image content SHA256 digest + * @return The corresponding {@link MirrorState} object or null if it cannot be found + */ + MirrorState loadMirrorState(String targetImage, String digest) { + final query = "select * from wave_mirror where targetImage = '$targetImage' and digest = '$digest'" + final json = surrealDb.sqlAsString(getAuthorization(), query) + final type = new TypeReference>>() {} + final data= json ? JacksonHelper.fromJson(json, type) : null + final result = data && data[0].result ? data[0].result[0] : null + return result + } + + /** + * Persists a {@link MirrorState} object + * + * @param mirror {@link MirrorState} object + */ + @Override + void saveMirrorState(MirrorState mirror) { + surrealDb.insertMirrorAsync(getAuthorization(), mirror).subscribe({ result-> + log.trace "Mirror request with id '$mirror.mirrorId' saved record: ${result}" + }, {error-> + def msg = error.message + if( error instanceof HttpClientResponseException ){ + msg += ":\n $error.response.body" + } + log.error("Error saving Mirror request record ${msg}\n${mirror}", error) + }) + } + } diff --git a/src/main/groovy/io/seqera/wave/service/validation/ValidationService.groovy b/src/main/groovy/io/seqera/wave/service/validation/ValidationService.groovy index 472105112..888e83c94 100644 --- a/src/main/groovy/io/seqera/wave/service/validation/ValidationService.groovy +++ b/src/main/groovy/io/seqera/wave/service/validation/ValidationService.groovy @@ -31,4 +31,6 @@ interface ValidationService { String checkBuildRepository(String repo, boolean cache) + String checkMirrorRegistry(String registry) + } diff --git a/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy index 6e5e220e4..cbecf6f61 100644 --- a/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy @@ -92,4 +92,20 @@ class ValidationServiceImpl implements ValidationService { return null } + @Override + String checkMirrorRegistry(String registry) { + if( !registry ) + return null + final prot = StringUtils.getUrlProtocol(registry) + if( prot ) + return "Mirror registry should not include any protocol prefix - offending value: $registry" + // check no tag is included + final coords = ContainerCoordinates.parse(registry) + if( coords.repository ) + return "Mirror registry syntax is invalid - offending value: ${registry}" + if( coords.registry == 'wave.seqera.io' || coords.registry?.contains('.wave.seqera.io') ) + return "Mirror registry not allowed - offending value: ${registry}" + return null + } + } diff --git a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy index 39dd5ae35..7348d27ce 100644 --- a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy @@ -27,7 +27,6 @@ import io.seqera.wave.api.PackagesSpec import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.api.SubmitContainerTokenResponse import io.seqera.wave.config.CondaOpts -import io.seqera.wave.config.SpackOpts import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.ContainerRequestData @@ -35,16 +34,12 @@ import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.token.TokenData import org.yaml.snakeyaml.Yaml import static io.seqera.wave.service.builder.BuildFormat.SINGULARITY -import static io.seqera.wave.util.DockerHelper.addPackagesToSpackYaml import static io.seqera.wave.util.DockerHelper.condaEnvironmentToCondaYaml import static io.seqera.wave.util.DockerHelper.condaFileToDockerFile import static io.seqera.wave.util.DockerHelper.condaFileToSingularityFile import static io.seqera.wave.util.DockerHelper.condaPackagesToCondaYaml import static io.seqera.wave.util.DockerHelper.condaPackagesToDockerFile import static io.seqera.wave.util.DockerHelper.condaPackagesToSingularityFile -import static io.seqera.wave.util.DockerHelper.spackFileToDockerFile -import static io.seqera.wave.util.DockerHelper.spackFileToSingularityFile -import static io.seqera.wave.util.DockerHelper.spackPackagesToSpackYaml /** * Container helper methods * @@ -82,16 +77,7 @@ class ContainerHelper { return result } - if( spec.type == PackagesSpec.Type.SPACK ) { - if( !spec.spackOpts ) - spec.spackOpts = new SpackOpts() - final result = formatSingularity - ? spackFileToSingularityFile(spec.spackOpts) - : spackFileToDockerFile(spec.spackOpts) - return result - } - - throw new IllegalArgumentException("Unexpected packages spec type: $spec.type") + throw new BadRequestException("Unexpected packages spec type: $spec.type") } static String condaFileFromRequest(SubmitContainerTokenRequest req) { @@ -129,26 +115,6 @@ class ContainerHelper { return result[0] } - static String spackFileFromRequest(SubmitContainerTokenRequest req) { - if( !req.packages ) - return decodeBase64OrFail(req.spackFile,'spackFile') - - if( req.packages.type != PackagesSpec.Type.SPACK ) - return null - - if( req.packages.environment ) { - final decoded = decodeBase64OrFail(req.packages.environment,'packages.envFile') - return addPackagesToSpackYaml(decoded, req.packages.spackOpts) - } - - if( req.packages.entries ) { - final String packages = req.packages.entries.join(' ') - return spackPackagesToSpackYaml(packages, req.packages.spackOpts) - } - - return null - } - static String decodeBase64OrFail(String value, String field) { if( !value ) return null @@ -167,16 +133,16 @@ class ContainerHelper { static SubmitContainerTokenResponse makeResponseV1(ContainerRequestData data, TokenData token, String waveImage) { final target = waveImage final build = data.buildNew ? data.buildId : null - return new SubmitContainerTokenResponse(token.value, target, token.expiration, data.containerImage, build, null, null) + return new SubmitContainerTokenResponse(token.value, target, token.expiration, data.containerImage, build, null, null, null) } static SubmitContainerTokenResponse makeResponseV2(ContainerRequestData data, TokenData token, String waveImage) { - final target = data.freeze ? data.containerImage : waveImage + final target = data.durable() ? data.containerImage : waveImage final build = data.buildId final Boolean cached = !data.buildNew - final expiration = !data.freeze ? token.expiration : null - final tokenId = !data.freeze ? token.value : null - return new SubmitContainerTokenResponse(tokenId, target, expiration, data.containerImage, build, cached, data.freeze) + final expiration = !data.durable() ? token.expiration : null + final tokenId = !data.durable() ? token.value : null + return new SubmitContainerTokenResponse(tokenId, target, expiration, data.containerImage, build, cached, data.freeze, data.mirror) } static String patchPlatformEndpoint(String endpoint) { @@ -268,56 +234,7 @@ class ContainerHelper { return new Tuple2(parts[0], parts[1]) } - static NameVersionPair guessSpackRecipeName(String spackFileContent, boolean split=false) { - if( !spackFileContent ) - return null - try { - final yaml = new Yaml().load(spackFileContent) as Map - final spack = yaml.spack as Map - - if( !spack ){ - throw new BadRequestException('Malformed Spack environment file - missing "spack:" section') - } - if( !spack.specs ){ - throw new BadRequestException('Malformed Spack environment file - missing "spack.specs:" section') - } - - if( spack.specs instanceof List ) { - final LinkedHashSet result = new LinkedHashSet() - final LinkedHashSet versions = new LinkedHashSet() - for( String it : spack.specs ) { - final p = it.indexOf(' ') - // remove everything after the first blank because they are supposed package directives - if( p!=-1 ) - it = it.substring(0,p) - if( split ) { - final pair = splitVersion(it, '@') - it = pair.v1 - versions.add(pair.v2) - } - else { - // replaces '@' version separator with `-` - it = it.replace('@','-') - } - if( it ) - result.add(it) - } - return split - ? new NameVersionPair(result, versions) - : new NameVersionPair(result) - } - return null - } - catch (BadRequestException e) { - throw e - } - catch (Throwable e) { - log.warn "Unable to infer spack recipe name - cause: ${e.message}", e - return null - } - } - - static String makeTargetImage(BuildFormat format, String repo, String id, @Nullable String condaFile, @Nullable String spackFile, @Nullable ImageNameStrategy nameStrategy) { + static String makeTargetImage(BuildFormat format, String repo, String id, @Nullable String condaFile, @Nullable ImageNameStrategy nameStrategy) { assert id, "Argument 'id' cannot be null or empty" assert repo, "Argument 'repo' cannot be null or empty" assert format, "Argument 'format' cannot be null" @@ -328,9 +245,6 @@ class ContainerHelper { if( condaFile && (tools=guessCondaRecipeName(condaFile,false)) ) { tag = "${normaliseTag(tools.qualifiedNames())}--${id}" } - else if( spackFile && (tools=guessSpackRecipeName(spackFile,false)) ) { - tag = "${normaliseTag(tools.qualifiedNames())}--${id}" - } } else if( nameStrategy==ImageNameStrategy.imageSuffix ) { if( condaFile && (tools=guessCondaRecipeName(condaFile,true)) ) { @@ -338,11 +252,6 @@ class ContainerHelper { if( tools.versions?.size()==1 && tools.versions[0] ) tag = "${normaliseTag(tools.versions[0])}--${id}" } - else if( spackFile && (tools=guessSpackRecipeName(spackFile, true)) ) { - repo = StringUtils.pathConcat(repo, normaliseName(tools.friendlyNames())) - if( tools.versions?.size()==1 && tools.versions[0] ) - tag = "${normaliseTag(tools.versions[0])}--${id}" - } } else if( nameStrategy!=ImageNameStrategy.none ) { throw new BadRequestException("Unsupported image naming strategy: '${nameStrategy}'") @@ -387,13 +296,12 @@ class ContainerHelper { value ? normalise0(value.toLowerCase(), maxLength, /[^a-z0-9_.\-\/]/) : null } - static String makeContainerId(String containerFile, String condaFile, String spackFile, ContainerPlatform platform, String repository, BuildContext buildContext) { + static String makeContainerId(String containerFile, String condaFile, ContainerPlatform platform, String repository, BuildContext buildContext) { final attrs = new LinkedHashMap(10) attrs.containerFile = containerFile attrs.condaFile = condaFile attrs.platform = platform?.toString() attrs.repository = repository - if( spackFile ) attrs.spackFile = spackFile if( buildContext ) attrs.buildContext = buildContext.tarDigest return RegHelper.sipHash(attrs) } diff --git a/src/main/groovy/io/seqera/wave/util/SpackHelper.groovy b/src/main/groovy/io/seqera/wave/util/SpackHelper.groovy deleted file mode 100644 index 82a324a98..000000000 --- a/src/main/groovy/io/seqera/wave/util/SpackHelper.groovy +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.util - - -import groovy.transform.CompileStatic -import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.builder.BuildFormat - -/** - * Helper class for Spack package manager - * - * @author Paolo Di Tommaso - */ -@CompileStatic -@Deprecated -class SpackHelper { - - static String builderDockerTemplate() { - SpackHelper.class - .getResourceAsStream('/io/seqera/wave/spack/spack-builder-dockerfile.txt') - .getText() - } - - static String builderSingularityTemplate() { - SpackHelper.class - .getResourceAsStream('/io/seqera/wave/spack/spack-builder-singularityfile.txt') - .getText() - } - - static String prependBuilderTemplate(String dockerContent, BuildFormat buildFormat) { - if(buildFormat == BuildFormat.SINGULARITY){ - return builderSingularityTemplate() + dockerContent - } - else if( buildFormat == BuildFormat.DOCKER ) { - return builderDockerTemplate() + dockerContent - } - else - throw new IllegalStateException("Unexpected build format: $buildFormat") - } - - static String toSpackArch(ContainerPlatform platform) { - if( !platform ) - throw new IllegalArgumentException("Missing container platform argument") - final value = platform.toString() - if( value=='linux/amd64' ) - return 'x86_64' - if( value=='linux/arm64' ) - return 'aarch64' - throw new IllegalArgumentException("Unable to map container platform '${platform}' to Spack architecture") - } - -} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 297fe0279..f9dc8c18f 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -4,8 +4,6 @@ wave: enabled: true build: workspace: 'build-workspace' - spack: - cacheDirectory: 'spack-cache' metrics: enabled: true accounts: diff --git a/src/main/resources/io/seqera/wave/build-notification.html b/src/main/resources/io/seqera/wave/build-notification.html index 61d14fb04..0c3a96c3a 100644 --- a/src/main/resources/io/seqera/wave/build-notification.html +++ b/src/main/resources/io/seqera/wave/build-notification.html @@ -111,11 +111,6 @@

Conda file

${build_condafile}
<% } %> - <% if (build_spackfile) { %> -

Spack file

- 
${build_spackfile}
- <% } %> -

Build logs

${build_log_data}
diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 09c459c87..d2d13f825 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -134,11 +134,6 @@ 
{{build_condafile}}
{{/if}} - {{#if build_spackfile}} -

Spack file

- 
{{build_spackfile}}
- {{/if}} - {{#if build_log_data}}

Build logs

{{build_log_data}}
diff --git a/src/main/resources/io/seqera/wave/spack/spack-builder-dockerfile.txt b/src/main/resources/io/seqera/wave/spack/spack-builder-dockerfile.txt deleted file mode 100644 index 5637b1533..000000000 --- a/src/main/resources/io/seqera/wave/spack/spack-builder-dockerfile.txt +++ /dev/null @@ -1,53 +0,0 @@ -# Builder image -FROM {{spack_builder_image}} as builder -COPY spack.yaml /opt/spack-env/spack.yaml - -# Assume the values from the environment -ARG AWS_STS_REGIONAL_ENDPOINTS -ARG AWS_REGION -ARG AWS_DEFAULT_REGION -ARG AWS_ROLE_ARN -ARG AWS_WEB_IDENTITY_TOKEN_FILE - -ENV \ - AWS_STS_REGIONAL_ENDPOINTS=${AWS_STS_REGIONAL_ENDPOINTS} \ - AWS_REGION=${AWS_REGION} \ - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} \ - AWS_ROLE_ARN=${AWS_ROLE_ARN} \ - AWS_WEB_IDENTITY_TOKEN_FILE=${AWS_WEB_IDENTITY_TOKEN_FILE} - -RUN mkdir -p /opt/spack-env \ -&& sed -i -e 's;compilers:;compilers::;' \ - -e 's;^ *flags: *{}; flags:\n cflags: -O3\n cxxflags: -O3\n fflags: -O3;' \ - /root/.spack/linux/compilers.yaml \ -&& cd /opt/spack-env \ -&& spack config add config:install_tree:/opt/software \ -&& spack config add concretizer:unify:true \ -&& spack config add concretizer:reuse:true \ -&& spack config add packages:all:target:[{{spack_arch}}] \ -&& printf " view: /opt/view\n" >> /opt/spack-env/spack.yaml - -# Install packages, clean afterward, finally strip binaries -RUN cd /opt/spack-env \ -&& fingerprint="$(spack gpg trust {{spack_key_file}} 2>&1 | tee /dev/stderr | sed -nr "s/^gpg: key ([0-9A-F]{16}): secret key imported$/\1/p")" \ -&& spack mirror add seqera-spack {{spack_cache_bucket}} \ -&& spack mirror add binary_mirror https://binaries.spack.io/releases/v0.20 \ -&& spack buildcache keys --install --trust \ -&& spack -e . concretize -f \ -&& spack --env . install \ -&& spack -e . buildcache push --allow-root --key "$fingerprint" {{spack_cache_bucket}} \ -&& spack gc -y \ -&& ( find -L /opt/._view/* -type f -exec readlink -f '{}' \; | \ - xargs file -i | \ - grep 'charset=binary' | \ - grep 'x-executable\|x-archive\|x-sharedlib' | \ - awk -F: '{print $1}' | xargs strip -s ) || true - -RUN cd /opt/spack-env && \ - spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \ - original_view=$( cd /opt ; ls -1d ._view/* ) && \ - sed -i "s;/view/;/$original_view/;" /opt/spack-env/z10_spack_environment.sh && \ - echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \ - echo "export PERL5LIB=$(eval ls -d /opt/._view/*/lib/5.*):$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \ - rm -rf /opt/view - diff --git a/src/main/resources/io/seqera/wave/spack/spack-builder-singularityfile.txt b/src/main/resources/io/seqera/wave/spack/spack-builder-singularityfile.txt deleted file mode 100644 index 5500cd652..000000000 --- a/src/main/resources/io/seqera/wave/spack/spack-builder-singularityfile.txt +++ /dev/null @@ -1,59 +0,0 @@ -Bootstrap: docker -From: {{spack_builder_image}} -Stage: build - -%setup -cat < ${SINGULARITY_ROOTFS}/aws-env.sh -export AWS_STS_REGIONAL_ENDPOINTS=${AWS_STS_REGIONAL_ENDPOINTS} -export AWS_REGION=${AWS_REGION} -export AWS_DEFAULT_REGION=${AWS_REGION} -export AWS_ROLE_ARN=${AWS_ROLE_ARN} -export AWS_WEB_IDENTITY_TOKEN_FILE=${AWS_WEB_IDENTITY_TOKEN_FILE} -EOF - -%files - {{wave_context_dir}}/spack.yaml /opt/spack-env/spack.yaml - {{spack_key_file}} {{spack_key_file}} - /var/run/secrets /var/run/secrets - -%post - # import AWS environment - . /aws-env.sh - - # Copy and modify spack.yaml - sed -i -e 's;compilers:;compilers::;' \ - -e 's;^ *flags: *{}; flags:\n cflags: -O3\n cxxflags: -O3\n fflags: -O3;' \ - /root/.spack/linux/compilers.yaml - - # Set up Spack environment - export PATH=/opt/spack/bin:$PATH - cd /opt/spack-env - spack config add config:install_tree:/opt/software - spack config add concretizer:unify:true - spack config add concretizer:reuse:true - spack config add packages:all:target:[{{spack_arch}}] - printf " view: /opt/view\n" >> /opt/spack-env/spack.yaml - - # Install packages, clean afterward, finally strip binaries - fingerprint="$(spack gpg trust {{spack_key_file}} 2>&1 | tee /dev/stderr | sed -nr "s/^gpg: key ([0-9A-F]{16}): secret key imported$/\1/p")" - spack mirror add seqera-spack {{spack_cache_bucket}} - spack mirror add binary_mirror https://binaries.spack.io/releases/v0.20 - spack buildcache keys --install --trust - spack -e . concretize -f - spack --env . install - spack -e . buildcache push --allow-root --key "$fingerprint" {{spack_cache_bucket}} - spack gc -y - ( find -L /opt/._view/* -type f -exec readlink -f '{}' \; | \ - xargs file -i | \ - grep 'charset=binary' | \ - grep 'x-executable\|x-archive\|x-sharedlib' | \ - awk -F: '{print $1}' | xargs strip -s ) || true - - # Set up environment variables - spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh - original_view=$( cd /opt ; ls -1d ._view/* ) - sed -i "s;/view/;/$original_view/;" /opt/spack-env/z10_spack_environment.sh - echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh - echo "export PERL5LIB=$(set +f; ls -d /opt/._view/*/lib/5.*):\$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh - rm -rf /opt/view - diff --git a/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy index aacb51586..74d2db23d 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy @@ -56,7 +56,6 @@ class BuildConfigTest extends Specification { config.singularityImage( ContainerPlatform.of('arm64') ) == 'bar' } - @Unroll def 'should validate build max duration' () { given: @@ -72,6 +71,5 @@ class BuildConfigTest extends Specification { 'xyz' | false | 5 | 10 | 5 'xtz' | true | 5 | 10 | 10 // <-- pick "trusted" because both "freeze" and "token" are provided 'xtz' | true | 20 | 10 | 20 // <-- pick "default" when it's greater than "trusted" - } } diff --git a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy index cdaf8473c..a081c962b 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy @@ -76,8 +76,8 @@ class BuildControllerTest extends Specification { final containerFile = 'FROM foo:latest' final format = BuildFormat.DOCKER final platform = ContainerPlatform.of('amd64') - final containerId = ContainerHelper.makeContainerId(containerFile, null, null, platform, 'buildrepo', null) - final targetImage = ContainerHelper.makeTargetImage(format, repo, containerId, null, null, null) + final containerId = ContainerHelper.makeContainerId(containerFile, null, platform, 'buildrepo', null) + final targetImage = ContainerHelper.makeTargetImage(format, repo, containerId, null, null) final build = new BuildRequest( containerId: containerId, containerFile: containerFile, diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy index 0d707d001..74ff3bf59 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy @@ -24,12 +24,14 @@ import spock.lang.Unroll import java.time.Instant import java.time.temporal.ChronoUnit +import io.micronaut.context.ApplicationContext import io.micronaut.context.annotation.Property import io.micronaut.http.HttpRequest import io.micronaut.http.HttpStatus import io.micronaut.http.client.HttpClient import io.micronaut.http.client.annotation.Client import io.micronaut.http.server.util.HttpClientAddressResolver +import io.micronaut.test.annotation.MockBean import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.ImageNameStrategy @@ -37,7 +39,6 @@ import io.seqera.wave.api.PackagesSpec import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.api.SubmitContainerTokenResponse import io.seqera.wave.config.CondaOpts -import io.seqera.wave.config.SpackOpts import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.core.RegistryProxyService @@ -50,6 +51,9 @@ import io.seqera.wave.service.builder.FreezeService import io.seqera.wave.service.builder.FreezeServiceImpl import io.seqera.wave.service.inclusion.ContainerInclusionService import io.seqera.wave.service.inspect.ContainerInspectServiceImpl +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobServiceImpl +import io.seqera.wave.service.mirror.ContainerMirrorService import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.service.pairing.socket.PairingChannel import io.seqera.wave.service.persistence.PersistenceService @@ -81,6 +85,14 @@ class ContainerControllerTest extends Specification { @Inject JwtAuthStore jwtAuthStore + @Inject + ApplicationContext applicationContext + + @MockBean(JobServiceImpl) + JobService mockJobService() { + Mock(JobService) + } + def setup() { jwtAuthStore.clear() } @@ -157,7 +169,7 @@ class ContainerControllerTest extends Specification { def builder = Mock(ContainerBuildService) def dockerAuth = Mock(ContainerInspectServiceImpl) def proxyRegistry = Mock(RegistryProxyService) - def controller = new ContainerController(buildService: builder, dockerAuthService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService)) + def controller = new ContainerController(buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService)) def DOCKER = 'FROM foo' def user = new PlatformId(new User(id: 100)) def cfg = new ContainerConfig() @@ -185,7 +197,7 @@ class ContainerControllerTest extends Specification { def dockerAuth = Mock(ContainerInspectServiceImpl) def proxyRegistry = Mock(RegistryProxyService) def persistenceService = Mock(PersistenceService) - def controller = new ContainerController(buildService: builder, dockerAuthService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, persistenceService:persistenceService, inclusionService: Mock(ContainerInclusionService)) + def controller = new ContainerController(buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, persistenceService:persistenceService, inclusionService: Mock(ContainerInclusionService)) def DOCKER = 'FROM foo' def user = new PlatformId(new User(id: 100)) def cfg = new ContainerConfig() @@ -213,7 +225,7 @@ class ContainerControllerTest extends Specification { def builder = Mock(ContainerBuildService) def dockerAuth = Mock(ContainerInspectServiceImpl) def proxyRegistry = Mock(RegistryProxyService) - def controller = new ContainerController(buildService: builder, dockerAuthService: dockerAuth, registryProxyService: proxyRegistry, buildConfig:buildConfig, inclusionService: Mock(ContainerInclusionService)) + def controller = new ContainerController(buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig:buildConfig, inclusionService: Mock(ContainerInclusionService)) def DOCKER = 'FROM foo' def user = new PlatformId(new User(id: 100)) def cfg = new ContainerConfig() @@ -237,10 +249,39 @@ class ContainerControllerTest extends Specification { data.platform.toString() == 'linux/arm64' } + def 'should make a mirror request' () { + given: + def jobService = applicationContext.getBean(JobService) + def mirrorService = applicationContext.getBean(ContainerMirrorService) + def inspectService = Mock(ContainerInspectServiceImpl) + def proxyRegistry = Mock(RegistryProxyService) + def controller = new ContainerController(mirrorService: mirrorService, inspectService: inspectService, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService)) + def user = new PlatformId(new User(id: 100)) + def req = new SubmitContainerTokenRequest( + containerImage: 'docker.io/source/image:latest', + containerPlatform: 'arm64', + mirrorRegistry: 'quay.io' + ) + + when: + def data = controller.makeRequestData(req, user, "") + then: + 1 * proxyRegistry.getImageDigest('docker.io/source/image:latest', user) >> 'sha256:12345' + 1 * proxyRegistry.getImageDigest('quay.io/source/image:latest', user) >> null + and: + data.identity.userId == 100 + data.containerImage == 'quay.io/source/image:latest' + data.platform.toString() == 'linux/arm64' + data.buildId =~ /mr-.+/ + data.buildNew + !data.freeze + data.mirror + } + def 'should create build request' () { given: def dockerAuth = Mock(ContainerInspectServiceImpl) - def controller = new ContainerController(dockerAuthService: dockerAuth, buildConfig: buildConfig) + def controller = new ContainerController(inspectService: dockerAuth, buildConfig: buildConfig) when: def submit = new SubmitContainerTokenRequest(containerFile: encode('FROM foo')) @@ -280,23 +321,12 @@ class ContainerControllerTest extends Specification { build.targetImage == 'wave/build:c6dac2e544419f71' build.platform == ContainerPlatform.of('arm64') - when: - submit = new SubmitContainerTokenRequest(containerFile: encode('FROM foo'), spackFile: encode('some::spack-recipe'), containerPlatform: 'arm64') - build = controller.makeBuildRequest(submit, PlatformId.NULL, "") - then: - build.containerId =~ /b7d730d274d1e057/ - build.containerFile.endsWith('\nFROM foo') - build.containerFile.startsWith('# Builder image\n') - build.condaFile == null - build.spackFile == 'some::spack-recipe' - build.targetImage == 'wave/build:b7d730d274d1e057' - build.platform == ContainerPlatform.of('arm64') } def 'should return a bad request exception when field is not encoded' () { given: def dockerAuth = Mock(ContainerInspectServiceImpl) - def controller = new ContainerController(dockerAuthService: dockerAuth, buildConfig: buildConfig) + def controller = new ContainerController(inspectService: dockerAuth, buildConfig: buildConfig) // validate containerFile when: @@ -318,16 +348,6 @@ class ContainerControllerTest extends Specification { e = thrown(BadRequestException) e.message == "Invalid 'condaFile' attribute - make sure it encoded as a base64 string" - // validate spackFile - when: - controller.makeBuildRequest( - new SubmitContainerTokenRequest(containerFile: encode('FROM foo'), spackFile: 'spack@123'), - Mock(PlatformId), - null) - then: - e = thrown(BadRequestException) - e.message == "Invalid 'spackFile' attribute - make sure it encoded as a base64 string" - } def 'should add library prefix' () { @@ -391,7 +411,7 @@ class ContainerControllerTest extends Specification { def pairing = Mock(PairingService) def channel = Mock(PairingChannel) def controller = new ContainerController(validationService: validation, pairingService: pairing, pairingChannel: channel) - def msg + def err when: controller.validateContainerRequest(new SubmitContainerTokenRequest()) @@ -411,17 +431,91 @@ class ContainerControllerTest extends Specification { when: controller.validateContainerRequest(new SubmitContainerTokenRequest(containerImage: 'http://docker.io/foo:latest')) then: - msg = thrown(BadRequestException) - msg.message == 'Invalid container repository name — offending value: http://docker.io/foo:latest' + err = thrown(BadRequestException) + err.message == 'Invalid container repository name — offending value: http://docker.io/foo:latest' when: controller.validateContainerRequest(new SubmitContainerTokenRequest(containerImage: 'http:docker.io/foo:latest')) then: - msg = thrown(BadRequestException) - msg.message == 'Invalid container image name — offending value: http:docker.io/foo:latest' + err = thrown(BadRequestException) + err.message == 'Invalid container image name — offending value: http:docker.io/foo:latest' } + def 'should validate mirror request' () { + given: + def validation = new ValidationServiceImpl() + def pairing = Mock(PairingService) + def channel = Mock(PairingChannel) + def controller = new ContainerController(validationService: validation, pairingService: pairing, pairingChannel: channel) + def err + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(containerImage: 'foo:latest'), false) + then: + noExceptionThrown() + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io'), false) + then: + err = thrown(BadRequestException) + err.message == 'Container mirroring requires the use of v2 API' + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io'), true) + then: + err = thrown(BadRequestException) + err.message == 'Attribute `containerImage` is required when specifying `mirrorRegistry`' + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo'), true) + then: + err = thrown(BadRequestException) + err.message == 'Container mirroring requires an authenticated request - specify the tower token attribute' + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'docker.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz'), true) + then: + err = thrown(BadRequestException) + err.message == "Source and target mirror registry as the same - offending value 'docker.io'" + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'docker.io', containerImage: 'foo', towerAccessToken: 'xyz'), true) + then: + err = thrown(BadRequestException) + err.message == "Source and target mirror registry as the same - offending value 'docker.io'" + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerFile: 'content'), true) + then: + err = thrown(BadRequestException) + err.message == "Attribute `mirrorRegistry` and `containerFile` conflict each other" + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', freeze: true), true) + then: + err = thrown(BadRequestException) + err.message == "Attribute `mirrorRegistry` and `freeze` conflict each other" + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerIncludes: ['include']), true) + then: + err = thrown(BadRequestException) + err.message == "Attribute `mirrorRegistry` and `containerIncludes` conflict each other" + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerConfig: new ContainerConfig(entrypoint: ['foo'])), true) + then: + err = thrown(BadRequestException) + err.message == "Attribute `mirrorRegistry` and `containerConfig` conflict each other" + + when: + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io/bar', containerImage: 'docker.io/foo', towerAccessToken: 'xyz'), true) + then: + err = thrown(BadRequestException) + err.message == "Mirror registry syntax is invalid - offending value: quay.io/bar" + } + def 'should create response with conda packages' () { given: def dockerAuth = Mock(ContainerInspectServiceImpl) @@ -431,7 +525,7 @@ class ContainerControllerTest extends Specification { def addressResolver = Mock(HttpClientAddressResolver) def tokenService = Mock(ContainerTokenService) def persistence = Mock(PersistenceService) - def controller = new ContainerController(freezeService: freeze, buildService: builder, dockerAuthService: dockerAuth, + def controller = new ContainerController(freezeService: freeze, buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService), addressResolver: addressResolver, tokenService: tokenService, persistenceService: persistence, serverUrl: 'http://wave.com') @@ -459,42 +553,6 @@ class ContainerControllerTest extends Specification { } } - def 'should create response with spack packages' () { - given: - def dockerAuth = Mock(ContainerInspectServiceImpl) - def freeze = new FreezeServiceImpl( inspectService: dockerAuth) - def builder = Mock(ContainerBuildService) - def proxyRegistry = Mock(RegistryProxyService) - def addressResolver = Mock(HttpClientAddressResolver) - def tokenService = Mock(ContainerTokenService) - def persistence = Mock(PersistenceService) - def controller = new ContainerController(freezeService: freeze, buildService: builder, dockerAuthService: dockerAuth, - registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService), - addressResolver: addressResolver, tokenService: tokenService, persistenceService: persistence, serverUrl: 'https://wave.seqera.io') - - when:'packages with spack' - def SPACK_OPTS = new SpackOpts([ - basePackages: 'foo bar', - commands: ['run','--this','--that'] - ]) - def packages = new PackagesSpec(type: PackagesSpec.Type.SPACK, spackOpts: SPACK_OPTS) - def req = new SubmitContainerTokenRequest(format: 'docker', packages: packages) - def response = controller.handleRequest(null, req, new PlatformId(new User(id: 100), 10), true) - - then: - 1 * builder.buildImage(_) >> new BuildTrack('build123', 'foo:1234', true) - and: - 1 * tokenService.computeToken(_) >> new TokenData('wavetoken123', Instant.now().plus(1, ChronoUnit.HOURS)) - and: - response.status.code == 200 - verifyAll(response.body.get() as SubmitContainerTokenResponse) { - targetImage == 'wave.seqera.io/wt/wavetoken123/library/foo:1234' - buildId == 'build123' - containerToken == 'wavetoken123' - cached == true - } - } - def 'should throw BadRequestException when more than one artifact (container image, container file or packages) is provided in the request' () { given: def controller = new ContainerController(inclusionService: Mock(ContainerInclusionService), allowAnonymous: false) diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index be0f6f67b..8317188e0 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.controller import spock.lang.Specification +import spock.lang.Unroll import java.time.Duration import java.time.Instant @@ -32,10 +33,9 @@ import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.core.spec.ContainerSpec -import io.seqera.wave.exception.DockerRegistryException import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.conda.CondaLockService +import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.logs.BuildLogService import io.seqera.wave.service.logs.BuildLogServiceImpl @@ -78,7 +78,7 @@ class ViewControllerTest extends Specification { @Inject private CondaLockService condaLockService - def 'should render build page' () { + def 'should return build page mapping' () { given: def controller = new ViewController(serverUrl: 'http://foo.com', buildLogService: buildLogService, condaLockService: condaLockService) and: @@ -86,7 +86,6 @@ class ViewControllerTest extends Specification { buildId: '12345', dockerFile: 'FROM foo', condaFile: 'conda::foo', - spackFile: 'some-spack-recipe', targetImage: 'docker.io/some:image', userName: 'paolo', userEmail: 'paolo@seqera.io', @@ -112,7 +111,6 @@ class ViewControllerTest extends Specification { binding.build_platform == 'linux/amd64' binding.build_containerfile == 'FROM foo' binding.build_condafile == 'conda::foo' - binding.build_spackfile == 'some-spack-recipe' binding.build_format == 'Docker' binding.build_log_data == 'log content' binding.build_log_truncated == false @@ -123,7 +121,7 @@ class ViewControllerTest extends Specification { binding.build_failed == false } - def 'should render a build page' () { + def 'should render build page' () { given: def record1 = new WaveBuildRecord( buildId: '112233', @@ -149,10 +147,9 @@ class ViewControllerTest extends Specification { response.body().contains('FROM docker.io/test:foo') and: !response.body().contains('Conda file') - !response.body().contains('Spack file') } - def 'should render a build page with conda file' () { + def 'should render build page with conda file' () { given: def record1 = new WaveBuildRecord( buildId: 'test', @@ -179,39 +176,6 @@ class ViewControllerTest extends Specification { and: response.body().contains('Conda file') response.body().contains('conda::foo') - and: - !response.body().contains('Spack file') - } - - def 'should render a build page with spack file' () { - given: - def record1 = new WaveBuildRecord( - buildId: 'test', - spackFile: 'foo/conda/recipe', - targetImage: 'test', - userName: 'test', - userEmail: 'test', - userId: 1, - requestIp: '127.0.0.1', - startTime: Instant.now(), - duration: Duration.ofSeconds(1), - exitStatus: 0 ) - - when: - persistenceService.saveBuild(record1) - and: - def request = HttpRequest.GET("/view/builds/${record1.buildId}") - def response = client.toBlocking().exchange(request, String) - then: - response.body().contains(record1.buildId) - and: - response.body().contains('Container file') - response.body().contains('-') - and: - !response.body().contains('Conda file') - and: - response.body().contains('Spack file') - response.body().contains('foo/conda/recipe') } def 'should render container view page' () { @@ -268,7 +232,6 @@ class ViewControllerTest extends Specification { buildId: '12345', dockerFile: 'FROM foo', condaFile: 'conda::foo', - spackFile: 'some-spack-recipe', targetImage: 'docker.io/some:image', userName: 'paolo', userEmail: 'paolo@seqera.io', @@ -293,7 +256,6 @@ class ViewControllerTest extends Specification { binding.build_platform == 'linux/amd64' binding.build_containerfile == 'FROM foo' binding.build_condafile == 'conda::foo' - binding.build_spackfile == 'some-spack-recipe' binding.build_format == 'Docker' binding.build_log_data == 'log content' binding.build_log_truncated == false @@ -311,7 +273,6 @@ class ViewControllerTest extends Specification { buildId: '12345', dockerFile: 'FROM foo', condaFile: 'conda::foo', - spackFile: 'some-spack-recipe', targetImage: 'docker.io/some:image', userName: 'paolo', userEmail: 'paolo@seqera.io', @@ -337,7 +298,6 @@ class ViewControllerTest extends Specification { binding.build_platform == 'linux/amd64' binding.build_containerfile == 'FROM foo' binding.build_condafile == 'conda::foo' - binding.build_spackfile == 'some-spack-recipe' binding.build_format == 'Docker' binding.build_log_data == 'log content' binding.build_log_truncated == false @@ -372,4 +332,45 @@ class ViewControllerTest extends Specification { binding.build_url == 'http://foo.com/view/builds/12345' } + @Unroll + def 'should validate redirection check' () { + given: + def service = Mock(ContainerBuildService) + def controller = new ViewController(buildService: service) + + when: + def result = controller.shouldRedirect1(BUILD) + then: + result == EXPECTED + + where: + BUILD | EXPECTED + '12345_1' | null + '12345-1' | '/view/builds/12345_1' + 'foo-887766-1' | '/view/builds/foo-887766_1' + + } + + + def 'should validate redirect 2' () { + given: + def service = Mock(ContainerBuildService) + def controller = new ViewController(buildService: service) + + when: + def result = controller.shouldRedirect2(BUILD) + then: + result == EXPECTED + TIMES * service.getLatestBuild(BUILD) >> LATEST + + where: + BUILD | TIMES | LATEST | EXPECTED + '12345_1' | 0 | null | null + '12345' | 1 | Mock(WaveBuildRecord) { buildId >> '12345_99' } | '/view/builds/12345_99' + '12345' | 1 | Mock(WaveBuildRecord) { buildId >> 'xyz_99' } | null + 'foo-887766' | 1 | Mock(WaveBuildRecord) { buildId >> 'foo-887766_99' } | '/view/builds/foo-887766_99' + 'foo-887766' | 1 | Mock(WaveBuildRecord) { buildId >> 'foo-887766' } | null + + } + } diff --git a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy index 9cee2c36f..361fa3c79 100644 --- a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy @@ -329,7 +329,6 @@ class MoshiEncodingStrategyTest extends Specification { containerId: '12345', containerFile: 'from foo', condaFile: 'conda spec', - spackFile: 'spack spec', workspace: Path.of("/some/path"), targetImage: 'docker.io/some:image:12345', identity: PlatformId.NULL, @@ -356,6 +355,21 @@ class MoshiEncodingStrategyTest extends Specification { copy == record1 } + def 'should decode legacy record with spack config' () { + given: + def legacy = '{"buildId":"12345_1","condaFile":"conda spec","dockerFile":"from foo","duration":"3000000000","exitStatus":-1,"platform":"linux/amd64","requestIp":"1.2.3.4","scanId":"scan12345","spackFile":"spack spec","targetImage":"docker.io/some:image:12345"}\n' + and: + def encoder = new MoshiEncodeStrategy() { } + + // verify decoding is OK when the payload contains `spackFile` not existing anymore in the WaveBuildRecord + when: + def rec = encoder.decode(legacy) + then: + rec.buildId == '12345_1' + rec.condaFile == 'conda spec' + rec.dockerFile == 'from foo' + } + def 'should encode and decode registry info' () { given: def encoder = new MoshiEncodeStrategy() { } @@ -405,7 +419,6 @@ class MoshiEncodingStrategyTest extends Specification { containerId: '12345', containerFile: 'from foo', condaFile: 'conda spec', - spackFile: 'spack spec', workspace: Path.of("/some/path"), targetImage: 'docker.io/some:image:12345', identity: PlatformId.NULL, diff --git a/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy b/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy index 6bff0952a..e8e05f867 100644 --- a/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy +++ b/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy @@ -72,8 +72,8 @@ class BuildServiceRateLimitTest extends Specification { RUN echo hi > hello.txt """.stripIndent() and: - def CONTAINER_ID = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def TARGET_IMAGE = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, CONTAINER_ID, null, null, null) + def CONTAINER_ID = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def TARGET_IMAGE = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, CONTAINER_ID, null, null) def REQ = new BuildRequest( containerId: CONTAINER_ID, containerFile: dockerfile, @@ -106,8 +106,8 @@ class BuildServiceRateLimitTest extends Specification { RUN echo hi > hello.txt """.stripIndent() and: - def CONTAINER_ID = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def TARGET_IMAGE = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, CONTAINER_ID, null, null, null) + def CONTAINER_ID = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def TARGET_IMAGE = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, CONTAINER_ID, null, null) def REQ = new BuildRequest( containerId: CONTAINER_ID, containerFile: dockerfile, diff --git a/src/test/groovy/io/seqera/wave/service/ContainerRequestDataTest.groovy b/src/test/groovy/io/seqera/wave/service/ContainerRequestDataTest.groovy index d807826bb..fd1c647ac 100644 --- a/src/test/groovy/io/seqera/wave/service/ContainerRequestDataTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/ContainerRequestDataTest.groovy @@ -20,6 +20,8 @@ package io.seqera.wave.service import spock.lang.Specification +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User @@ -38,7 +40,59 @@ class ContainerRequestDataTest extends Specification { then: req.identity req.identity == new PlatformId(new User(id:1)) + } + + def 'should validate constructor' () { + when: + def cfg = Mock(ContainerConfig) + def req = new ContainerRequestData( + new PlatformId(new User(id:1)), + 'foo', + 'from docker', + cfg, + 'conda file', + ContainerPlatform.DEFAULT, + '12345', + true, + true, + true ) + then: + req.identity == new PlatformId(new User(id:1)) + req.containerImage == 'foo' + req.containerFile == 'from docker' + req.containerConfig == cfg + req.condaFile == 'conda file' + req.platform == ContainerPlatform.DEFAULT + req.buildId == '12345' + req.buildNew + req.freeze + req.mirror + + } + + def 'should validate durable flag' () { + given: + def req = new ContainerRequestData( + new PlatformId(new User(id:1)), + null, + null, + null, + null, + null, + null, + null, + FREEZE, + MIRROR ) + + expect: + req.durable() == EXPECTED + where: + FREEZE | MIRROR | EXPECTED + false | false | false + true | false | true + false | true | true + true | true | true } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy index e5eaa2172..9959f7873 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy @@ -52,15 +52,14 @@ class BuildRequestTest extends Specification { def CONTEXT = Mock(BuildContext) def PLATFORM = ContainerPlatform.of('amd64') def FORMAT = BuildFormat.DOCKER - def CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, null, null, PLATFORM, BUILD_REPO, CONTEXT) - def TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, null, null, null) + def CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, null, PLATFORM, BUILD_REPO, CONTEXT) + def TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, null, null) when: def req = new BuildRequest( CONTAINER_ID, CONTENT, null, - null, PATH, TARGET_IMAGE, USER, @@ -85,7 +84,6 @@ class BuildRequestTest extends Specification { req.cacheRepository == CACHE_REPO req.format == BuildFormat.DOCKER req.condaFile == null - req.spackFile == null req.platform == ContainerPlatform.of('amd64') req.configJson == '{"config":"json"}' req.scanId == SCAN_ID @@ -93,8 +91,6 @@ class BuildRequestTest extends Specification { req.offsetId == OFFSET req.containerConfig == CONFIG req.buildContext == CONTEXT - and: - !req.isSpackBuild // ==== provide a Conda recipe ==== when: @@ -103,13 +99,12 @@ class BuildRequestTest extends Specification { - samtools=1.0 ''' and: - CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, CONDA_RECIPE, null, PLATFORM, BUILD_REPO, CONTEXT) - TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, CONDA_RECIPE, null, null) + CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, CONDA_RECIPE, PLATFORM, BUILD_REPO, CONTEXT) + TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, CONDA_RECIPE, null) req = new BuildRequest( CONTAINER_ID, CONTENT, CONDA_RECIPE, - null, PATH, TARGET_IMAGE, USER, @@ -128,24 +123,14 @@ class BuildRequestTest extends Specification { req.containerId == '8026e3a63b5c863f' req.targetImage == 'docker.io/wave:samtools-1.0--8026e3a63b5c863f' req.condaFile == CONDA_RECIPE - req.spackFile == null - and: - !req.isSpackBuild - - // ===== spack content ==== - def SPACK_RECIPE = '''\ - spack: - specs: [bwa@0.7.15] - ''' when: - CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, null, SPACK_RECIPE, PLATFORM, BUILD_REPO, CONTEXT) - TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, null, SPACK_RECIPE, null) + CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, null, PLATFORM, BUILD_REPO, CONTEXT) + TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, null, null) req = new BuildRequest( CONTAINER_ID, CONTENT, null, - SPACK_RECIPE, PATH, TARGET_IMAGE, USER, @@ -161,12 +146,9 @@ class BuildRequestTest extends Specification { TIMEOUT ) then: - req.containerId == '8726782b1d9bb8fb' - req.targetImage == 'docker.io/wave:bwa-0.7.15--8726782b1d9bb8fb' - req.spackFile == SPACK_RECIPE + req.containerId == '181ec22b26ae6d04' + req.targetImage == 'docker.io/wave:181ec22b26ae6d04' req.condaFile == null - and: - req.isSpackBuild } def 'should create singularity build request'() { @@ -183,15 +165,14 @@ class BuildRequestTest extends Specification { def CONTEXT = Mock(BuildContext) def PLATFORM = ContainerPlatform.of('amd64') def FORMAT = BuildFormat.SINGULARITY - def CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, null, null, PLATFORM, BUILD_REPO, CONTEXT) - def TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, null, null, null) + def CONTAINER_ID = ContainerHelper.makeContainerId(CONTENT, null, PLATFORM, BUILD_REPO, CONTEXT) + def TARGET_IMAGE = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID, null, null) when: def req = new BuildRequest( CONTAINER_ID, CONTENT, null, - null, PATH, TARGET_IMAGE, USER, @@ -220,8 +201,6 @@ class BuildRequestTest extends Specification { req.offsetId == OFFSET req.containerConfig == CONFIG req.buildContext == CONTEXT - and: - !req.isSpackBuild } @@ -238,28 +217,28 @@ class BuildRequestTest extends Specification { def FOO_CONTENT = 'from foo' def BAR_CONTENT = 'from bar' and: - def CONTAINER_ID1 = ContainerHelper.makeContainerId(FOO_CONTENT, null, null, PLATFORM, BUILD_REPO, null) - def TARGET_IMAGE1 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID1, null, null, null) - def req1 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) + def CONTAINER_ID1 = ContainerHelper.makeContainerId(FOO_CONTENT, null, PLATFORM, BUILD_REPO, null) + def TARGET_IMAGE1 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID1, null, null) + def req1 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def req2 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) + def req2 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def CONTAINER_ID3 = ContainerHelper.makeContainerId(BAR_CONTENT, null, null, PLATFORM, BUILD_REPO, null) - def TARGET_IMAGE3 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID3, null, null, null) - def req3 = new BuildRequest(CONTAINER_ID3, BAR_CONTENT, null, null, PATH, TARGET_IMAGE3, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) + def CONTAINER_ID3 = ContainerHelper.makeContainerId(BAR_CONTENT, null, PLATFORM, BUILD_REPO, null) + def TARGET_IMAGE3 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID3, null, null) + def req3 = new BuildRequest(CONTAINER_ID3, BAR_CONTENT, null, PATH, TARGET_IMAGE3, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def CONTAINER_ID4 = ContainerHelper.makeContainerId(BAR_CONTENT, CONDA_CONTENT, null, PLATFORM, BUILD_REPO, null) - def TARGET_IMAGE4 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID4, CONDA_CONTENT, null, null) - def req4 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) + def CONTAINER_ID4 = ContainerHelper.makeContainerId(BAR_CONTENT, CONDA_CONTENT, PLATFORM, BUILD_REPO, null) + def TARGET_IMAGE4 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID4, CONDA_CONTENT, null) + def req4 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def req5 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) + def req5 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: CONDA_CONTENT = 'salmon=1.2.5' - def CONTAINER_ID6 = ContainerHelper.makeContainerId(BAR_CONTENT, CONDA_CONTENT, null, PLATFORM, BUILD_REPO, null) - def TARGET_IMAGE6 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID6, CONDA_CONTENT, null, null) - def req6 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) + def CONTAINER_ID6 = ContainerHelper.makeContainerId(BAR_CONTENT, CONDA_CONTENT, PLATFORM, BUILD_REPO, null) + def TARGET_IMAGE6 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID6, CONDA_CONTENT, null) + def req6 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def req7 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', "UTC+2", null, null, null, FORMAT, TIMEOUT) + def req7 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', "UTC+2", null, null, null, FORMAT, TIMEOUT) expect: req1 == req2 diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy index 8151c836f..fbab4a86c 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy @@ -128,13 +128,12 @@ class BuildStrategyTest extends Specification { def content = 'FROM foo:latest' def workspace = Path.of("some/path") def buildrepo = 'foo.com/repo' - def containerId = ContainerHelper.makeContainerId(content, null, null, ContainerPlatform.of('amd64'), buildrepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildrepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(content, null, ContainerPlatform.of('amd64'), buildrepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildrepo, containerId, null, null) def build = new BuildRequest( containerId, content, null, - null, workspace, targetImage, PlatformId.NULL, diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy index eae4f8c42..359e93043 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy @@ -76,8 +76,8 @@ class ContainerBuildServiceLiveTest extends Specification { '''.stripIndent() and: def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, cacheRepo, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) def req = new BuildRequest( containerId: containerId, @@ -122,8 +122,8 @@ class ContainerBuildServiceLiveTest extends Specification { '''.stripIndent() and: def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) def req = new BuildRequest( containerId: containerId, @@ -168,8 +168,8 @@ class ContainerBuildServiceLiveTest extends Specification { and: def buildRepo = "quay.io/pditommaso/wave-tests" def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('linux/arm64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, ContainerPlatform.of('linux/arm64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) def req = new BuildRequest( containerId: containerId, @@ -214,8 +214,8 @@ class ContainerBuildServiceLiveTest extends Specification { and: def duration = Duration.ofMinutes(1) def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) def req = new BuildRequest( containerId: containerId, @@ -266,8 +266,8 @@ class ContainerBuildServiceLiveTest extends Specification { and: def duration = Duration.ofMinutes(1) def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) def req = new BuildRequest( containerId: containerId, diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index 7e0e6d647..21c05e66d 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -40,7 +40,6 @@ import io.seqera.wave.auth.RegistryCredentialsProvider import io.seqera.wave.auth.RegistryLookupService import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.HttpClientConfig -import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.service.builder.store.BuildRecordStore @@ -56,7 +55,6 @@ import io.seqera.wave.test.TestHelper import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.ContainerHelper import io.seqera.wave.util.Packer -import io.seqera.wave.util.SpackHelper import io.seqera.wave.util.TemplateRenderer import jakarta.inject.Inject import jakarta.inject.Singleton @@ -110,11 +108,9 @@ class ContainerBuildServiceTest extends Specification { def buildRepo = buildConfig.defaultBuildRepository def cacheRepo = buildConfig.defaultCacheRepository and: - def cfg = 'some credentials' def dockerFile = ''' FROM busybox RUN echo Hello > hello.txt - RUN {{spack_cache_bucket}} {{spack_key_file}} '''.stripIndent() and: def condaFile = ''' @@ -122,22 +118,13 @@ class ContainerBuildServiceTest extends Specification { - salmon=1.6.0 ''' and: - def spackFile = ''' - spack: - specs: [bwa@0.7.15, salmon@1.1.1] - concretizer: {unify: true, reuse: true} - ''' - and: - def spackConfig = new SpackConfig(cacheBucket: 's3://bucket/cache', secretMountPath: '/mnt/secret') - def containerId = ContainerHelper.makeContainerId(dockerFile, condaFile, spackFile, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, condaFile, spackFile, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, condaFile, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, condaFile, null) def req = new BuildRequest( containerId: containerId, containerFile: dockerFile, condaFile: condaFile, - spackFile: spackFile, - isSpackBuild: true, workspace: folder, targetImage: targetImage, identity: Mock(PlatformId), @@ -151,17 +138,17 @@ class ContainerBuildServiceTest extends Specification { and: def store = Mock(BuildStore) def jobService = Mock(JobService) - def builder = new ContainerBuildServiceImpl(buildStore: store, buildConfig: buildConfig, spackConfig:spackConfig, jobService: jobService) + def builder = new ContainerBuildServiceImpl(buildStore: store, buildConfig: buildConfig, jobService: jobService) def RESPONSE = Mock(JobSpec) - + when: builder.launch(req) + then: 1 * jobService.launchBuild(req) >> RESPONSE and: - req.workDir.resolve('Containerfile').text == new TemplateRenderer().render(dockerFile, [spack_cache_bucket:'s3://bucket/cache', spack_key_file:'/mnt/secret']) + req.workDir.resolve('Containerfile').text == new TemplateRenderer().render(dockerFile, [:]) req.workDir.resolve('context/conda.yml').text == condaFile - req.workDir.resolve('context/spack.yaml').text == spackFile cleanup: folder?.deleteDir() @@ -172,11 +159,10 @@ class ContainerBuildServiceTest extends Specification { def folder = Files.createTempDirectory('test') def builder = new ContainerBuildServiceImpl() def buildRepo = buildConfig.defaultBuildRepository - def cacheRepo = buildConfig.defaultCacheRepository and: def dockerFile = 'FROM something; {{foo}}' - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) def req = new BuildRequest( containerId: containerId, @@ -189,112 +175,16 @@ class ContainerBuildServiceTest extends Specification { startTime: Instant.now() ) .withBuildId('1') - and: - def spack = Mock(SpackConfig) when: - def result = builder.containerFile0(req, null, spack) + def result = builder.containerFile0(req, null) then: - 0* spack.getCacheMountPath() >> null - 0* spack.getSecretMountPath() >> null - 0* spack.getBuilderImage() >> null - and: result == 'FROM something; {{foo}}' cleanup: folder?.deleteDir() } - def 'should resolve docker file with spack config' () { - given: - def folder = Files.createTempDirectory('test') - def builder = new ContainerBuildServiceImpl() - and: - def dockerFile = SpackHelper.builderDockerTemplate() - def spackFile = 'some spack packages' - def containerId = ContainerHelper.makeContainerId(dockerFile, null, spackFile, ContainerPlatform.of('amd64'), 'buildRepo', null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, 'foo.com/repo', containerId, null, spackFile, null) - def req = new BuildRequest( - containerId: containerId, - containerFile: dockerFile, - spackFile: spackFile, - isSpackBuild: true, - workspace: folder, - targetImage: targetImage, - identity:Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), - format: BuildFormat.DOCKER, - startTime: Instant.now() - ) - .withBuildId('1') - and: - def spack = Mock(SpackConfig) - - when: - def result = builder.containerFile0(req, null, spack) - then: - 1* spack.getCacheBucket() >> 's3://bucket/cache' - 1* spack.getSecretMountPath() >> '/mnt/key' - 1* spack.getBuilderImage() >> 'spack-builder:2.0' - 1* spack.getRunnerImage() >> 'ubuntu:22.04' - and: - result.contains('FROM spack-builder:2.0 as builder') - result.contains('spack config add packages:all:target:[x86_64]') - result.contains('spack mirror add seqera-spack s3://bucket/cache') - result.contains('fingerprint="$(spack gpg trust /mnt/key 2>&1 | tee /dev/stderr | sed -nr "s/^gpg: key ([0-9A-F]{16}): secret key imported$/\\1/p")"') - - cleanup: - folder?.deleteDir() - } - - def 'should resolve singularity file with spack config' () { - given: - def folder = Files.createTempDirectory('test') - def builder = new ContainerBuildServiceImpl() - and: - def context = Path.of('/some/context/dir') - def dockerFile = SpackHelper.builderSingularityTemplate() - def spackFile = 'some spack packages' - def containerId = ContainerHelper.makeContainerId(dockerFile, null, spackFile, ContainerPlatform.of('amd64'), 'buildRepo', null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.SINGULARITY, 'foo.com/repo', containerId, null, spackFile, null) - def req = - new BuildRequest( - containerId: containerId, - containerFile: dockerFile, - spackFile: spackFile, - isSpackBuild: true, - workspace: folder, - targetImage: targetImage, - identity: Mock(PlatformId), - platform: ContainerPlatform.of('amd64'), - format: BuildFormat.SINGULARITY, - startTime: Instant.now() - ) - - .withBuildId('1') - and: - def spack = Mock(SpackConfig) - - when: - def result = builder.containerFile0(req, context, spack) - then: - 1* spack.getCacheBucket() >> 's3://bucket/cache' - 1* spack.getSecretMountPath() >> '/mnt/key' - 1* spack.getBuilderImage() >> 'spack-builder:2.0' - 1* spack.getRunnerImage() >> 'ubuntu:22.04' - and: - result.contains('Bootstrap: docker\n' + - 'From: spack-builder:2.0\n' + - 'Stage: build') - result.contains('spack config add packages:all:target:[x86_64]') - result.contains('spack mirror add seqera-spack s3://bucket/cache') - result.contains('fingerprint="$(spack gpg trust /mnt/key 2>&1 | tee /dev/stderr | sed -nr "s/^gpg: key ([0-9A-F]{16}): secret key imported$/\\1/p")"') - result.contains('/some/context/dir/spack.yaml /opt/spack-env/spack.yaml') - - cleanup: - folder?.deleteDir() - } - def 'should replace context path' () { given: def folder = Path.of('/some/work/dir') @@ -306,8 +196,8 @@ class ContainerBuildServiceTest extends Specification { '''.stripIndent() and: def builder = new ContainerBuildServiceImpl() - def containerId = ContainerHelper.makeContainerId(containerFile, null, null, ContainerPlatform.of('amd64'), 'buildRepo', null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.SINGULARITY, 'foo.com/repo', containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(containerFile, null, ContainerPlatform.of('amd64'), 'buildRepo', null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.SINGULARITY, 'foo.com/repo', containerId, null, null) def req = new BuildRequest( containerId: containerId, @@ -322,7 +212,7 @@ class ContainerBuildServiceTest extends Specification { .withBuildId('1') when: - def result = builder.containerFile0(req, Path.of('/some/context/'), null) + def result = builder.containerFile0(req, Path.of('/some/context/')) then: result == '''\ BootStrap: docker @@ -386,8 +276,8 @@ class ContainerBuildServiceTest extends Specification { and: def dockerFile = 'from foo' def buildRepo = 'quay.io/org/name' - def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) + def containerId = ContainerHelper.makeContainerId(dockerFile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) def req = new BuildRequest( containerId: containerId, @@ -420,7 +310,6 @@ class ContainerBuildServiceTest extends Specification { containerId: 'container1234', containerFile: 'test', condaFile: 'test', - spackFile: 'test', workspace: Path.of("."), targetImage: 'docker.io/my/repo:container1234', identity: PlatformId.NULL, @@ -452,7 +341,6 @@ class ContainerBuildServiceTest extends Specification { containerId: 'container1234', containerFile:'test', condaFile: 'test', - spackFile: 'test', workspace: Path.of("."), targetImage: 'docker.io/my/repo:container1234', identity: PlatformId.NULL, @@ -488,7 +376,6 @@ class ContainerBuildServiceTest extends Specification { containerId: 'container1234', containerFile: 'test', condaFile: 'test', - spackFile: 'test', workspace: Path.of("."), targetImage: 'docker.io/my/repo:container1234', identity: PlatformId.NULL, diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index 3294f88a0..52c4c1f8c 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -24,7 +24,6 @@ import java.nio.file.Path import io.micronaut.context.ApplicationContext import io.micronaut.test.extensions.spock.annotation.MicronautTest -import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform /** * @@ -35,17 +34,13 @@ class DockerBuildStrategyTest extends Specification { def 'should get docker command' () { given: - def props = [ - 'wave.build.spack.secretKeyFile':'/host/spack/key', - 'wave.build.spack.secretMountPath':'/opt/spack/key' ] - def ctx = ApplicationContext.run(props) + def ctx = ApplicationContext.run() and: def service = ctx.getBean(DockerBuildStrategy) - def spackConfig = ctx.getBean(SpackConfig) and: def work = Path.of('/work/foo') when: - def cmd = service.cmdForBuildkit('build-job-name', work, null, null, null) + def cmd = service.cmdForBuildkit('build-job-name', work, null, null) then: cmd == ['docker', 'run', @@ -59,7 +54,7 @@ class DockerBuildStrategyTest extends Specification { 'moby/buildkit:v0.14.1-rootless'] when: - cmd = service.cmdForBuildkit('build-job-name', work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) + cmd = service.cmdForBuildkit('build-job-name', work, Path.of('/foo/creds.json'), ContainerPlatform.of('arm64')) then: cmd == ['docker', 'run', @@ -75,7 +70,7 @@ class DockerBuildStrategyTest extends Specification { 'moby/buildkit:v0.14.1-rootless'] when: - cmd = service.cmdForBuildkit('build-job-name', work, Path.of('/foo/creds.json'), spackConfig, null) + cmd = service.cmdForBuildkit('build-job-name', work, Path.of('/foo/creds.json'), null) then: cmd == ['docker', 'run', @@ -87,7 +82,6 @@ class DockerBuildStrategyTest extends Specification { '--entrypoint', 'buildctl-daemonless.sh', '-v', '/foo/creds.json:/home/user/.docker/config.json:ro', - '-v', '/host/spack/key:/opt/spack/key:ro', 'moby/buildkit:v0.14.1-rootless'] cleanup: @@ -146,13 +140,8 @@ class DockerBuildStrategyTest extends Specification { def 'should get singularity build command' () { given: - def props = [ - 'wave.build.spack.secretKeyFile':'/host/spack/key', - 'wave.build.spack.secretMountPath':'/opt/spack/key' ] - def ctx = ApplicationContext.run(props) + def ctx = ApplicationContext.run() def service = ctx.getBean(DockerBuildStrategy) - SpackConfig spackConfig = ctx.getBean(SpackConfig) - service.setSpackConfig(spackConfig) and: def creds = Path.of('/work/creds.json') and: @@ -161,8 +150,7 @@ class DockerBuildStrategyTest extends Specification { platform: ContainerPlatform.of('linux/amd64'), targetImage: 'oras://repo:d4869cc39b8d7d55', cacheRepository: 'reg.io/wave/build/cache', - format: BuildFormat.SINGULARITY, - isSpackBuild: true ) + format: BuildFormat.SINGULARITY ) when: def cmd = service.buildCmd('build-job-name', req, creds) then: @@ -176,27 +164,21 @@ class DockerBuildStrategyTest extends Specification { '-v', '/work/foo/d4869cc39b8d7d55:/work/foo/d4869cc39b8d7d55', '-v', '/work/creds.json:/root/.singularity/docker-config.json:ro', '-v', '/work/singularity-remote.yaml:/root/.singularity/remote.yaml:ro', - '-v', '/host/spack/key:/opt/spack/key:ro', '--platform', 'linux/amd64', 'quay.io/singularity/singularity:v3.11.4-slim', 'sh', '-c', 'singularity build image.sif /work/foo/d4869cc39b8d7d55/Containerfile && singularity push image.sif oras://repo:d4869cc39b8d7d55' ] - + cleanup: ctx.close() } def 'should get singularity build command for arm64 architecture' () { given: - def props = [ - 'wave.build.spack.secretKeyFile':'/host/spack/key', - 'wave.build.spack.secretMountPath':'/opt/spack/key' ] - def ctx = ApplicationContext.run(props) + def ctx = ApplicationContext.run() def service = ctx.getBean(DockerBuildStrategy) - SpackConfig spackConfig = ctx.getBean(SpackConfig) - service.setSpackConfig(spackConfig) and: def creds = Path.of('/work/creds.json') and: @@ -205,8 +187,7 @@ class DockerBuildStrategyTest extends Specification { platform: ContainerPlatform.of('linux/arm64'), targetImage: 'oras://repo:9c68af894bb2419c', cacheRepository: 'reg.io/wave/build/cache', - format: BuildFormat.SINGULARITY, - isSpackBuild: true ) + format: BuildFormat.SINGULARITY ) when: def cmd = service.buildCmd('build-job-name', req, creds) then: @@ -220,7 +201,6 @@ class DockerBuildStrategyTest extends Specification { '-v', '/work/foo/9c68af894bb2419c:/work/foo/9c68af894bb2419c', '-v', '/work/creds.json:/root/.singularity/docker-config.json:ro', '-v', '/work/singularity-remote.yaml:/root/.singularity/remote.yaml:ro', - '-v', '/host/spack/key:/opt/spack/key:ro', '--platform', 'linux/arm64', 'quay.io/singularity/singularity:v3.11.4-slim-arm64', 'sh', diff --git a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy index 957a32146..0904a7988 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy @@ -46,9 +46,9 @@ class FutureContainerBuildServiceTest extends Specification { RUN echo 'hello' > hello.txt """.stripIndent() and: - def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) + def req = new BuildRequest(containerId, dockerfile, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') def res = new BuildResult("", 0, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') and: def buildStore = Mock(BuildStore) @@ -81,9 +81,9 @@ class FutureContainerBuildServiceTest extends Specification { RUN echo 'hello' > hello.txt """.stripIndent() and: - def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), buildRepo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) + def req = new BuildRequest(containerId, dockerfile, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') def res = new BuildResult("", 1, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') and: def buildStore = Mock(BuildStore) diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 1b8ea51de..5e79c4824 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -68,22 +68,22 @@ class KubeBuildStrategyTest extends Specification { def dockerfile = 'from foo' when: - def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), repo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), repo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null) + def req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') Files.createDirectories(req.workDir) strategy.build('build-job-name', req) then: - 1 * k8sService.launchBuildJob(_, _, _, _, _, _, _, [service:'wave-build']) >> null + 1 * k8sService.launchBuildJob( _, _, _, _, _, _, [service:'wave-build']) >> null when: - def req2 = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def req2 = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') Files.createDirectories(req2.workDir) strategy.build('job-name', req2) then: - 1 * k8sService.launchBuildJob(_, _, _, _, _, _, _, [service:'wave-build-arm64']) >> null + 1 * k8sService.launchBuildJob( _, _, _, _, _, _, [service:'wave-build-arm64']) >> null } @@ -96,21 +96,21 @@ class KubeBuildStrategyTest extends Specification { def dockerfile = 'from foo' when:'getting docker with amd64 arch in build request' - def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), repo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), repo, null) + def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null) + def req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') then: 'should return buildkit image' strategy.getBuildImage(req) == 'moby/buildkit:v0.14.1-rootless' when:'getting singularity with amd64 arch in build request' - req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY,Duration.ofMinutes(1)).withBuildId('1') + req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY,Duration.ofMinutes(1)).withBuildId('1') then:'should return singularity amd64 image' strategy.getBuildImage(req) == 'quay.io/singularity/singularity:v3.11.4-slim' when:'getting singularity with arm64 arch in build request' - req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY, Duration.ofMinutes(1)).withBuildId('1') + req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY, Duration.ofMinutes(1)).withBuildId('1') then:'should return singularity arm64 image' strategy.getBuildImage(req) == 'quay.io/singularity/singularity:v3.11.4-slim-arm64' diff --git a/src/test/groovy/io/seqera/wave/service/cache/impl/AbstractCacheStoreTest.groovy b/src/test/groovy/io/seqera/wave/service/cache/impl/AbstractCacheStoreTest.groovy new file mode 100644 index 000000000..f89db42fd --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/cache/impl/AbstractCacheStoreTest.groovy @@ -0,0 +1,273 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.cache.impl + +import spock.lang.Specification + +import java.time.Duration + +import groovy.transform.Canonical +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.service.cache.AbstractCacheStore +import io.seqera.wave.service.cache.StateRecord +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class AbstractCacheStoreTest extends Specification { + + @Inject LocalCacheProvider provider + + static public long ttlMillis = 100 + + @Canonical + static class MyObject { + String field1 + String field2 + } + + static class MyState extends MyObject implements StateRecord { + + MyState(String field1, String field2) { + super(field1, field2) + } + + @Override + String getRecordId() { + return field1 + } + } + + static class MyCacheStore extends AbstractCacheStore { + + MyCacheStore(CacheProvider provider) { + super(provider, new MoshiEncodeStrategy() {}) + } + + @Override + protected String getPrefix() { + return 'test/v1:' + } + + @Override + protected Duration getDuration() { + return Duration.ofMillis(ttlMillis) + } + } + + def 'should get key' () { + given: + def store = new MyCacheStore(provider) + + expect: + store.key0('one') == 'test/v1:one' + } + + def 'should get record id' () { + given: + def store = new MyCacheStore(provider) + + expect: + store.recordId0('one') == 'test/v1:state-id/one' + } + + def 'should get and put a value' () { + given: + def store = new MyCacheStore(provider) + def key = UUID.randomUUID().toString() + + expect: + store.get(key) == null + + when: + store.put(key, new MyObject('this','that')) + then: + store.get(key) == new MyObject('this','that') + } + + def 'should get and put a value' () { + given: + def store = new MyCacheStore(provider) + def key = UUID.randomUUID().toString() + + expect: + store.get(key) == null + + when: + store.put(key, new MyObject('this','that')) + then: + store.get(key) == new MyObject('this','that') + + when: + sleep ttlMillis *2 + then: + store.get(key) == null + } + + def 'should get and put a value with custom ttl' () { + given: + def store = new MyCacheStore(provider) + def key = UUID.randomUUID().toString() + + expect: + store.get(key) == null + + when: + store.put(key, new MyObject('this','that'), Duration.ofSeconds(10)) + then: + store.get(key) == new MyObject('this','that') + + when: + sleep ttlMillis *2 + then: + store.get(key) == new MyObject('this','that') + } + + def 'should put and remove and item' () { + given: + def store = new MyCacheStore(provider) + def key = UUID.randomUUID().toString() + + when: + store.put(key, new MyObject('this','that'), Duration.ofSeconds(10)) + then: + store.get(key) == new MyObject('this','that') + + when: + store.remove(key) + then: + store.get(key) == null + } + + def 'should put if absent' () { + given: + def store = new MyCacheStore(provider) + def key = UUID.randomUUID().toString() + + when: + def done = store.putIfAbsent(key, new MyObject('this','that')) + then: + done + and: + store.get(key) == new MyObject('this','that') + + when: + done = store.putIfAbsent(key, new MyObject('xx','yy')) + then: + !done + and: + store.get(key) == new MyObject('this','that') + + when: + sleep ttlMillis*2 + done = store.putIfAbsent(key, new MyObject('xx','yy')) + then: + done + and: + store.get(key) == new MyObject('xx','yy') + + } + + def 'should put if absent with custom ttl' () { + given: + def store = new MyCacheStore(provider) + def key = UUID.randomUUID().toString() + + when: + def done = store.putIfAbsent(key, new MyObject('this','that'), Duration.ofSeconds(10)) + then: + done + and: + store.get(key) == new MyObject('this','that') + + when: + done = store.putIfAbsent(key, new MyObject('xx','yy')) + then: + !done + and: + store.get(key) == new MyObject('this','that') + + when: + sleep ttlMillis*2 + done = store.putIfAbsent(key, new MyObject('xx','yy')) + then: + !done + and: + store.get(key) == new MyObject('this','that') + + } + + + def 'should put and get value by record id' () { + given: + def store = new MyCacheStore(provider) + def recId = UUID.randomUUID().toString() + def key = UUID.randomUUID().toString() + + expect: + store.get(key) == null + store.getByRecordId(recId) == null + + when: + def value = new MyState(recId, 'value') + store.put(key, value) + then: + store.get(key) == value + store.getByRecordId(recId) == value + and: + store.get(recId) == null + store.getByRecordId(key) == null + } + + + def 'should put and get value by record id if absent' () { + given: + def store = new MyCacheStore(provider) + def recId = UUID.randomUUID().toString() + def key = UUID.randomUUID().toString() + + expect: + store.get(key) == null + store.getByRecordId(recId) == null + + when: + def value = new MyState(recId, 'value') + def done = store.putIfAbsent(key, value) + then: + done + and: + store.get(key) == value + store.getByRecordId(recId) == value + and: + store.get(recId) == null + store.getByRecordId(key) == null + + when: + done = store.putIfAbsent(key, new MyState('xx', 'yy')) + then: + !done + and: + store.get(key) == value + store.getByRecordId(recId) == value + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/cache/impl/LocalCacheProviderTest.groovy b/src/test/groovy/io/seqera/wave/service/cache/impl/LocalCacheProviderTest.groovy index 5c2a14956..6b05d76f5 100644 --- a/src/test/groovy/io/seqera/wave/service/cache/impl/LocalCacheProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/cache/impl/LocalCacheProviderTest.groovy @@ -32,93 +32,102 @@ class LocalCacheProviderTest extends Specification { LocalCacheProvider localCacheProvider - def 'conditional put with current value when ke is not set'() { - when: 'conditionally set a key that has no current value' - def current = localCacheProvider.putIfAbsentAndGetCurrent('key', 'new-value', Duration.ofMillis(Long.MAX_VALUE)) + def 'should get and put a key-value pair' () { + given: + def k = UUID.randomUUID().toString() - then: 'the provided value is returned' - current == 'new-value' + expect: + localCacheProvider.get(k) == null - and: 'the value is set in the store' - localCacheProvider.get('key') == 'new-value' - when: - def other = localCacheProvider.putIfAbsentAndGetCurrent('key', 'hola', Duration.ofMillis(Long.MAX_VALUE)) + localCacheProvider.put(k, "hello") then: - // should not be set because it already exists - other == 'new-value' + localCacheProvider.get(k) == 'hello' } - def 'conditional put with current value when key is already set'() { - given: 'a store containing a mapping for key that is not expired' - localCacheProvider.put('key','existing', Duration.ofMillis(Long.MAX_VALUE)) + def 'should get and put a key-value pair with ttl' () { + given: + def TTL = 100 + def k = UUID.randomUUID().toString() - when: 'try to conditionally set the key to a new value' - def current = localCacheProvider.putIfAbsentAndGetCurrent('key', 'new-value', Duration.ofMillis(Long.MAX_VALUE)) + expect: + localCacheProvider.get(k) == null - then: 'the existing value is returned' - current == 'existing' - - and: 'the value is not updated in the store' - localCacheProvider.get('key') == 'existing' + when: + localCacheProvider.put(k, "hello", Duration.ofMillis(TTL)) + then: + localCacheProvider.get(k) == 'hello' + then: + sleep(TTL *2) + and: + localCacheProvider.get(k) == null } + def 'should get and put only if absent' () { + given: + def k = UUID.randomUUID().toString() - def 'conditional put with current value when key is set and has expired'() { - given: 'a store containing a mapping for key that will expire' - localCacheProvider.put('key', 'existing', Duration.ofMillis(100)) - // give time for cache store to expire the key - sleep(Duration.ofMillis(200).toMillis()) - - when: 'try to conditionally set the key to a new value' - def current = localCacheProvider.putIfAbsentAndGetCurrent('key', 'new-value', Duration.ofMillis(100)) + expect: + localCacheProvider.get(k) == null - then: 'the provided value is returned' - current == 'new-value' + when: + def done = localCacheProvider.putIfAbsent(k, 'foo') + then: + done + and: + localCacheProvider.get(k) == 'foo' - and: 'the value is updated is set in the store' - localCacheProvider.get('key') == 'new-value' + when: + done = localCacheProvider.putIfAbsent(k, 'bar') + then: + !done + and: + localCacheProvider.get(k) == 'foo' } - def 'should add and find keys for values' () { - when: - localCacheProvider.biPut('x1', 'a', Duration.ofMinutes(1)) - localCacheProvider.biPut('x2', 'b', Duration.ofMinutes(1)) - localCacheProvider.biPut('x3', 'a', Duration.ofMinutes(1)) - localCacheProvider.biPut('x4', 'c', Duration.ofMinutes(1)) + def 'should get and put if absent with ttl' () { + given: + def TTL = 100 + def k = UUID.randomUUID().toString() + when: + def done = localCacheProvider.putIfAbsent(k, 'foo', Duration.ofMillis(TTL)) then: - localCacheProvider.biKeysFor('a') == ['x1', 'x3'] as Set - localCacheProvider.biKeysFor('c') == ['x4'] as Set - localCacheProvider.biKeysFor('d') == [] as Set + done + and: + localCacheProvider.get(k) == 'foo' when: - localCacheProvider.biRemove('x1') + done = localCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) then: - localCacheProvider.biKeysFor('a') == ['x3'] as Set + !done + and: + localCacheProvider.get(k) == 'foo' when: - localCacheProvider.biRemove('x3') + sleep(TTL *2) + and: + done = localCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) then: - localCacheProvider.biKeysFor('a') == [] as Set + done + and: + localCacheProvider.get(k) == 'bar' } - def 'should add and find keys for values' () { - when: - localCacheProvider.biPut('x1', 'a', Duration.ofMillis(100)) - localCacheProvider.biPut('x2', 'b', Duration.ofMinutes(1)) - localCacheProvider.biPut('x3', 'a', Duration.ofMinutes(1)) - localCacheProvider.biPut('x4', 'c', Duration.ofMinutes(1)) + def 'should put and remove a value' () { + given: + def TTL = 100 + def k = UUID.randomUUID().toString() + when: + localCacheProvider.put(k, 'foo') then: - localCacheProvider.biKeyFind('a', true) == 'x1' - and: - localCacheProvider.biKeysFor('a') == ['x1','x3'] as Set - and: - sleep 500 - and: - localCacheProvider.biKeyFind('a', true) == 'x3' - localCacheProvider.biKeysFor('a') == ['x3'] as Set + localCacheProvider.get(k) == 'foo' + when: + localCacheProvider.remove(k) + then: + localCacheProvider.get(k) == null } + } diff --git a/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy b/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy index 79c679b7e..e11c4b7a2 100644 --- a/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/cache/impl/RedisCacheProviderTest.groovy @@ -47,112 +47,101 @@ class RedisCacheProviderTest extends Specification implements RedisTestContainer applicationContext.close() } - def 'conditional put with current value when ke is not set'() { - when: 'conditionally set a key that has no current value' - def current = redisCacheProvider.putIfAbsentAndGetCurrent('key', 'new-value', Duration.ofSeconds(100)) + def 'should get and put a key-value pair' () { + given: + def k = UUID.randomUUID().toString() - then: 'the provided value is returned' - current == 'new-value' + expect: + redisCacheProvider.get(k) == null - and: 'the value is set in the store' - redisCacheProvider.get('key') == 'new-value' + when: + redisCacheProvider.put(k, "hello") + then: + redisCacheProvider.get(k) == 'hello' } - def 'conditional put with current value when key is already set'() { - given: 'a store containing a mapping for key that is not expired' - redisCacheProvider.put('key', 'existing', Duration.ofSeconds(100)) - - when: 'try to conditionally set the key to a new value' - def current = redisCacheProvider.putIfAbsentAndGetCurrent('key', 'new-value', Duration.ofSeconds(100)) - - then: 'the existing value is returned' - current == 'existing' + def 'should get and put a key-value pair with ttl' () { + given: + def TTL = 100 + def k = UUID.randomUUID().toString() - and: 'the value is not updated in the store' - redisCacheProvider.get('key') == 'existing' + expect: + redisCacheProvider.get(k) == null + when: + redisCacheProvider.put(k, "hello", Duration.ofMillis(TTL)) + then: + redisCacheProvider.get(k) == 'hello' + then: + sleep(TTL *2) + and: + redisCacheProvider.get(k) == null } - def 'conditional put with current value when key is set and has expired'() { - given: 'a store containing a mapping for key that will expire' - redisCacheProvider.put('key', 'existing', Duration.ofSeconds(1)) - // give time for redis to expire the key - sleep(Duration.ofSeconds(2).toMillis()) - - when: 'try to conditionally set the key to a new value' - def current = redisCacheProvider.putIfAbsentAndGetCurrent('key', 'new-value', Duration.ofSeconds(100)) + def 'should get and put only if absent' () { + given: + def k = UUID.randomUUID().toString() - then: 'the provided value is returned' - current == 'new-value' - - and: 'the value is updated is set in the store' - redisCacheProvider.get('key') == 'new-value' - } + expect: + redisCacheProvider.get(k) == null - def 'should add and find keys for values' () { when: - redisCacheProvider.biPut('x1', 'a', Duration.ofMinutes(1)) - redisCacheProvider.biPut('x2', 'b', Duration.ofMinutes(1)) - redisCacheProvider.biPut('x3', 'a', Duration.ofMinutes(1)) - redisCacheProvider.biPut('x4', 'c', Duration.ofMinutes(1)) - + def done = redisCacheProvider.putIfAbsent(k, 'foo') then: - redisCacheProvider.biKeysFor('a') == ['x1', 'x3'] as Set - redisCacheProvider.biKeysFor('c') == ['x4'] as Set - redisCacheProvider.biKeysFor('d') == [] as Set + done + and: + redisCacheProvider.get(k) == 'foo' when: - redisCacheProvider.biRemove('x1') + done = redisCacheProvider.putIfAbsent(k, 'bar') then: - redisCacheProvider.biKeysFor('a') == ['x3'] as Set + !done + and: + redisCacheProvider.get(k) == 'foo' + } + + def 'should get and put if absent with ttl' () { + given: + def TTL = 100 + def k = UUID.randomUUID().toString() when: - redisCacheProvider.biRemove('x3') + def done = redisCacheProvider.putIfAbsent(k, 'foo', Duration.ofMillis(TTL)) then: - redisCacheProvider.biKeysFor('a') == [] as Set - - cleanup: - redisCacheProvider.clear() - } + done + and: + redisCacheProvider.get(k) == 'foo' - def 'should add and find single key for value' () { when: - redisCacheProvider.biPut('x1', 'a', Duration.ofSeconds(1)) - redisCacheProvider.biPut('x2', 'b', Duration.ofMinutes(1)) - redisCacheProvider.biPut('x3', 'a', Duration.ofMinutes(1)) - redisCacheProvider.biPut('x4', 'c', Duration.ofMinutes(1)) - + done = redisCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) then: - redisCacheProvider.biKeyFind('a', true) == 'x1' + !done and: - redisCacheProvider.biKeysFor('a') == ['x1','x3'] as Set + redisCacheProvider.get(k) == 'foo' + + when: + sleep(TTL *2) and: - sleep 1500 + done = redisCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) + then: + done and: - redisCacheProvider.biKeyFind('a', true) == 'x3' - redisCacheProvider.biKeysFor('a') == ['x3'] as Set - - cleanup: - redisCacheProvider.clear() + redisCacheProvider.get(k) == 'bar' } - def 'should update expiration when re-putting the value' () { + def 'should put and remove a value' () { + given: + def TTL = 100 + def k = UUID.randomUUID().toString() + when: - redisCacheProvider.biPut('x1', 'a', Duration.ofSeconds(1)) + redisCacheProvider.put(k, 'foo') then: - redisCacheProvider.biKeyFind('a', true) == 'x1' + redisCacheProvider.get(k) == 'foo' when: - sleep 500 - redisCacheProvider.biPut('x1', 'a', Duration.ofSeconds(1)) - sleep 500 - redisCacheProvider.biPut('x1', 'a', Duration.ofSeconds(1)) - sleep 500 + redisCacheProvider.remove(k) then: - redisCacheProvider.biKeyFind('a', true) == 'x1' - - cleanup: - redisCacheProvider.clear() + redisCacheProvider.get(k) == null } - } diff --git a/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy index 2fa094560..633ca6814 100644 --- a/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy @@ -28,6 +28,8 @@ import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest /** @@ -100,4 +102,30 @@ class JobFactoryTest extends Specification { job.creationTime == request.creationTime job.workDir == workdir } + + def 'should create mirror job' () { + given: + def workspace = Path.of('/some/work/dir') + def duration = Duration.ofMinutes(1) + def config = new MirrorConfig(maxDuration: duration) + def factory = new JobFactory(mirrorConfig: config) + and: + def request = MirrorRequest.create( + 'source/foo', + 'target/foo', + 'sha256:12345', + Mock(ContainerPlatform), + workspace, + '{config}' ) + + when: + def job = factory.mirror(request) + then: + job.recordId == "target/foo" + job.operationName == /mirror-${request.id.substring(3)}/ + job.type == JobSpec.Type.Mirror + job.maxDuration == duration + job.workDir == workspace.resolve(/mirror-${request.id.substring(3)}/) + job.creationTime == request.creationTime + } } diff --git a/src/test/groovy/io/seqera/wave/service/job/JobSpecTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobSpecTest.groovy index 507ca9712..a4e657db8 100644 --- a/src/test/groovy/io/seqera/wave/service/job/JobSpecTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobSpecTest.groovy @@ -93,4 +93,19 @@ class JobSpecTest extends Specification { job.operationName == 'xyz' job.workDir == Path.of('/some/path') } + + def 'should create mirror job' () { + given: + def now = Instant.now() + def job = JobSpec.mirror('12345','xyz', now, Duration.ofMinutes(1), Path.of('/some/path')) + + expect: + job.id + job.recordId == '12345' + job.type == JobSpec.Type.Mirror + job.creationTime == now + job.maxDuration == Duration.ofMinutes(1) + job.operationName == 'xyz' + job.workDir == Path.of('/some/path') + } } diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy deleted file mode 100644 index 27ef124ac..000000000 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.k8s - -import spock.lang.Ignore -import spock.lang.Specification - -import java.nio.file.Path - -import io.micronaut.test.extensions.spock.annotation.MicronautTest -import jakarta.inject.Inject -/** - * - * @author Paolo Di Tommaso - */ -@Ignore -@MicronautTest -class K8sClientTest extends Specification { - - @Inject K8sService k8sService - - def 'should create job' () { - when: - def job = k8sService.createJob('foo-2', 'busybox', ['sh', '-c', 'slep 10']) - println job - then: - job - } - - def 'should get job' () { - when: - def job = k8sService.getJob('foo-2') - job.status.succeeded == 1 - then: - true - } - - def 'should create pod' () { - when: - def pod = k8sService.buildContainer( - 'my-pod', - 'busybox', - ['cat','/home/user/.docker/config.json'], - Path.of('/work/dir'), - Path.of('/creds'), - Path.of('/spack/dir'), - ['my-creds': 'selector']) - then: - true - - when: - def str = k8sService.logsPod('my-pod') - then: - str - - } -} diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 3f85ebb63..5ce208064 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -42,7 +42,7 @@ import io.micronaut.context.annotation.Replaces import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.configuration.SpackConfig +import io.seqera.wave.service.mirror.MirrorConfig /** * * @author Paolo Di Tommaso @@ -188,30 +188,6 @@ class K8sServiceImplTest extends Specification { ctx.close() } - def 'should get spack dir vol' () { - given: - def PROPS = [ - 'wave.build.workspace': '/build/work', - 'wave.build.k8s.namespace': 'foo', - 'wave.build.k8s.configPath': '/home/kube.config', - 'wave.build.k8s.storage.claimName': 'bar', - 'wave.build.k8s.storage.mountPath': '/build' ] - and: - def ctx = ApplicationContext.run(PROPS) - def k8sService = ctx.getBean(K8sServiceImpl) - - when: - def mount = k8sService.mountSpackCacheDir(Path.of('/foo/work/x1'), '/foo', '/opt/spack/cache') - then: - mount.name == 'build-data' - mount.mountPath == '/opt/spack/cache' - mount.subPath == 'work/x1' - !mount.readOnly - - cleanup: - ctx.close() - } - def 'should create build pod for buildkit' () { given: def PROPS = [ @@ -225,7 +201,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this', 'that'], Path.of('/build/work/xyz'), Path.of('/build/work/xyz/config.json'), Duration.ofSeconds(10), null, [:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this', 'that'], Path.of('/build/work/xyz'), Path.of('/build/work/xyz/config.json'), Duration.ofSeconds(10), [:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -268,7 +244,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) def workDir = Path.of('/build/work/xyz') when: - def result = k8sService.buildSpec('foo', 'singularity:latest', ['this','that'], workDir, workDir.resolve('config.json'), Duration.ofSeconds(10), null, [:]) + def result = k8sService.buildSpec('foo', 'singularity:latest', ['this','that'], workDir, workDir.resolve('config.json'), Duration.ofSeconds(10), [:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -302,52 +278,6 @@ class K8sServiceImplTest extends Specification { ctx.close() } - def 'should create build pod with spack cache' () { - given: - def PROPS = [ - 'wave.build.workspace': '/build/work', - 'wave.build.k8s.namespace': 'my-ns', - 'wave.build.k8s.configPath': '/home/kube.config', - 'wave.build.k8s.storage.claimName': 'build-claim', - 'wave.build.k8s.storage.mountPath': '/build', - 'wave.build.spack.secretKeyFile':'/build/host/spack/key', - 'wave.build.spack.secretMountPath':'/opt/container/spack/key' - ] - and: - def ctx = ApplicationContext.run(PROPS) - def k8sService = ctx.getBean(K8sServiceImpl) - def spackConfig = ctx.getBean(SpackConfig) - when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null,Duration.ofSeconds(10), spackConfig, [:]) - then: - result.metadata.name == 'foo' - result.metadata.namespace == 'my-ns' - and: - result.spec.activeDeadlineSeconds == 10 - and: - verifyAll(result.spec.containers.get(0)) { - name == 'foo' - image == 'my-image:latest' - args == ['this', 'that'] - env.name == ['BUILDKITD_FLAGS'] - env.value == ['--oci-worker-no-process-sandbox'] - volumeMounts.size() == 2 - volumeMounts.get(0).name == 'build-data' - volumeMounts.get(0).mountPath == '/build/work/xyz' - volumeMounts.get(0).subPath == 'work/xyz' - volumeMounts.get(1).name == 'build-data' - volumeMounts.get(1).mountPath == '/opt/container/spack/key' - volumeMounts.get(1).subPath == 'host/spack/key' - volumeMounts.get(1).readOnly - } - and: - result.spec.volumes.get(0).name == 'build-data' - result.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' - - cleanup: - ctx.close() - } - def 'should create build pod without init container' () { given: def PROPS = [ @@ -361,7 +291,8 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), [:]) + then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -405,7 +336,8 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), [:]) + then: result.metadata.name == 'foo' result.metadata.labels.toString() == PROPS['wave.build.k8s.labels'].toString() @@ -433,7 +365,8 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null, PROPS['wave.build.k8s.node-selector'] as Map) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), PROPS['wave.build.k8s.node-selector'] as Map) + then: result.spec.nodeSelector.toString() == PROPS['wave.build.k8s.node-selector'].toString() and: @@ -458,7 +391,8 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), [:]) + then: result.spec.serviceAccount == PROPS['wave.build.k8s.service-account'] and: @@ -695,34 +629,49 @@ class K8sServiceImplTest extends Specification { given: def PROPS = [ 'wave.build.workspace': '/build/work', - 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.namespace': 'my-ns', 'wave.build.k8s.configPath': '/home/kube.config', - 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.claimName': 'build-claim', 'wave.build.k8s.storage.mountPath': '/build', - 'wave.build.k8s.service-account': 'theAdminAccount', - 'wave.build.deleteAfterFinished': '1d', 'wave.build.retry-attempts': 3 ] and: def ctx = ApplicationContext.run(PROPS) def k8sService = ctx.getBean(K8sServiceImpl) - def name = 'test-job' - def containerImage = 'singularity://test-image' - def args = ['arg1', 'arg2'] - def workDir = Path.of('/work/dir') - def credsFile = Path.of('/creds/file') + def name = 'the-job-name' + def containerImage = 'singularity:latest' + def args = ['singularity', '--this', '--that'] + def workDir = Path.of('/build/work/xyz') + def credsFile = workDir.resolve('config.json') def timeout = Duration.ofMinutes(10) - def spackConfig = new SpackConfig(secretKeyFile: Path.of('/build/secret/key'), secretMountPath: '/secret/mount') def nodeSelector = [key: 'value'] when: - def job = k8sService.buildJobSpec(name, containerImage, args, workDir, credsFile, timeout, spackConfig, nodeSelector) + def job = k8sService.buildJobSpec(name, containerImage, args, workDir, credsFile, timeout, nodeSelector) then: job.spec.backoffLimit == 3 job.spec.template.spec.containers[0].image == containerImage job.spec.template.spec.containers[0].command == args job.spec.template.spec.containers[0].securityContext.privileged + and: + job.spec.template.spec.containers.get(0).getWorkingDir() == '/tmp' + and: + job.spec.template.spec.containers.get(0).volumeMounts.size() == 3 + job.spec.template.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' + job.spec.template.spec.containers.get(0).volumeMounts.get(0).mountPath == '/root/.singularity/docker-config.json' + job.spec.template.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz/config.json' + and: + job.spec.template.spec.containers.get(0).volumeMounts.get(1).name == 'build-data' + job.spec.template.spec.containers.get(0).volumeMounts.get(1).mountPath == '/root/.singularity/remote.yaml' + job.spec.template.spec.containers.get(0).volumeMounts.get(1).subPath == 'work/xyz/singularity-remote.yaml' + and: + job.spec.template.spec.containers.get(0).volumeMounts.get(2).name == 'build-data' + job.spec.template.spec.containers.get(0).volumeMounts.get(2).mountPath == '/build/work/xyz' + job.spec.template.spec.containers.get(0).volumeMounts.get(2).subPath == 'work/xyz' + and: + job.spec.template.spec.volumes.get(0).name == 'build-data' + job.spec.template.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' cleanup: ctx.close() @@ -732,11 +681,11 @@ class K8sServiceImplTest extends Specification { given: def PROPS = [ 'wave.build.workspace': '/build/work', - 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.namespace': 'my-ns', 'wave.build.k8s.configPath': '/home/kube.config', - 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.claimName': 'build-claim', 'wave.build.k8s.storage.mountPath': '/build', - 'wave.build.k8s.service-account': 'theAdminAccount' + 'wave.build.retry-attempts': 3 ] and: def ctx = ApplicationContext.run(PROPS) @@ -744,14 +693,13 @@ class K8sServiceImplTest extends Specification { def name = 'test-job' def containerImage = 'docker://test-image' def args = ['arg1', 'arg2'] - def workDir = Path.of('/work/dir') - def credsFile = Path.of('/creds/file') + def workDir = Path.of('/build/work/xyz') + def credsFile = workDir.resolve('config.json') def timeout = Duration.ofMinutes(10) - def spackConfig = new SpackConfig(secretKeyFile: Path.of('/build/secret/key'), secretMountPath: '/secret/mount') def nodeSelector = [key: 'value'] when: - def job = k8sService.buildJobSpec(name, containerImage, args, workDir, credsFile, timeout, spackConfig, nodeSelector) + def job = k8sService.buildJobSpec(name, containerImage, args, workDir, credsFile, timeout, nodeSelector) then: job.spec.template.spec.containers[0].image == containerImage @@ -759,6 +707,21 @@ class K8sServiceImplTest extends Specification { job.spec.template.spec.containers[0].command == ['buildctl-daemonless.sh'] job.spec.template.spec.containers[0].args == args + and: + job.spec.template.spec.containers.get(0).getWorkingDir() == '/tmp' + and: + job.spec.template.spec.containers.get(0).volumeMounts.size() == 2 + job.spec.template.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' + job.spec.template.spec.containers.get(0).volumeMounts.get(0).mountPath == '/home/user/.docker/config.json' + job.spec.template.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz/config.json' + and: + job.spec.template.spec.containers.get(0).volumeMounts.get(1).name == 'build-data' + job.spec.template.spec.containers.get(0).volumeMounts.get(1).mountPath == '/build/work/xyz' + job.spec.template.spec.containers.get(0).volumeMounts.get(1).subPath == 'work/xyz' + and: + job.spec.template.spec.volumes.get(0).name == 'build-data' + job.spec.template.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' + cleanup: ctx.close() } @@ -897,6 +860,65 @@ class K8sServiceImplTest extends Specification { ctx.close() } + def 'should create mirror job spec'() { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.k8s.service-account': 'theAdminAccount', + 'wave.mirror.retry-attempts': 3 + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def name = 'scan-job' + def containerImage = 'scan-image:latest' + def args = ['arg1', 'arg2'] + def workDir = Path.of('/build/work/dir') + def credsFile = Path.of('/build/work/dir/creds/file') + def mirrorConfig = Mock(MirrorConfig) { + getRequestsCpu() >> null + getRequestsMemory() >> null + getRetryAttempts() >> 3 + } + + when: + def job = k8sService.mirrorJobSpec(name, containerImage, args, workDir, credsFile, mirrorConfig) + + then: + job.metadata.name == name + job.metadata.namespace == 'foo' + job.spec.backoffLimit == 3 + job.spec.template.spec.containers[0].image == containerImage + job.spec.template.spec.containers[0].args == args + job.spec.template.spec.containers[0].resources.requests == null + job.spec.template.spec.containers[0].env == [new V1EnvVar().name('REGISTRY_AUTH_FILE').value('/tmp/config.json')] + and: + job.spec.template.spec.containers[0].volumeMounts.size() == 2 + and: + with(job.spec.template.spec.containers[0].volumeMounts[0]) { + mountPath == '/tmp/config.json' + readOnly == true + subPath == 'work/dir/creds/file' + } + and: + with(job.spec.template.spec.containers[0].volumeMounts[1]) { + mountPath == '/build/work/dir' + readOnly == true + subPath == 'work/dir' + } + and: + job.spec.template.spec.volumes.size() == 1 + job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar' + job.spec.template.spec.restartPolicy == 'Never' + + cleanup: + ctx.close() + } + def 'should create scan job spec without resource requests'() { given: def PROPS = [ diff --git a/src/test/groovy/io/seqera/wave/service/mail/MailServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/mail/MailServiceImplTest.groovy index e29034537..bd082d6fe 100644 --- a/src/test/groovy/io/seqera/wave/service/mail/MailServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mail/MailServiceImplTest.groovy @@ -50,13 +50,11 @@ class MailServiceImplTest extends Specification { 1* request.getTargetImage() >> 'wave/build:xyz' 1* request.getPlatform() >> ContainerPlatform.DEFAULT 1* request.getCondaFile() >> null - 1* request.getSpackFile() >> null and: mail.to == recipient mail.body.contains('from foo') and: !mail.body.contains('Conda file') - !mail.body.contains('Spack file') // check it adds the Conda file content when: @@ -70,17 +68,6 @@ class MailServiceImplTest extends Specification { mail.body.contains('Conda file') mail.body.contains('bioconda::foo') - // check it add the spack file content - when: - mail = service.buildCompletionMail(request, result, recipient) - then: - 1* request.getTargetImage() >> 'wave/build:xyz' - 1* request.getPlatform() >> ContainerPlatform.DEFAULT - 1* request.getSpackFile() >> 'some-spac-recipe' - and: - mail.to == recipient - mail.body.contains('Spack file') - mail.body.contains('some-spac-recipe') } } diff --git a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy new file mode 100644 index 000000000..4664c05aa --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy @@ -0,0 +1,205 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import spock.lang.Requires +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration +import java.time.Instant +import java.util.concurrent.TimeUnit + +import groovy.util.logging.Slf4j +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.inspect.ContainerInspectService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.tower.PlatformId +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@Slf4j +@MicronautTest +class ContainerMirrorServiceTest extends Specification { + + @Inject + ContainerMirrorServiceImpl mirrorService + + @Inject + MirrorStateStore mirrorStateStore + + @Inject + PersistenceService persistenceService + + @Inject + ContainerInspectService dockerAuthService + + @Requires({System.getenv('DOCKER_USER') && System.getenv('DOCKER_PAT')}) + def 'should mirror a container' () { + given: + def source = 'docker.io/hello-world:latest' + def target = 'docker.io/pditommaso/wave-tests' + def folder = Files.createTempDirectory('test') + println "Temp path: $folder" + when: + def creds = dockerAuthService.credentialsConfigJson(null, source, target, Mock(PlatformId)) + def request = MirrorRequest.create( + source, + target, + 'sha256:12345', + ContainerPlatform.DEFAULT, + folder, + creds ) + and: + mirrorService.mirrorImage(request) + then: + mirrorService.awaitCompletion(target) + .get(90, TimeUnit.SECONDS) + .succeeded() + + cleanup: + folder?.deleteDir() + } + + def 'should get mirror result from state store' () { + given: + def request = MirrorRequest.create( + 'source/foo', + 'target/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/some/dir'), + '{config}' ) + and: + def state = MirrorState.from(request) + and: + persistenceService.saveMirrorState(state) + when: + def copy = mirrorService.getMirrorState(request.id) + then: + copy == state + } + + def 'should get mirror result from persistent service' () { + given: + def request = MirrorRequest.create( + 'source/foo', + 'target/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/some/dir'), + '{config}' ) + and: + def state = MirrorState.from(request) + and: + mirrorStateStore.put('target/foo', state) + when: + def copy = mirrorService.getMirrorState(request.id) + then: + copy == state + } + + def 'should update mirror state on job completion' () { + given: + def request = MirrorRequest.create( + 'source/foo', + 'target/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/some/dir'), + '{config}' ) + and: + def state = MirrorState.from(request) + def job = JobSpec.mirror(request.id, 'mirror-123', Instant.now(), Duration.ofMillis(1), Mock(Path)) + when: + mirrorService.onJobCompletion(job, state, new JobState(JobState.Status.SUCCEEDED, 0, 'OK')) + then: + def s1 = mirrorStateStore.get(request.targetImage) + and: + s1.done() + s1.succeeded() + s1.exitCode == 0 + s1.logs == 'OK' + and: + def s2 = persistenceService.loadMirrorState(request.id) + and: + s2 == s1 + } + + def 'should update mirror state on job exception' () { + given: + def request = MirrorRequest.create( + 'source/foo', + 'target/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/some/dir'), + '{config}' ) + and: + def state = MirrorState.from(request) + def job = JobSpec.mirror(request.id, 'mirror-123', Instant.now(), Duration.ofMillis(1), Mock(Path)) + when: + mirrorService.onJobException(job, state, new Exception('Oops something went wrong')) + then: + def s1 = mirrorStateStore.get(request.targetImage) + and: + s1.done() + !s1.succeeded() + s1.exitCode == null + s1.logs == 'Oops something went wrong' + and: + def s2 = persistenceService.loadMirrorState(request.id) + and: + s2 == s1 + } + + def 'should update mirror state on job timeout' () { + given: + def request = MirrorRequest.create( + 'source/foo', + 'target/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/some/dir'), + '{config}' ) + and: + def state = MirrorState.from(request) + def job = JobSpec.mirror(request.id, 'mirror-123', Instant.now(), Duration.ofMillis(1), Mock(Path)) + when: + mirrorService.onJobTimeout(job, state) + then: + def s1 = mirrorStateStore.get(request.targetImage) + and: + s1.done() + !s1.succeeded() + s1.exitCode == null + s1.logs == 'Container mirror timed out' + and: + def s2 = persistenceService.loadMirrorState(request.id) + and: + s2 == s1 + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/mirror/MirrorRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/MirrorRequestTest.groovy new file mode 100644 index 000000000..be2e54727 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/mirror/MirrorRequestTest.groovy @@ -0,0 +1,56 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Instant + +import io.seqera.wave.core.ContainerPlatform + +/** + * + * @author Paolo Di Tommaso + */ +class MirrorRequestTest extends Specification { + + def 'should create mirror request' () { + when: + def ts = Instant.now() + def req = MirrorRequest.create( + 'docker.io/foo:latest', + 'quay.io/foo:latest', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/workspace'), + '{json config}') + then: + req.id + req.sourceImage == 'docker.io/foo:latest' + req.targetImage == 'quay.io/foo:latest' + req.digest == 'sha256:12345' + req.platform == ContainerPlatform.DEFAULT + req.workDir == Path.of("/workspace/mirror-${req.id.substring(3)}") + req.authJson == '{json config}' + req.creationTime >= ts + req.creationTime <= Instant.now() + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/mirror/MirrorResultTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/MirrorResultTest.groovy new file mode 100644 index 000000000..33c8eff16 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/mirror/MirrorResultTest.groovy @@ -0,0 +1,121 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Instant + +import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.core.ContainerPlatform +/** + * + * @author Paolo Di Tommaso + */ +class MirrorResultTest extends Specification { + + def 'should create a result from a request' () { + given: + def request = MirrorRequest.create( + 'source.io/foo', + 'target.io/foo', + 'sha256:12345', + Mock(ContainerPlatform), + Path.of('/workspace'), + '{auth json}' ) + + when: + def result = MirrorState.from(request) + then: + result.mirrorId == request.id + result.digest == request.digest + result.platform == request.platform + result.sourceImage == request.sourceImage + result.targetImage == request.targetImage + result.creationTime == request.creationTime + result.status == MirrorState.Status.PENDING + and: + result.duration == null + result.exitCode == null + result.logs == null + } + + def 'should complete a result result' () { + given: + def request = MirrorRequest.create( + 'source.io/foo', + 'target.io/foo', + 'sha256:12345', + Mock(ContainerPlatform), + Path.of('/workspace'), + '{auth json}' ) + + when: + def m1 = MirrorState.from(request) + then: + m1.status == MirrorState.Status.PENDING + m1.duration == null + m1.exitCode == null + m1.logs == null + + when: + def m2 = m1.complete(0, 'Some logs') + then: + m2.mirrorId == request.id + m2.digest == request.digest + m2.sourceImage == request.sourceImage + m2.targetImage == request.targetImage + m2.creationTime == request.creationTime + m2.platform == request.platform + and: + m2.status == MirrorState.Status.COMPLETED + m2.duration != null + m2.exitCode == 0 + m2.logs == 'Some logs' + } + + def 'should convert to status response' () { + when: + def result1 = new MirrorState('mr-123', 'sha256:12345', 'source/foo', 'target/foo', Mock(ContainerPlatform), Instant.now()) + def resp = result1.toStatusResponse() + then: + resp.id == result1.mirrorId + resp.status == BuildStatusResponse.Status.PENDING + resp.startTime == result1.creationTime + and: + resp.duration == null + resp.succeeded == null + + when: + final result2 = result1.complete(1, 'Some error') + final resp2 = result2.toStatusResponse() + then: + resp2.duration == result2.duration + resp2.succeeded == false + + when: + final result3 = result1.complete(0, 'OK') + final resp3 = result3.toStatusResponse() + then: + resp3.duration == result3.duration + resp3.succeeded == true + + } +} diff --git a/src/test/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategyTest.groovy new file mode 100644 index 000000000..1e0bf8d35 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategyTest.groovy @@ -0,0 +1,53 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror.strategy + +import spock.lang.Specification + +import java.nio.file.Path + +import io.seqera.wave.service.mirror.MirrorConfig + +/** + * + * @author Paolo Di Tommaso + */ +class DockerMirrorStrategyTest extends Specification { + + def 'should build docker command' () { + given: + def config = new MirrorConfig(skopeoImage: 'skopeo:latest') + def strategy = new DockerMirrorStrategy(mirrorConfig: config) + + when: + def result = strategy.mirrorCmd('foo', Path.of('/work/dir'), Path.of('/work/dir/creds.json')) + then: + result == ['docker', + 'run', + '--detach', + '--name', 'foo', + '-v', '/work/dir:/work/dir', + '-v', '/work/dir/creds.json:/tmp/config.json:ro', + '-e', 'REGISTRY_AUTH_FILE=/tmp/config.json', + 'skopeo:latest' + ] + + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategyTest.groovy new file mode 100644 index 000000000..99d8205c5 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategyTest.groovy @@ -0,0 +1,62 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror.strategy + +import io.seqera.wave.service.mirror.MirrorRequest + +import spock.lang.Specification +import spock.lang.Unroll + +import java.nio.file.Path + +import io.seqera.wave.core.ContainerPlatform + +/** + * + * @author Paolo Di Tommaso + */ +class MirrorStrategyTest extends Specification { + + @Unroll + def 'should return copy command' () { + given: + def strategy = Spy(MirrorStrategy) + and: + def request = MirrorRequest.create( + 'source.io/foo', + 'target.io/foo', + 'sha256:12345', + PLATFORM ? ContainerPlatform.of(PLATFORM) : null, + Path.of('/workspace'), + '{auth json}') + when: + def cmd = strategy.copyCommand(request) + then: + cmd == EXPECTED.tokenize(' ') + + where: + PLATFORM | EXPECTED + null | "copy --preserve-digests --multi-arch all docker://source.io/foo docker://target.io/foo" + 'linux/amd64' | "--override-os linux --override-arch amd64 copy --preserve-digests --multi-arch system docker://source.io/foo docker://target.io/foo" + 'linux/arm64' | "--override-os linux --override-arch arm64 copy --preserve-digests --multi-arch system docker://source.io/foo docker://target.io/foo" + 'linux/arm64/7'| "--override-os linux --override-arch arm64 --override-variant 7 copy --preserve-digests --multi-arch system docker://source.io/foo docker://target.io/foo" + + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy index fa9d5cf36..75ebd6c8c 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy @@ -45,7 +45,6 @@ class WaveBuildRecordTest extends Specification { 'container1234', 'FROM foo:latest', 'conda::recipe', - 'some-spack-recipe', Path.of("/some/path"), 'docker.io/my/repo:container1234', PlatformId.NULL, @@ -77,7 +76,6 @@ class WaveBuildRecordTest extends Specification { 'container1234', 'FROM foo:latest', 'conda::recipe', - 'some-spack-recipe', Path.of("/some/path"), 'docker.io/my/repo:container1234', PlatformId.NULL, diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index 2755b843e..3fdccf208 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -28,20 +28,24 @@ import io.micronaut.context.ApplicationContext import io.micronaut.http.HttpRequest import io.micronaut.http.client.HttpClient import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.api.SubmitContainerTokenRequest -import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.api.ContainerLayer +import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.core.ContainerDigestPair import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.persistence.WaveCondaLockRecord import io.seqera.wave.service.scan.ScanVulnerability +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.builder.BuildEvent +import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult +import io.seqera.wave.service.mirror.MirrorRequest +import io.seqera.wave.service.mirror.MirrorState import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord import io.seqera.wave.service.persistence.WaveScanRecord +import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.test.SurrealDBTestContainer import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User @@ -105,7 +109,6 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe 'container1234', dockerFile, condaFile, - null, Path.of("."), 'docker.io/my/repo:container1234', PlatformId.NULL, @@ -142,7 +145,6 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe 'container1234', 'FROM foo:latest', 'conda::recipe', - null, Path.of("."), 'docker.io/my/repo:container1234', PlatformId.NULL, @@ -172,6 +174,31 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe loaded == record } + def 'should find latest build' () { + given: + def surreal = applicationContext.getBean(SurrealClient) + def persistence = applicationContext.getBean(SurrealPersistenceService) + def auth = persistence.getAuthorization() + def request1 = new BuildRequest( containerId: 'abc', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL ).withBuildId('1') + def request2 = new BuildRequest( containerId: 'abc', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL ).withBuildId('2') + def request3 = new BuildRequest( containerId: 'abc', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL ).withBuildId('3') + + def result1 = new BuildResult(request1.buildId, -1, "ok", request1.startTime, Duration.ofSeconds(2), null) + surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request1, result1))) + and: + def result2 = new BuildResult(request2.buildId, -1, "ok", request2.startTime, Duration.ofSeconds(2), null) + surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request2, result2))) + and: + def result3 = new BuildResult(request3.buildId, -1, "ok", request3.startTime, Duration.ofSeconds(2), null) + surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request3, result3))) + + when: + def loaded = persistence.latestBuild('abc') + + then: + loaded.buildId == 'abc_3' + } + def 'should save and update a build' () { given: def persistence = applicationContext.getBean(SurrealPersistenceService) @@ -179,7 +206,6 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe 'container1234', 'FROM foo:latest', 'conda::recipe', - null, Path.of("/some/path"), 'buildrepo:recipe-container1234', PlatformId.NULL, @@ -313,4 +339,54 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe loaded == record } + //== mirror records tests + + void "should save and load a mirror record by id"() { + given: + def storage = applicationContext.getBean(SurrealPersistenceService) + and: + def request = MirrorRequest.create( + 'source.io/foo', + 'target.io/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/workspace'), + '{auth json}' ) + and: + storage.initializeDb() + and: + def result = MirrorState.from(request) + storage.saveMirrorState(result) + sleep 100 + + when: + def stored = storage.loadMirrorState(request.id) + then: + stored == result + } + + void "should save and load a mirror record by target and digest"() { + given: + def storage = applicationContext.getBean(SurrealPersistenceService) + and: + def request = MirrorRequest.create( + 'source.io/foo', + 'target.io/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/workspace'), + '{auth json}' ) + and: + storage.initializeDb() + and: + def result = MirrorState.from(request) + storage.saveMirrorState(result) + sleep 100 + + when: + def stored = storage.loadMirrorState(request.targetImage, request.digest) + then: + stored == result + } + } diff --git a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy index 6578aa863..e2e4d7c4d 100644 --- a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy @@ -95,4 +95,19 @@ class ValidationServiceTest extends Specification { } + @Unroll + def 'should check registry' () { + expect: + validationService.checkMirrorRegistry(REG)==EXPECTED + + where: + REG | EXPECTED + null | null + 'docker.io' | null + 'docker.io/foo' | 'Mirror registry syntax is invalid - offending value: docker.io/foo' + 'docker://foo.io' | 'Mirror registry should not include any protocol prefix - offending value: docker://foo.io' + 'wave.seqera.io' | 'Mirror registry not allowed - offending value: wave.seqera.io' + 'cr.wave.seqera.io' | 'Mirror registry not allowed - offending value: cr.wave.seqera.io' + } + } diff --git a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy index 7aedec722..bb68799fe 100644 --- a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy @@ -28,7 +28,6 @@ import io.seqera.wave.api.ImageNameStrategy import io.seqera.wave.api.PackagesSpec import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.config.CondaOpts -import io.seqera.wave.config.SpackOpts import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.builder.BuildFormat @@ -134,73 +133,6 @@ class ContainerHelperTest extends Specification { '''.stripIndent() } - def 'should create spack singularity file'() { - given: - def SPACK_OPTS = new SpackOpts([ - basePackages: 'foo bar', - commands: ['run','--this','--that'] - ]) - def packages = new PackagesSpec(type: PackagesSpec.Type.SPACK, spackOpts: SPACK_OPTS) - - when: - def result = ContainerHelper.containerFileFromPackages(packages, true) - - then: - result == '''\ - Bootstrap: docker - From: {{spack_runner_image}} - stage: final - - %files from build - /opt/spack-env /opt/spack-env - /opt/software /opt/software - /opt/._view /opt/._view - /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh - - %post - run - --this - --that - '''.stripIndent() - } - - def 'should create spack docker file'() { - given: - def SPACK_OPTS = new SpackOpts([ - basePackages: 'foo bar', - commands: ['run','--this','--that'] - ]) - def packages = new PackagesSpec(type: PackagesSpec.Type.SPACK, spackOpts: SPACK_OPTS) - - when: - def result = ContainerHelper.containerFileFromPackages(packages, false) - - then: - result == '''\ - # Runner image - FROM {{spack_runner_image}} - - COPY --from=builder /opt/spack-env /opt/spack-env - COPY --from=builder /opt/software /opt/software - COPY --from=builder /opt/._view /opt/._view - - # Entrypoint for Singularity - RUN mkdir -p /.singularity.d/env && \\ - cp -p /opt/spack-env/z10_spack_environment.sh /.singularity.d/env/91-environment.sh - # Entrypoint for Docker - RUN echo "#!/usr/bin/env bash\\n\\nset -ef -o pipefail\\nsource /opt/spack-env/z10_spack_environment.sh\\nexec \\"\\$@\\"" \\ - >/opt/spack-env/spack_docker_entrypoint.sh && chmod a+x /opt/spack-env/spack_docker_entrypoint.sh - - run - --this - --that - - ENTRYPOINT [ "/opt/spack-env/spack_docker_entrypoint.sh" ] - CMD [ "/bin/bash" ] - '''.stripIndent() - } - - def 'should validate conda file helper' () { given: def CONDA = 'this and that' @@ -274,48 +206,6 @@ class ContainerHelperTest extends Specification { '''.stripIndent() } - def 'should validate spack file helper' () { - given: - def SPACK = 'this and that' - def req = new SubmitContainerTokenRequest(spackFile: SPACK.bytes.encodeBase64().toString()) - when: - def result = ContainerHelper.spackFileFromRequest(req) - then: - result == SPACK - } - - def 'should validate spack env file helper' () { - given: - def SPACK = '''\ - spack: - specs: [bwa@0.7.15, salmon@1.1.1] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - and: - def spec = new PackagesSpec(type: PackagesSpec.Type.SPACK, environment: SPACK.bytes.encodeBase64().toString()) - def req = new SubmitContainerTokenRequest(packages: spec) - - when: - def result = ContainerHelper.spackFileFromRequest(req) - then: - result == SPACK - } - - def 'should validate spack env packages helper' () { - given: - def spec = new PackagesSpec(type: PackagesSpec.Type.SPACK, entries: ['foo', 'bar']) - def req = new SubmitContainerTokenRequest(packages: spec) - - when: - def result = ContainerHelper.spackFileFromRequest(req) - then: - result == '''\ - spack: - specs: [foo, bar] - concretizer: {unify: true, reuse: false} - '''.stripIndent(true) - } - def 'should create response v1' () { given: def data = new ContainerRequestData(null, @@ -474,48 +364,6 @@ class ContainerHelperTest extends Specification { ContainerHelper.guessCondaRecipeName(CONDA,true) == new NameVersionPair(['pip','pandas'] as Set, [null, '2.2.2'] as Set) } - def 'should find spack recipe names from spack yaml file' () { - def SPACK = '''\ - spack: - specs: [bwa@0.7.15, salmon@1.1.1, nano@1.0 x=one] - concretizer: {unify: true, reuse: true} - '''.stripIndent(true) - - expect: - ContainerHelper.guessSpackRecipeName(null) == null - ContainerHelper.guessSpackRecipeName(SPACK) == new NameVersionPair(['bwa-0.7.15', 'salmon-1.1.1', 'nano-1.0'] as Set) - and: - ContainerHelper.guessSpackRecipeName(SPACK,true) == new NameVersionPair(['bwa', 'salmon', 'nano'] as Set, ['0.7.15', '1.1.1', '1.0'] as Set) - } - - def 'should throw an exception when spack section is not present in spack yaml file' () { - def SPACK = '''\ - specs: [bwa@0.7.15, salmon@1.1.1, nano@1.0 x=one] - concretizer: {unify: true, reuse: true} - '''.stripIndent(true) - - when: - ContainerHelper.guessSpackRecipeName(SPACK) - then: - def e = thrown(BadRequestException) - and: - e.message == 'Malformed Spack environment file - missing "spack:" section' - } - - def 'should throw an exception when spack.specs section is not present in spack yaml file' () { - def SPACK = '''\ - spack: - concretizer: {unify: true, reuse: true} - '''.stripIndent(true) - - when: - ContainerHelper.guessSpackRecipeName(SPACK) - then: - def e = thrown(BadRequestException) - and: - e.message == 'Malformed Spack environment file - missing "spack.specs:" section' - } - @Unroll def 'should normalise tag' () { expect: @@ -565,10 +413,10 @@ class ContainerHelperTest extends Specification { def 'should make request target' () { expect: - ContainerHelper.makeTargetImage(BuildFormat.DOCKER, 'quay.io/org/name', '12345', null, null, null) + ContainerHelper.makeTargetImage(BuildFormat.DOCKER, 'quay.io/org/name', '12345', null, null) == 'quay.io/org/name:12345' and: - ContainerHelper.makeTargetImage(BuildFormat.SINGULARITY, 'quay.io/org/name', '12345', null, null, null) + ContainerHelper.makeTargetImage(BuildFormat.SINGULARITY, 'quay.io/org/name', '12345', null, null) == 'oras://quay.io/org/name:12345' and: @@ -576,16 +424,9 @@ class ContainerHelperTest extends Specification { dependencies: - salmon=1.2.3 ''' - ContainerHelper.makeTargetImage(BuildFormat.DOCKER, 'quay.io/org/name', '12345', conda, null, null) + ContainerHelper.makeTargetImage(BuildFormat.DOCKER, 'quay.io/org/name', '12345', conda, null) == 'quay.io/org/name:salmon-1.2.3--12345' - and: - def spack = '''\ - spack: - specs: [bwa@0.7.15] - ''' - ContainerHelper.makeTargetImage(BuildFormat.DOCKER, 'quay.io/org/name', '12345', null, spack, null) - == 'quay.io/org/name:bwa-0.7.15--12345' } @@ -633,16 +474,6 @@ class ContainerHelperTest extends Specification { - numpy=1.0 '''.stripIndent(true) - @Shared def SPACK1 = '''\ - spack: - specs: [bwa@0.7.15] - ''' - - @Shared def SPACK2 = '''\ - spack: - specs: [bwa@0.7.15, salmon@1.1.1] - ''' - @Unroll def 'should make request target with name strategy' () { expect: @@ -651,62 +482,50 @@ class ContainerHelperTest extends Specification { REPO, ID, CONDA, - SPACK, STRATEGY ? ImageNameStrategy.valueOf(STRATEGY) : null) == EXPECTED where: - FORMAT | REPO | ID | CONDA | SPACK | STRATEGY | EXPECTED - 'DOCKER' | 'foo.com/build' | '123' | null | null | null | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | null | null | 'none' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | null | null | 'tagPrefix' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | null | null | 'imageSuffix' | 'foo.com/build:123' - and: - 'SINGULARITY' | 'foo.com/build' | '123' | null | null | null | 'oras://foo.com/build:123' - 'SINGULARITY' | 'foo.com/build' | '123' | null | null | 'none' | 'oras://foo.com/build:123' - 'SINGULARITY' | 'foo.com/build' | '123' | null | null | 'tagPrefix' | 'oras://foo.com/build:123' - 'SINGULARITY' | 'foo.com/build' | '123' | null | null | 'imageSuffix' | 'oras://foo.com/build:123' + FORMAT | REPO | ID | CONDA | STRATEGY | EXPECTED + 'DOCKER' | 'foo.com/build' | '123' | null | null | 'foo.com/build:123' + 'DOCKER' | 'foo.com/build' | '123' | null | 'none' | 'foo.com/build:123' + 'DOCKER' | 'foo.com/build' | '123' | null | 'tagPrefix' | 'foo.com/build:123' + 'DOCKER' | 'foo.com/build' | '123' | null | 'imageSuffix' | 'foo.com/build:123' and: - 'DOCKER' | 'foo.com/build' | '123' | CONDA1| null | null | 'foo.com/build:samtools-1.0--123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA1| null | 'none' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA1| null | 'tagPrefix' | 'foo.com/build:samtools-1.0--123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA1| null | 'imageSuffix' | 'foo.com/build/samtools:1.0--123' + 'SINGULARITY' | 'foo.com/build' | '123' | null | null | 'oras://foo.com/build:123' + 'SINGULARITY' | 'foo.com/build' | '123' | null | 'none' | 'oras://foo.com/build:123' + 'SINGULARITY' | 'foo.com/build' | '123' | null | 'tagPrefix' | 'oras://foo.com/build:123' + 'SINGULARITY' | 'foo.com/build' | '123' | null | 'imageSuffix' | 'oras://foo.com/build:123' and: - 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| null | null | 'oras://foo.com/build:samtools-1.0--123' - 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| null | 'none' | 'oras://foo.com/build:123' - 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| null | 'tagPrefix' | 'oras://foo.com/build:samtools-1.0--123' - 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| null | 'imageSuffix' | 'oras://foo.com/build/samtools:1.0--123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA1| null | 'foo.com/build:samtools-1.0--123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA1| 'none' | 'foo.com/build:123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA1| 'tagPrefix' | 'foo.com/build:samtools-1.0--123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA1| 'imageSuffix' | 'foo.com/build/samtools:1.0--123' and: - 'DOCKER' | 'foo.com/build' | '123' | CONDA2| null | null | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15--123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA2| null | 'none' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA2| null | 'tagPrefix' | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15--123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA2| null | 'imageSuffix' | 'foo.com/build/samtools_bamtools_multiqc:123' + 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| null | 'oras://foo.com/build:samtools-1.0--123' + 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| 'none' | 'oras://foo.com/build:123' + 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| 'tagPrefix' | 'oras://foo.com/build:samtools-1.0--123' + 'SINGULARITY' | 'foo.com/build' | '123' | CONDA1| 'imageSuffix' | 'oras://foo.com/build/samtools:1.0--123' and: - 'DOCKER' | 'foo.com/build' | '123' | CONDA3| null | null | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15_bwa-1.2.3_pruned--123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA3| null | 'none' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA3| null | 'tagPrefix' | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15_bwa-1.2.3_pruned--123' - 'DOCKER' | 'foo.com/build' | '123' | CONDA3| null | 'imageSuffix' | 'foo.com/build/samtools_bamtools_multiqc_bwa_pruned:123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA2| null | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15--123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA2| 'none' | 'foo.com/build:123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA2| 'tagPrefix' | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15--123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA2| 'imageSuffix' | 'foo.com/build/samtools_bamtools_multiqc:123' and: - 'DOCKER' | 'foo.com/build' | '123' | PIP1 | null | null | 'foo.com/build:pip_pandas-2.2.2--123' - 'DOCKER' | 'foo.com/build' | '123' | PIP1 | null | 'none' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | PIP1 | null | 'tagPrefix' | 'foo.com/build:pip_pandas-2.2.2--123' - 'DOCKER' | 'foo.com/build' | '123' | PIP1 | null | 'imageSuffix' | 'foo.com/build/pip_pandas:123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA3| null | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15_bwa-1.2.3_pruned--123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA3| 'none' | 'foo.com/build:123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA3| 'tagPrefix' | 'foo.com/build:samtools-1.0_bamtools-2.0_multiqc-1.15_bwa-1.2.3_pruned--123' + 'DOCKER' | 'foo.com/build' | '123' | CONDA3| 'imageSuffix' | 'foo.com/build/samtools_bamtools_multiqc_bwa_pruned:123' and: - 'DOCKER' | 'foo.com/build' | '123' | PIP2 | null | null | 'foo.com/build:pip_pandas-2.2.2_numpy-1.0--123' - 'DOCKER' | 'foo.com/build' | '123' | PIP2 | null | 'tagPrefix' | 'foo.com/build:pip_pandas-2.2.2_numpy-1.0--123' - 'DOCKER' | 'foo.com/build' | '123' | PIP2 | null | 'imageSuffix' | 'foo.com/build/pip_pandas_numpy:123' - 'DOCKER' | 'foo.com/build' | '123' | PIP2 | null | 'none' | 'foo.com/build:123' - + 'DOCKER' | 'foo.com/build' | '123' | PIP1 | null | 'foo.com/build:pip_pandas-2.2.2--123' + 'DOCKER' | 'foo.com/build' | '123' | PIP1 | 'none' | 'foo.com/build:123' + 'DOCKER' | 'foo.com/build' | '123' | PIP1 | 'tagPrefix' | 'foo.com/build:pip_pandas-2.2.2--123' + 'DOCKER' | 'foo.com/build' | '123' | PIP1 | 'imageSuffix' | 'foo.com/build/pip_pandas:123' and: - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK1| null | 'foo.com/build:bwa-0.7.15--123' - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK1| 'none' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK1| 'tagPrefix' | 'foo.com/build:bwa-0.7.15--123' - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK1| 'imageSuffix' | 'foo.com/build/bwa:0.7.15--123' + 'DOCKER' | 'foo.com/build' | '123' | PIP2 | null | 'foo.com/build:pip_pandas-2.2.2_numpy-1.0--123' + 'DOCKER' | 'foo.com/build' | '123' | PIP2 | 'tagPrefix' | 'foo.com/build:pip_pandas-2.2.2_numpy-1.0--123' + 'DOCKER' | 'foo.com/build' | '123' | PIP2 | 'imageSuffix' | 'foo.com/build/pip_pandas_numpy:123' + 'DOCKER' | 'foo.com/build' | '123' | PIP2 | 'none' | 'foo.com/build:123' - and: - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK2| null | 'foo.com/build:bwa-0.7.15_salmon-1.1.1--123' - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK2| 'none' | 'foo.com/build:123' - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK2| 'tagPrefix' | 'foo.com/build:bwa-0.7.15_salmon-1.1.1--123' - 'DOCKER' | 'foo.com/build' | '123' | null | SPACK2| 'imageSuffix' | 'foo.com/build/bwa_salmon:123' } def 'should validate containerfile' () { diff --git a/src/test/groovy/io/seqera/wave/util/SpackHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/SpackHelperTest.groovy deleted file mode 100644 index 6e5ac880f..000000000 --- a/src/test/groovy/io/seqera/wave/util/SpackHelperTest.groovy +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.util - -import spock.lang.Specification - -import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.builder.BuildFormat - -/** - * - * @author Paolo Di Tommaso - */ -class SpackHelperTest extends Specification { - - def 'should load builder template' () { - expect: - SpackHelper.builderDockerTemplate().startsWith('# Builder image') - } - - def 'should prepend builder template' () { - expect: - SpackHelper.prependBuilderTemplate('foo', BuildFormat.DOCKER).startsWith('# Builder image') - SpackHelper.prependBuilderTemplate('foo', BuildFormat.SINGULARITY).endsWith('\nfoo') - } - - def 'should map platform to spack arch' () { - expect: - SpackHelper.toSpackArch(ContainerPlatform.of('x86_64')) == 'x86_64' - SpackHelper.toSpackArch(ContainerPlatform.of('linux/x86_64')) == 'x86_64' - SpackHelper.toSpackArch(ContainerPlatform.of('amd64')) == 'x86_64' - SpackHelper.toSpackArch(ContainerPlatform.of('aarch64')) == 'aarch64' - SpackHelper.toSpackArch(ContainerPlatform.of('arm64')) == 'aarch64' - SpackHelper.toSpackArch(ContainerPlatform.of('linux/arm64/v8')) == 'aarch64' - - when: - SpackHelper.toSpackArch(ContainerPlatform.of('linux/arm64/v7')) - then: - thrown(IllegalArgumentException) - } -} diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index f7ff1edfa..b56b9bd5d 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -53,8 +53,6 @@ wave: password: 'bar' build: workspace: 'build-workspace' - spack: - cacheDirectory: "spack-cache" logs : enabled : true bucket : 'nextflow-ci' diff --git a/typespec/models/Packages.tsp b/typespec/models/Packages.tsp index 6fa59e16d..ce534501b 100644 --- a/typespec/models/Packages.tsp +++ b/typespec/models/Packages.tsp @@ -1,5 +1,4 @@ import "./CondaOpts.tsp"; -import "./SpackOpts.tsp"; @doc("Package configurations for container builds.") model Packages { @@ -7,6 +6,5 @@ model Packages { condaOpts?: CondaOpts; entries: string[]; environment: string; - spackOpts?: SpackOpts; - type: "CONDA" | "SPACK"; -} \ No newline at end of file + type: "CONDA"; +} diff --git a/typespec/models/SpackOpts.tsp b/typespec/models/SpackOpts.tsp deleted file mode 100644 index 813caa6a2..000000000 --- a/typespec/models/SpackOpts.tsp +++ /dev/null @@ -1,5 +0,0 @@ -@doc("Options for Spack environments. Spack support will be removed in future releases") -model SpackOpts { - basePackages: string; - commands: string[]; -} diff --git a/typespec/models/WaveBuildRecord.tsp b/typespec/models/WaveBuildRecord.tsp index f8e40bc4c..50f6966d1 100644 --- a/typespec/models/WaveBuildRecord.tsp +++ b/typespec/models/WaveBuildRecord.tsp @@ -10,7 +10,6 @@ model WaveBuildRecord { platform: string; requestIp: string; scanId: string; - spackFile: string; startTime: string; succeeded: boolean; targetImage: string; @@ -18,4 +17,3 @@ model WaveBuildRecord { userId: int64; userName: string; } - \ No newline at end of file