Merge branch 'master' into remove-keyid

build.gradle

@@ -70,6 +70,7 @@ dependencies {
     implementation "software.amazon.awssdk:ecrpublic"
     implementation 'software.amazon.awssdk:ses'
     implementation 'org.yaml:snakeyaml:2.0'
+    implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8'
     //object storage dependency
     implementation("io.micronaut.objectstorage:micronaut-object-storage-aws")
     // include sts to allow the use of service account role - https://stackoverflow.com/a/73306570

configuration.md

@@ -216,7 +216,7 @@ Wave offers a feature to provide a cache for Docker blobs, which improves the pe
 
 - **`wave.blobCache.enabled`**: whether to enable the blob cache. It is `false` by default. *Optional*.
 
-- **`wave.blobCache.s5cmdImage`**: the Docker image that supplies the [s5cmd tool](https://github.com/peak/s5cmd). This tool is used to upload blob binaries to the S3 bucket. The default image used by Wave is `cr.seqera.io/public/wave/s5cmd:v2.2.2`. *Optional*.
+- **`wave.blobCache.s5cmdImage`**: the Docker image that supplies the [s5cmd tool](https://github.com/peak/s5cmd). This tool is used to upload blob binaries to the S3 bucket. The default image used by Wave is `public.cr.seqera.io/wave/s5cmd:v2.2.2`. *Optional*.
 
 - **`wave.blobCache.status.delay`**: the time delay in checking the status of the transfer of the blob binary from the repository to the cache. Its default value is `5s`. *Optional*.
 

docs/get-started.mdx

@@ -18,8 +18,8 @@ In this guide, you'll request a containerized Conda package from Seqera Containe
 ### Request a Conda package as a Seqera Container
 
 1. Open [Seqera Containers][sc] in a browser.
-1. In the search box, enter `faker`.
-1. In the search results, select **Add** in the `conda-forge::faker` result, and then **Get Container** to initiate the container build.
+1. In the search box, enter `samtools`.
+1. In the search results, select **Add** in the `bioconda::samtools` result, and then **Get Container** to initiate the container build.
 1. From the **Fetching container** modal, copy the the durable container image URI that Seqera Containers provides.
 1. Optional: Select **View build details** to watch Seqera Containers build the requested container in real time.
 
@@ -39,25 +39,25 @@ Nextflow can use the container that Seqera Containers built in the previous sect
 1. Create a `main.nf` file with the following contents:
 
     ```groovy
-    process FAKER {
+    process SAMTOOLS {
       container '<container_uri>'
       debug true
-
       """
-      faker address
+      samtools --version-only
       """
     }
-
     workflow {
-      FAKER()
+      SAMTOOLS()
     }
     ```
 
-    Substitute `<container_uri>` for the container URI that you received from Seqera Containers in the previous section.
+    Substitute `<container_uri>` for the container URI that you received from Seqera Containers in the previous section. e.g.
+    - `community.wave.seqera.io/library/samtools:1.20--b5dfbd93de237464` for linux/amd64.
+    - `community.wave.seqera.io/library/samtools:1.20--497854c5df637867` for linux/arm64.
 
 ### Run the Nextflow pipeline
 
-To confirm that the `faker` command is available from your pipeline, run the following command:
+To confirm that the `samtools` command is available from your pipeline, run the following command:
 
 ```
 nextflow run main.nf
@@ -66,12 +66,14 @@ nextflow run main.nf
 The output from a successful execution is displayed in the following example:
 
 ```
-Launching `main.nf` [jolly_edison] DSL2 - revision: 5c414bd927
+ N E X T F L O W   ~  version 24.04.4
+
+Launching `samtools.nf` [furious_carlsson] DSL2 - revision: 04817f962f
 
 executor >  local (1)
-[86/0d56e8] faker | 1 of 1 ✔
-234 Nicholas Circle
-Masonport, MS 98018
+[2f/d2ccc7] process > SAMTOOLS [100%] 1 of 1 ✔
+1.20+htslib-1.20
+
 ```
 ## Nextflow
 

s5cmd/Makefile

@@ -5,5 +5,5 @@ build:
 			--push \
 			--platform linux/amd64,linux/arm64 \
 			--build-arg version=${version} \
-			--tag cr.seqera.io/public/wave/s5cmd:v${version} \
+			--tag public.cr.seqera.io/wave/s5cmd:v${version} \
 			.

s5cmd/dist.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#
+#  Wave, containers provisioning service
+#  Copyright (c) 2023-2024, Seqera Labs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+arch=$(uname -m)
+
+case $arch in
+    x86_64|amd64)
+        echo "https://github.com/peak/s5cmd/releases/download/v$1/s5cmd_$1_Linux-64bit.tar.gz"
+        ;;
+    aarch64|arm64)
+	echo "https://github.com/peak/s5cmd/releases/download/v$1/s5cmd_$1_Linux-arm64.tar.gz"
+        ;;
+    *)
+        echo "Unknown architecture: $arch"
+        ;;
+esac

src/main/groovy/io/seqera/wave/ErrorHandler.groovy

@@ -18,8 +18,6 @@
 
 package io.seqera.wave
 
-import java.util.function.BiFunction
-
 import groovy.util.logging.Slf4j
 import io.micronaut.context.annotation.Value
 import io.micronaut.http.HttpRequest
@@ -32,6 +30,7 @@ import io.seqera.wave.exception.DockerRegistryException
 import io.seqera.wave.exception.ForbiddenException
 import io.seqera.wave.exception.HttpResponseException
 import io.seqera.wave.exception.NotFoundException
+import io.seqera.wave.exception.RegistryForwardException
 import io.seqera.wave.exception.SlowDownException
 import io.seqera.wave.exception.UnauthorizedException
 import io.seqera.wave.exception.WaveException
@@ -46,10 +45,14 @@ import jakarta.inject.Singleton
 @Singleton
 class ErrorHandler {
 
+    static interface Mapper<T> {
+        T apply(String message, String errorCode)
+    }
+
     @Value('${wave.debug:false}')
     private Boolean debug
 
-    def <T> HttpResponse<T> handle(HttpRequest httpRequest, Throwable t, BiFunction<String,String,T> responseFactory) {
+    def <T> HttpResponse<T> handle(HttpRequest httpRequest, Throwable t, Mapper<T> responseFactory) {
         final errId = LongRndKey.rndHex()
         final request = httpRequest?.toString()
         def msg = t.message
@@ -78,6 +81,14 @@ class ErrorHandler {
             log.error(render, t)
         }
 
+        if( t instanceof RegistryForwardException ) {
+            // report this error as it has been returned by the target registry
+            return HttpResponse
+                    .status(HttpStatus.valueOf(t.statusCode))
+                    .body(t.response)
+                    .headers(t.headers)
+        }
+
         if( t instanceof DockerRegistryException ) {
             final resp = responseFactory.apply(msg, t.error)
             return HttpResponseFactory.INSTANCE.status(t.statusCode).body(resp)

src/main/groovy/io/seqera/wave/WaveDefault.groovy

@@ -17,11 +17,14 @@
  */
 
 package io.seqera.wave
+
+import groovy.transform.CompileStatic
 /**
  * Wave app defaults
  *
  * @author Paolo Di Tommaso <[email protected]>
  */
+@CompileStatic
 interface WaveDefault {
 
     final static public String DOCKER_IO = 'docker.io'
@@ -38,7 +41,7 @@ interface WaveDefault {
                     'application/vnd.docker.distribution.manifest.list.v2+json' ) )
 
 
-    final public static int[] HTTP_REDIRECT_CODES = List.of(301, 302, 303, 307, 308)
+    final public static List<Integer> HTTP_REDIRECT_CODES = List.of(301, 302, 303, 307, 308)
 
     final public static List<Integer> HTTP_SERVER_ERRORS = List.of(500, 502, 503, 504)
 

src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy

@@ -18,6 +18,8 @@
 
 package io.seqera.wave.auth
 
+import io.seqera.wave.exception.RegistryUnauthorizedAccessException
+
 /**
  * Declares container registry authentication & authorization operations
  *

src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy

@@ -35,14 +35,16 @@ import groovy.transform.PackageScope
 import groovy.transform.ToString
 import groovy.util.logging.Slf4j
 import io.seqera.wave.configuration.HttpClientConfig
+import io.seqera.wave.exception.RegistryForwardException
+import io.seqera.wave.exception.RegistryUnauthorizedAccessException
 import io.seqera.wave.http.HttpClientFactory
 import io.seqera.wave.util.RegHelper
 import io.seqera.wave.util.Retryable
 import io.seqera.wave.util.StringUtils
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static io.seqera.wave.WaveDefault.DOCKER_IO
-import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS
+import static io.seqera.wave.auth.RegistryUtils.isServerError
 /**
  * Implement Docker authentication & login service
  *
@@ -110,7 +112,6 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
 
     @Inject RegistryCredentialsFactory credentialsFactory
 
-
     /**
      * Implements container registry login
      *
@@ -143,10 +144,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
                 .header("Authorization", "Basic $basic")
                 .build()
         // retry strategy
+        // note: do not retry on 429 error code because it just continues to report the error
+        // for a while. better returning the error to the upstream client
+        // see also https://github.com/docker/hub-feedback/issues/1907#issuecomment-631028965
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf( (response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS)
-                .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event}"))
+                .retryIf((response) -> isServerError(response))
+                .onRetry((event) -> log.warn("Unable to connect '$endpoint' - attempt: ${event.attempt} status: ${event.result?.statusCode()}; body: ${event.result?.body()}"))
         // make the request
         final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString()))
         final body = response.body()
@@ -230,10 +234,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
         log.trace "Token request=$req"
 
         // retry strategy
+        // note: do not retry on 429 error code because it just continues to report the error
+        // for a while. better returning the error to the upstream client
+        // see also https://github.com/docker/hub-feedback/issues/1907#issuecomment-631028965
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_RETRYABLE_ERRORS )
-                .onRetry((event) -> log.warn("Unable to connect '$login' - event: $event"))
+                .retryIf((response) -> isServerError(response))
+                .onRetry((event) -> log.warn("Unable to connect '$login' - attempt: ${event.attempt} status: ${event.result?.statusCode()}; body: ${event.result?.body()}"))
         // submit http request
         final response = retryable.apply(()-> httpClient.send(req, HttpResponse.BodyHandlers.ofString()))
         // check the response
@@ -248,8 +255,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
                 return token
             }
         }
-
-        throw new RegistryUnauthorizedAccessException("Unable to authorize request: $login", response.statusCode(), body)
+        throw new RegistryForwardException("Unexpected response acquiring token for '$login' [${response.statusCode()}]", response)
     }
 
     String buildLoginUrl(URI realm, String image, String service){

src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy

@@ -20,21 +20,24 @@ package io.seqera.wave.auth
 
 import java.net.http.HttpRequest
 import java.net.http.HttpResponse
+import java.util.concurrent.ExecutionException
 import java.util.concurrent.TimeUnit
 
 import com.google.common.cache.CacheBuilder
 import com.google.common.cache.CacheLoader
 import com.google.common.cache.LoadingCache
+import com.google.common.util.concurrent.UncheckedExecutionException
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import io.seqera.wave.configuration.HttpClientConfig
+import io.seqera.wave.exception.RegistryForwardException
 import io.seqera.wave.http.HttpClientFactory
 import io.seqera.wave.util.Retryable
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static io.seqera.wave.WaveDefault.DOCKER_IO
 import static io.seqera.wave.WaveDefault.DOCKER_REGISTRY_1
-import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS
+import static io.seqera.wave.auth.RegistryUtils.isServerError
 /**
  * Lookup service for container registry. The role of this component
  * is to registry the retrieve the registry authentication realm
@@ -81,13 +84,15 @@ class RegistryLookupServiceImpl implements RegistryLookupService {
         final httpClient = HttpClientFactory.followRedirectsHttpClient()
         final request = HttpRequest.newBuilder() .uri(endpoint) .GET() .build()
         // retry strategy
+        // note: do not retry on 429 error code because it just continues to report the error
+        // for a while. better returning the error to the upstream client
+        // see also https://github.com/docker/hub-feedback/issues/1907#issuecomment-631028965
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf((response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS )
-                .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event"))
+                .retryIf((response) -> isServerError(response))
+                .onRetry((event) -> log.warn("Unable to connect '$endpoint' - attempt: ${event.attempt} status: ${event.result?.statusCode()}; body: ${event.result?.body()}"))
         // submit the request
         final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString()))
-        final body = response.body()
         // check response
         final code = response.statusCode()
         if( code == 401 ) {
@@ -102,9 +107,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService {
         else if( code == 200 ) {
             return new RegistryAuth(endpoint)
         }
-        else {
-            throw new IllegalArgumentException("Request '$endpoint' unexpected response code: $code; message: ${body} ")
-        }
+        throw new RegistryForwardException("Unexpected response for '$endpoint' [${response.statusCode()}]", response)
     }
 
     /**
@@ -117,8 +120,10 @@ class RegistryLookupServiceImpl implements RegistryLookupService {
             final auth = cache.get(endpoint)
             return new RegistryInfo(registry, endpoint, auth)
         }
-        catch (Throwable t) {
-            throw new RegistryLookupException("Unable to lookup authority for registry '$registry'", t)
+        catch (UncheckedExecutionException | ExecutionException e) {
+            // this catches the exception thrown in the cache loader lookup
+            // and throws the causing exception that should be `RegistryUnauthorizedAccessException`
+            throw e.cause
         }
     }
 

src/main/groovy/io/seqera/wave/auth/RegistryUtils.groovy

@@ -0,0 +1,37 @@
+/*
+ *  Wave, containers provisioning service
+ *  Copyright (c) 2023-2024, Seqera Labs
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package io.seqera.wave.auth
+
+import java.net.http.HttpResponse
+
+import groovy.transform.CompileStatic
+import io.seqera.wave.WaveDefault
+/**
+ * Utility class for registry functions
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+@CompileStatic
+class RegistryUtils {
+
+    static boolean isServerError(HttpResponse response) {
+        response.statusCode() in WaveDefault.HTTP_SERVER_ERRORS
+    }
+
+}