diff --git a/onyxia-api/pom.xml b/onyxia-api/pom.xml index e66d0f18..5f1bbb7f 100644 --- a/onyxia-api/pom.xml +++ b/onyxia-api/pom.xml @@ -72,11 +72,6 @@ 0.0.1-SNAPSHOT - - io.micrometer - micrometer-registry-prometheus - - org.springframework.boot spring-boot-starter-actuator diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/metrics/CustomMetrics.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/metrics/CustomMetrics.java deleted file mode 100644 index f4db5bff..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/metrics/CustomMetrics.java +++ /dev/null @@ -1,26 +0,0 @@ -package fr.insee.onyxia.api.configuration.metrics; - -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.binder.MeterBinder; -import org.springframework.stereotype.Service; - -@Service -public class CustomMetrics implements MeterBinder { - - private static Counter counter; - - public void plusUn() { - counter.increment(); - } - - @Override - public void bindTo(MeterRegistry registry) { - counter = - Counter.builder("servicespopes") - .description("Nombre de services popes") - .tags("services", "nombre") - .baseUnit("services") - .register(registry); - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java index 519d3635..8451c222 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/MyLabController.java @@ -232,7 +232,7 @@ private void suspendOrResume(Region region, Project project, String serviceId, b CatalogWrapper catalog = elligibleCatalogs.getFirst(); catalogId = catalog.getId(); } - Optional catalog = catalogService.getCatalogById(catalogId); + Optional catalog = catalogService.getCatalogById(catalogId, user); if (catalog.isEmpty()) { throw new IllegalStateException( "Catalog " + catalogId + " is not available anymore"); diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/QuotaController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/QuotaController.java index c184d0db..ca393943 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/QuotaController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/api/mylab/QuotaController.java @@ -76,10 +76,8 @@ public QuotaUsage getQuota( } final QuotaUsage quotaUsage = new QuotaUsage(); - final Quota spec = new Quota(); - final Quota usage = new Quota(); - mapKubQuotaToQuota(resourceQuota.getStatus().getHard(), spec); - mapKubQuotaToQuota(resourceQuota.getStatus().getUsed(), usage); + final Quota spec = kubQuotaToQuota(resourceQuota.getStatus().getHard()); + final Quota usage = kubQuotaToQuota(resourceQuota.getStatus().getUsed()); quotaUsage.setSpec(spec); quotaUsage.setUsage(usage); @@ -199,10 +197,10 @@ private Owner getOwner(Region region, Project project) { return owner; } - private void mapKubQuotaToQuota(Map resourceQuota, Quota quota) { + private Quota kubQuotaToQuota(Map resourceQuota) { final Map rawData = new HashMap<>(); - resourceQuota.entrySet().forEach(e -> rawData.put(e.getKey(), e.getValue().toString())); - quota.loadFromMap(rawData); + resourceQuota.forEach((key, value) -> rawData.put(key, value.toString())); + return Quota.from(rawData); } public KubernetesService getKubernetesService() { diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 173b03e1..5eaa802a 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -10,11 +10,7 @@ import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.helm.Repository; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; +import java.io.*; import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -173,38 +169,32 @@ private void refreshPackage(CatalogWrapper cw, Pkg pkg) } public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { - GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(in); - // HelmConfig config = null; - try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) { + String chartName = chart.getName(); + + try (GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(in); + TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) { + TarArchiveEntry entry; while ((entry = tarIn.getNextEntry()) != null) { - if (entry.getName().endsWith(chart.getName() + "/values.schema.json") - && !entry.getName() - .endsWith("charts/" + chart.getName() + "/values.schema.json")) { + String entryName = entry.getName(); + if (entryName.endsWith(chartName + "/values.schema.json") + && !entryName.endsWith("charts/" + chartName + "/values.schema.json")) { // TODO : mutualize objectmapper ObjectMapper mapper = new ObjectMapper(); mapper.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int len; - while ((len = tarIn.read(buffer)) != -1) { - baos.write(buffer, 0, len); + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + tarIn.transferTo(baos); + chart.setConfig(mapper.readTree(baos.toString(UTF_8))); } - chart.setConfig(mapper.readTree(baos.toString("UTF-8"))); - } - if (entry.getName().endsWith(chart.getName() + "/values.yaml") - && !entry.getName() - .endsWith("charts/" + chart.getName() + "/values.yaml")) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int len; - while ((len = tarIn.read(buffer)) != -1) { - baos.write(buffer, 0, len); + } else if (entryName.endsWith(chartName + "/values.yaml") + && !entryName.endsWith("charts/" + chartName + "/values.yaml")) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + tarIn.transferTo(baos); + chart.setDefaultValues(baos.toString()); } - byte[] fileContent = baos.toByteArray(); - chart.setDefaultValues(new String(fileContent)); } } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index bcd95952..5be9b072 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -38,7 +38,6 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.FastDateFormat; import org.slf4j.Logger; @@ -67,6 +66,10 @@ public class HelmAppsService implements AppsService { final OnyxiaEventPublisher onyxiaEventPublisher; public static final String ONYXIA_SECRET_PREFIX = "sh.onyxia.release.v1."; + private static final String CATALOG = "catalog"; + private static final String OWNER = "owner"; + private static final String FRIENDLY_NAME = "friendlyName"; + private static final String SHARE = "share"; @Autowired public HelmAppsService( @@ -131,13 +134,12 @@ public Collection installApp( requestDTO.getFriendlyName()); onyxiaEventPublisher.publishEvent(installServiceEvent); Map metadata = new HashMap<>(); - metadata.put("catalog", Base64Utils.base64Encode(catalogId)); - metadata.put("owner", Base64Utils.base64Encode(user.getIdep())); + metadata.put(CATALOG, Base64Utils.base64Encode(catalogId)); + metadata.put(OWNER, Base64Utils.base64Encode(user.getIdep())); if (requestDTO.getFriendlyName() != null) { - metadata.put( - "friendlyName", Base64Utils.base64Encode(requestDTO.getFriendlyName())); + metadata.put(FRIENDLY_NAME, Base64Utils.base64Encode(requestDTO.getFriendlyName())); } - metadata.put("share", Base64Utils.base64Encode(String.valueOf(requestDTO.isShare()))); + metadata.put(SHARE, Base64Utils.base64Encode(String.valueOf(requestDTO.isShare()))); kubernetesService.createOnyxiaSecret( region, namespaceId, requestDTO.getName(), metadata); return List.of(res.getManifest()); @@ -182,18 +184,13 @@ public CompletableFuture getUserServices( installedCharts.parallelStream() .map(release -> getHelmApp(region, user, release)) .filter( - service -> { - boolean canUserSeeThisService = false; - if (project.getGroup() == null - || service.isShare() - || user.getIdep() - .equalsIgnoreCase(service.getOwner())) { - // Personal group - canUserSeeThisService = true; - } - return canUserSeeThisService; - }) - .collect(Collectors.toList()); + service -> + // Check if user can see this service + project.getGroup() == null + || service.isShare() + || user.getIdep() + .equalsIgnoreCase(service.getOwner())) + .toList(); ServicesListing listing = new ServicesListing(); listing.setApps(services); return CompletableFuture.completedFuture(listing); @@ -307,18 +304,18 @@ private Service getHelmApp(Region region, User user, HelmLs release) { .get(); if (secret != null && secret.getData() != null) { Map data = secret.getData(); - if (data.containsKey("friendlyName")) { - service.setFriendlyName(Base64Utils.base64Decode(data.get("friendlyName"))); + if (data.containsKey(FRIENDLY_NAME)) { + service.setFriendlyName(Base64Utils.base64Decode(data.get(FRIENDLY_NAME))); } - if (data.containsKey("owner")) { - service.setOwner(Base64Utils.base64Decode(data.get("owner"))); + if (data.containsKey(OWNER)) { + service.setOwner(Base64Utils.base64Decode(data.get(OWNER))); } - if (data.containsKey("catalog")) { - service.setCatalogId(Base64Utils.base64Decode(data.get("catalog"))); + if (data.containsKey(CATALOG)) { + service.setCatalogId(Base64Utils.base64Decode(data.get(CATALOG))); } - if (data.containsKey("share")) { + if (data.containsKey(SHARE)) { service.setShare( - Boolean.parseBoolean(Base64Utils.base64Decode(data.get("share")))); + Boolean.parseBoolean(Base64Utils.base64Decode(data.get(SHARE)))); } } } catch (Exception e) { @@ -341,8 +338,7 @@ private Service getHelmApp(Region region, User user, HelmLs release) { Map result = new HashMap<>(); node.fields() .forEachRemaining( - currentNode -> - mapAppender(result, currentNode, new ArrayList())); + currentNode -> mapAppender(result, currentNode, new ArrayList<>())); service.setEnv(result); service.setSuspendable(service.getEnv().containsKey(SUSPEND_KEY)); if (service.getEnv().containsKey(SUSPEND_KEY)) { @@ -364,13 +360,13 @@ private Service getHelmApp(Region region, User user, HelmLs release) { public void rename( Region region, Project project, User user, String serviceId, String friendlyName) throws IOException, InterruptedException, TimeoutException { - patchOnyxiaSecret(region, project, user, serviceId, Map.of("friendlyName", friendlyName)); + patchOnyxiaSecret(region, project, user, serviceId, Map.of(FRIENDLY_NAME, friendlyName)); } @Override public void share(Region region, Project project, User user, String serviceId, boolean share) throws IOException, InterruptedException, TimeoutException { - patchOnyxiaSecret(region, project, user, serviceId, Map.of("share", String.valueOf(share))); + patchOnyxiaSecret(region, project, user, serviceId, Map.of(SHARE, String.valueOf(share))); } private void patchOnyxiaSecret( @@ -386,10 +382,7 @@ private void patchOnyxiaSecret( if (secret != null) { Map secretData = secret.getData() != null ? secret.getData() : new HashMap<>(); - data.forEach( - (k, v) -> { - secretData.put(k, Base64Utils.base64Encode(v)); - }); + data.forEach((k, v) -> secretData.put(k, Base64Utils.base64Encode(v))); secret.setData(secretData); if (secret.getMetadata().getManagedFields() != null) { secret.getMetadata().getManagedFields().clear(); @@ -401,7 +394,7 @@ private void patchOnyxiaSecret( .serverSideApply(); } else { Map metadata = new HashMap<>(); - metadata.put("owner", user.getIdep()); + metadata.put(OWNER, user.getIdep()); metadata.putAll(data); kubernetesService.createOnyxiaSecret(region, namespaceId, serviceId, metadata); } @@ -590,7 +583,7 @@ private Service getServiceFromRelease( currentTask.setStatus(status); return currentTask; }) - .collect(Collectors.toList())); + .toList()); return service; } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/ServiceUrlResolver.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/ServiceUrlResolver.java index 43b0f6c2..605d3d53 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/ServiceUrlResolver.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/ServiceUrlResolver.java @@ -12,8 +12,15 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class ServiceUrlResolver { + + private ServiceUrlResolver() {} + + private static final Logger LOGGER = LoggerFactory.getLogger(ServiceUrlResolver.class); + static List getServiceUrls(Region region, String manifest, KubernetesClient client) { Region.Expose expose = region.getServices().getExpose(); boolean isIstioEnabled = expose.getIstio() != null && expose.getIstio().isEnabled(); @@ -49,9 +56,8 @@ static List getServiceUrls(Region region, String manifest, KubernetesCli .getPath())) .toList()); } catch (Exception e) { - System.out.println( - "Warning : could not read urls from ingress " - + ingress.getFullResourceName()); + LOGGER.warn( + "Could not read urls from ingress {}", ingress.getFullResourceName()); } } } @@ -66,9 +72,9 @@ static List getServiceUrls(Region region, String manifest, KubernetesCli try { urls.add(resource.get("spec", "host")); } catch (Exception e) { - System.out.println( - "Warning : could not read urls from OpenShift Route " - + resource.getFullResourceName()); + LOGGER.warn( + "Could not read urls from OpenShift Route {}", + resource.getFullResourceName()); } } } @@ -85,9 +91,9 @@ static List getServiceUrls(Region region, String manifest, KubernetesCli // One should consider to add support for 'spec/http[*]/match[*]/uri/prefix' urls.addAll(resource.get("spec", "hosts")); } catch (Exception e) { - System.out.println( - "Warning : could not read urls from Istio Virtual Service " - + resource.getFullResourceName()); + LOGGER.warn( + "Could not read urls from Istio Virtual Service {}", + resource.getFullResourceName()); } } } diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/service/quota/Quota.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/quota/Quota.java index 687605ac..148eea11 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/service/quota/Quota.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/service/quota/Quota.java @@ -10,7 +10,18 @@ "Resource quotas are a tool for administrators to address the fair sharing of cluster resources between namespaces https://kubernetes.io/docs/concepts/policy/resource-quotas/") public class Quota { - @JsonProperty("requests.memory") + private static final String REQUESTS_MEMORY = "requests.memory"; + private static final String REQUESTS_CPU = "requests.cpu"; + private static final String LIMITS_MEMORY = "limits.memory"; + private static final String LIMITS_CPU = "limits.cpu"; + private static final String REQUESTS_STORAGE = "requests.storage"; + private static final String COUNT_PODS = "count/pods"; + private static final String REQUESTS_EPHEMERAL_STORAGE = "requests.ephemeral-storage"; + private static final String LIMITS_EPHEMERAL_STORAGE = "limits.ephemeral-storage"; + private static final String REQUESTS_NVIDIA_COM_GPU = "requests.nvidia.com/gpu"; + private static final String LIMITS_NVIDIA_COM_GPU = "limits.nvidia.com/gpu"; + + @JsonProperty(REQUESTS_MEMORY) @Schema( description = "Across all pods in a non-terminal state, the sum of memory requests cannot exceed this value.") @@ -19,82 +30,84 @@ public class Quota { @Schema( description = "Across all pods in a non-terminal state, the sum of CPU requests cannot exceed this value.") - @JsonProperty("requests.cpu") + @JsonProperty(REQUESTS_CPU) private String cpuRequests; @Schema( description = "Across all pods in a non-terminal state, the sum of memory limits cannot exceed this value.") - @JsonProperty("limits.memory") + @JsonProperty(LIMITS_MEMORY) private String memoryLimits; @Schema( description = "Across all pods in a non-terminal state, the sum of CPU limits cannot exceed this value.") - @JsonProperty("limits.cpu") + @JsonProperty(LIMITS_CPU) private String cpuLimits; @Schema( description = "Across all persistent volume claims, the sum of storage requests cannot exceed this value.") - @JsonProperty("requests.storage") + @JsonProperty(REQUESTS_STORAGE) private String storageRequests; @Schema(description = "The number of pod in the namespace cannot exceed this value.") - @JsonProperty("count/pods") + @JsonProperty(COUNT_PODS) private Integer podsCount; @Schema(description = "The request ephemeralStorage allowed") - @JsonProperty("requests.ephemeral-storage") + @JsonProperty(REQUESTS_EPHEMERAL_STORAGE) private String ephemeralStorageRequests; @Schema(description = "The limit ephemeralStorage allowed") - @JsonProperty("limits.ephemeral-storage") + @JsonProperty(LIMITS_EPHEMERAL_STORAGE) private String ephemeralStorageLimits; @Schema(description = "The request nvidia gpu") - @JsonProperty("requests.nvidia.com/gpu") + @JsonProperty(REQUESTS_NVIDIA_COM_GPU) private Integer nvidiaGpuRequests; @Schema(description = "The limit nvidia gpu") - @JsonProperty("limits.nvidia.com/gpu") + @JsonProperty(LIMITS_NVIDIA_COM_GPU) private Integer nvidiaGpuLimits; public Map asMap() { final Map quotas = new HashMap<>(); - quotas.put("requests.memory", getMemoryRequests()); - quotas.put("requests.cpu", getCpuRequests()); - quotas.put("limits.memory", getMemoryLimits()); - quotas.put("limits.cpu", getCpuLimits()); - quotas.put("requests.storage", getStorageRequests()); - quotas.put("count/pods", getPodsCount() == null ? null : String.valueOf(getPodsCount())); - quotas.put("requests.ephemeral-storage", getEphemeralStorageRequests()); - quotas.put("limits.ephemeral-storage", getEphemeralStorageLimits()); + quotas.put(REQUESTS_MEMORY, getMemoryRequests()); + quotas.put(REQUESTS_CPU, getCpuRequests()); + quotas.put(LIMITS_MEMORY, getMemoryLimits()); + quotas.put(LIMITS_CPU, getCpuLimits()); + quotas.put(REQUESTS_STORAGE, getStorageRequests()); + quotas.put(COUNT_PODS, getPodsCount() == null ? null : String.valueOf(getPodsCount())); + quotas.put(REQUESTS_EPHEMERAL_STORAGE, getEphemeralStorageRequests()); + quotas.put(LIMITS_EPHEMERAL_STORAGE, getEphemeralStorageLimits()); quotas.put( - "requests.nvidia.com/gpu", + REQUESTS_NVIDIA_COM_GPU, getNvidiaGpuRequests() == null ? null : String.valueOf(getNvidiaGpuRequests())); quotas.put( - "limits.nvidia.com/gpu", + LIMITS_NVIDIA_COM_GPU, getNvidiaGpuLimits() == null ? null : String.valueOf(getNvidiaGpuLimits())); return quotas; } - public void loadFromMap(Map data) { - setMemoryRequests(data.get("requests.memory")); - setCpuRequests(data.get("requests.cpu")); - setMemoryLimits(data.get("limits.memory")); - setCpuLimits(data.get("limits.cpu")); - setEphemeralStorageRequests(data.get("requests.ephemeral-storage")); - setEphemeralStorageLimits(data.get("limits.ephemeral-storage")); - setStorageRequests(data.get("requests.storage")); - if (data.containsKey("requests.nvidia.com/gpu")) { - setNvidiaGpuRequests(Integer.parseInt(data.get("requests.nvidia.com/gpu"))); + public static Quota from(Map data) { + Quota quota = new Quota(); + quota.setMemoryRequests(data.get(REQUESTS_MEMORY)); + quota.setCpuRequests(data.get(REQUESTS_CPU)); + quota.setMemoryLimits(data.get(LIMITS_MEMORY)); + quota.setCpuLimits(data.get(LIMITS_CPU)); + quota.setEphemeralStorageRequests(data.get(REQUESTS_EPHEMERAL_STORAGE)); + quota.setEphemeralStorageLimits(data.get(LIMITS_EPHEMERAL_STORAGE)); + quota.setStorageRequests(data.get(REQUESTS_STORAGE)); + if (data.containsKey(REQUESTS_NVIDIA_COM_GPU)) { + quota.setNvidiaGpuRequests(Integer.parseInt(data.get(REQUESTS_NVIDIA_COM_GPU))); } - if (data.containsKey("limits.nvidia.com/gpu")) { - setNvidiaGpuLimits(Integer.parseInt(data.get("limits.nvidia.com/gpu"))); + if (data.containsKey(LIMITS_NVIDIA_COM_GPU)) { + quota.setNvidiaGpuLimits(Integer.parseInt(data.get(LIMITS_NVIDIA_COM_GPU))); } - setPodsCount( - data.get("count/pods") == null ? null : Integer.valueOf(data.get("count/pods"))); + quota.setPodsCount( + data.get(COUNT_PODS) == null ? null : Integer.valueOf(data.get(COUNT_PODS))); + return quota; } public String getMemoryRequests() {