From 4a9ff1edcdbb808334f5b1a4c608ac465f70e6df Mon Sep 17 00:00:00 2001 From: gibbleyg <63262871+gibbleyg@users.noreply.github.com> Date: Tue, 5 Dec 2023 13:27:25 -0800 Subject: [PATCH] Linter, Test fixes, and Container Updates (#620) * Fix lint errors * Fix test errors by removing six * Fix lint errors * Update containers with new helm/kubectl versions (#621) * Update helm, kubectl and container dependencies * Fix docker install * Revert changes to test docker container --- marketplace/deployer_envsubst_base/Dockerfile | 6 +++--- marketplace/deployer_helm_base/Dockerfile | 11 +++++------ marketplace/deployer_util/bash_util.py | 10 +++++----- marketplace/deployer_util/make_dns1123_name.py | 4 +--- marketplace/deployer_util/process_helm_hooks.py | 7 ++++--- marketplace/dev/Dockerfile | 17 +++++++---------- 6 files changed, 25 insertions(+), 30 deletions(-) diff --git a/marketplace/deployer_envsubst_base/Dockerfile b/marketplace/deployer_envsubst_base/Dockerfile index fce79580..93ce6651 100644 --- a/marketplace/deployer_envsubst_base/Dockerfile +++ b/marketplace/deployer_envsubst_base/Dockerfile @@ -1,4 +1,4 @@ -FROM marketplace.gcr.io/google/ubuntu2004 +FROM marketplace.gcr.io/google/ubuntu2204 RUN apt-get update && apt-get install -y --no-install-recommends \ bash \ @@ -17,7 +17,7 @@ RUN pip3 install \ pyyaml \ six -RUN for full_version in 1.26.5 1.26.5 1.27.2; \ +RUN for full_version in 1.27.3 1.28.3; \ do \ version=${full_version%.*} \ && mkdir -p /opt/kubectl/$version \ @@ -25,7 +25,7 @@ RUN for full_version in 1.26.5 1.26.5 1.27.2; \ https://storage.googleapis.com/kubernetes-release/release/v$full_version/bin/linux/amd64/kubectl \ && chmod 755 /opt/kubectl/$version/kubectl; \ done; -RUN ln -s /opt/kubectl/1.26 /opt/kubectl/default +RUN ln -s /opt/kubectl/1.27 /opt/kubectl/default COPY marketplace/deployer_envsubst_base/* /bin/ COPY marketplace/deployer_util/* /bin/ diff --git a/marketplace/deployer_helm_base/Dockerfile b/marketplace/deployer_helm_base/Dockerfile index 8ce7b1c6..b9f1d59a 100644 --- a/marketplace/deployer_helm_base/Dockerfile +++ b/marketplace/deployer_helm_base/Dockerfile @@ -1,4 +1,4 @@ -FROM marketplace.gcr.io/google/ubuntu2004 +FROM marketplace.gcr.io/google/ubuntu2204 RUN apt-get update && apt-get install -y --no-install-recommends \ bash \ @@ -14,10 +14,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN pip3 install \ wheel \ pyOpenSSL \ - pyyaml \ - six + pyyaml -RUN for full_version in 1.26.5 1.26.5 1.27.2; \ +RUN for full_version in 1.27.3 1.28.3; \ do \ version=${full_version%.*} \ && mkdir -p /opt/kubectl/$version \ @@ -25,11 +24,11 @@ RUN for full_version in 1.26.5 1.26.5 1.27.2; \ https://storage.googleapis.com/kubernetes-release/release/v$full_version/bin/linux/amd64/kubectl \ && chmod 755 /opt/kubectl/$version/kubectl; \ done; -RUN ln -s /opt/kubectl/1.26 /opt/kubectl/default +RUN ln -s /opt/kubectl/1.27 /opt/kubectl/default RUN mkdir -p /bin/helm-downloaded \ && wget -q -O /bin/helm-downloaded/helm.tar.gz \ - https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz \ + https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ && tar -zxvf /bin/helm-downloaded/helm.tar.gz -C /bin/helm-downloaded \ && mv /bin/helm-downloaded/linux-amd64/helm /bin/ \ && rm -rf /bin/helm-downloaded diff --git a/marketplace/deployer_util/bash_util.py b/marketplace/deployer_util/bash_util.py index 91d65faf..a64cfd46 100644 --- a/marketplace/deployer_util/bash_util.py +++ b/marketplace/deployer_util/bash_util.py @@ -16,8 +16,6 @@ import shlex import subprocess -import six - class CommandException(Exception): @@ -39,7 +37,11 @@ def __init__(self, cmd, print_call=False, print_result=False): parsedCmd = shlex.split(cmd) self._process = subprocess.Popen( - parsedCmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + parsedCmd, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8') self._exitcode = None self._output = None self._print_call = print_call @@ -48,8 +50,6 @@ def __init__(self, cmd, print_call=False, print_result=False): def _run(self): self._output, error_message = self._process.communicate() - self._output = six.ensure_str(self._output, 'utf-8') - error_message = six.ensure_str(error_message, 'utf-8') self._exitcode = self._process.returncode if self._print_result: result = (f"result: {self._exitcode}\n" diff --git a/marketplace/deployer_util/make_dns1123_name.py b/marketplace/deployer_util/make_dns1123_name.py index 00c3c264..cdfdb73c 100755 --- a/marketplace/deployer_util/make_dns1123_name.py +++ b/marketplace/deployer_util/make_dns1123_name.py @@ -18,8 +18,6 @@ import re from argparse import ArgumentParser -import six - _PROG_HELP = """ Turns a name into a proper DNS-1123 subdomain, with limitations. """ @@ -54,7 +52,7 @@ def limit_name(name, length=127): result = result[:length - 5] # Hash and get the first 4 characters of the hash. m = hashlib.sha256() - m.update(six.ensure_binary(name, 'utf-8')) + m.update(name.encode('utf-8')) h4sh = m.hexdigest()[:4] result = '{}-{}'.format(result, h4sh) return result diff --git a/marketplace/deployer_util/process_helm_hooks.py b/marketplace/deployer_util/process_helm_hooks.py index aca424a2..5bd0ccb0 100755 --- a/marketplace/deployer_util/process_helm_hooks.py +++ b/marketplace/deployer_util/process_helm_hooks.py @@ -25,7 +25,8 @@ _HELM_HOOK_KEY = 'helm.sh/hook' _HOOK_SUCCESS = ['test-success', 'test'] -_HOOK_FAILURE = ['test-failure', 'test'] +_HOOK_FAILURE = ['test-failure'] + def main(): parser = ArgumentParser() @@ -44,13 +45,13 @@ def main(): helm_hook = deep_get(resource, "metadata", "annotations", _HELM_HOOK_KEY) if helm_hook is None: filtered_resources.append(resource) - elif _HOOK_SUCCESS.index(helm_hook): + elif helm_hook in _HOOK_SUCCESS: if args.deploy_tests: annotations = deep_get(resource, "metadata", "annotations") del annotations[_HELM_HOOK_KEY] annotations[GOOGLE_CLOUD_TEST] = "test" filtered_resources.append(resource) - elif _HOOK_FAILURE.index(helm_hook): + elif helm_hook in _HOOK_FAILURE: if args.deploy_tests: raise Exception("Helm hook {} is not supported".format(helm_hook)) else: diff --git a/marketplace/dev/Dockerfile b/marketplace/dev/Dockerfile index 5d7c48bb..4bf4f175 100644 --- a/marketplace/dev/Dockerfile +++ b/marketplace/dev/Dockerfile @@ -1,4 +1,4 @@ -FROM marketplace.gcr.io/google/ubuntu2004 +FROM marketplace.gcr.io/google/ubuntu2204 RUN apt-get update && apt-get install -y --no-install-recommends \ apt-transport-https \ @@ -13,7 +13,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gnupg \ python3 \ python3-pip \ - python-is-python3 \ && rm -rf /var/lib/apt/lists/* RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ @@ -24,10 +23,9 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.c RUN pip3 install \ wheel \ pyOpenSSL \ - pyyaml \ - six + pyyaml -RUN for full_version in 1.26.5 1.26.5 1.27.2; \ +RUN for full_version in 1.27.3 1.28.3; \ do \ version=${full_version%.*} \ && mkdir -p /opt/kubectl/$version \ @@ -35,16 +33,15 @@ RUN for full_version in 1.26.5 1.26.5 1.27.2; \ https://storage.googleapis.com/kubernetes-release/release/v$full_version/bin/linux/amd64/kubectl \ && chmod 755 /opt/kubectl/$version/kubectl; \ done; -RUN ln -s /opt/kubectl/1.26 /opt/kubectl/default +RUN ln -s /opt/kubectl/1.27 /opt/kubectl/default -RUN echo "deb [signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu focal edge" | tee /etc/apt/sources.list.d/docker.list \ +RUN echo "deb [signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" | tee /etc/apt/sources.list.d/docker.list \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key --keyring /usr/share/keyrings/docker.gpg add - \ && apt-get -y update \ - && apt-get -y install docker-ce=5:19.03.13~3-0~ubuntu-focal - + && apt-get -y install docker-ce RUN mkdir -p /bin/helm-downloaded \ && wget -q -O /bin/helm-downloaded/helm.tar.gz \ - https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz \ + https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ && tar -zxvf /bin/helm-downloaded/helm.tar.gz -C /bin/helm-downloaded \ && mv /bin/helm-downloaded/linux-amd64/helm /bin/ \ && rm -rf /bin/helm-downloaded