Fixing upgrade N-1 error

Signed-off-by: Tushar Tathgur <[email protected]>
antrea-io · Jul 26, 2023 · 1ee72aa · 1ee72aa
1 parent 3465564
commit 1ee72aa
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 14 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -146,6 +146,13 @@ jobs:
           docker tag antrea/theia-clickhouse-server:latest projects.registry.vmware.com/antrea/theia-clickhouse-server:latest
           docker load -i theia-manager.tar
           docker tag antrea/theia-manager:latest projects.registry.vmware.com/antrea/theia-manager:latest
+      - name: Remove Theia images tar files
+        run:  |
+          rm -rf clickhouse-server.tar
+          rm -rf clickhouse-monitor.tar
+          rm -rf spark-jobs.tar
+          rm -rf theia-manager.tar
+          df -h
       - name: Install Kind
         run: |
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -177,40 +184,63 @@ jobs:
   test-upgrade-from-N-1:
     name: Upgrade from Theia version N-1
     needs: 
+      - build-spark-jobs-image
       - build-clickhouse-monitor-image
-      - build-clickhouse-server-image 
+      - build-clickhouse-server-image
+      - build-theia-manager-image
     runs-on: [ubuntu-latest]
     steps:
       - name: Free disk space
         # https://github.com/actions/virtual-environments/issues/709
         run: |
+          df -h
           sudo apt-get clean
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo rm -rf "/usr/local/lib/android"
+          sudo apt-get autoremove
+          sudo apt-get clean
+          sudo apt-get autoclean
           df -h
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
+      - name: Download Spark jobs images from previous jobs
+        uses: actions/download-artifact@v3
+        with:
+          name: spark-jobs
       - name: Download ClickHouse monitor images from previous jobs
         uses: actions/download-artifact@v3
         with:
           name: clickhouse-monitor
-      - name: Load Theia image
-        run:  |
-          docker load -i clickhouse-monitor.tar
-          docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
       - name: Download ClickHouse server images from previous jobs
         uses: actions/download-artifact@v3
         with:
           name: clickhouse-server
+      - name: Download Theia Manager images from previous jobs
+        uses: actions/download-artifact@v3
+        with:
+          name: theia-manager
       - name: Load Theia image
         run:  |
+          docker load -i spark-jobs.tar
+          docker tag antrea/theia-spark-jobs:latest projects.registry.vmware.com/antrea/theia-spark-jobs:latest
+          docker load -i clickhouse-monitor.tar
+          docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
           docker load -i clickhouse-server.tar
           docker tag antrea/theia-clickhouse-server:latest projects.registry.vmware.com/antrea/theia-clickhouse-server:latest
+          docker load -i theia-manager.tar
+          docker tag antrea/theia-manager:latest projects.registry.vmware.com/antrea/theia-manager:latest
+      - name: Remove Theia images tar files
+        run:  |
+          rm -rf clickhouse-server.tar
+          rm -rf clickhouse-monitor.tar
+          rm -rf spark-jobs.tar
+          rm -rf theia-manager.tar
+          df -h
       - name: Install Kind
         run: |
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -247,6 +277,8 @@ jobs:
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo rm -rf "/usr/local/lib/android"
+          sudo apt-get autoremove
+          sudo apt-get autoclean
           df -h
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
@@ -256,18 +288,22 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: clickhouse-monitor
-      - name: Load Theia image
+      - name: Load Theia clickhouse-monitor image
         run:  |
           docker load -i clickhouse-monitor.tar
           docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
       - name: Download ClickHouse server images from previous jobs
         uses: actions/download-artifact@v3
         with:
           name: clickhouse-server
-      - name: Load Theia image
+      - name: Load Theia clickhouse-server image
         run:  |
           docker load -i clickhouse-server.tar
           docker tag antrea/theia-clickhouse-server:latest projects.registry.vmware.com/antrea/theia-clickhouse-server:latest
+      - name: Remove Theia images tar files
+        run:  |
+          rm -rf clickhouse-server.tar
+          rm -rf clickhouse-monitor.tar
       - name: Install Kind
         run: |
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64

diff --git a/build/images/Dockerfile.spark-jobs.ubuntu b/build/images/Dockerfile.spark-jobs.ubuntu
@@ -1,4 +1,4 @@
-FROM apache/spark-py:v3.4.0
+FROM spark:3.4.1-python3
 
 LABEL maintainer="Antrea <[email protected]>"
 LABEL description="A docker image to deploy policy recommendation and throughput anomaly detection Spark job."
@@ -8,15 +8,19 @@ USER root
 
 RUN apt-get --allow-releaseinfo-change update && \
     apt-get install -y --no-install-recommends wget ca-certificates && \
-    wget https://github.com/ClickHouse/clickhouse-jdbc/releases/download/v0.3.1/clickhouse-jdbc-0.3.1.jar -P /opt/spark/jars/
+    wget https://github.com/ClickHouse/clickhouse-jdbc/releases/download/v0.3.1/clickhouse-jdbc-0.3.1.jar -P /opt/spark/jars/ && \
+    rm -rf /var/cache/apt/* /var/lib/apt/lists/* && \
+    rm -rf /tmp/* && \
+    apt-get clean && \
+    apt-get autoremove
 
 COPY plugins/policy-recommendation/policy_recommendation_job.py /opt/spark/work-dir/policy_recommendation_job.py
 COPY plugins/policy-recommendation/policy_recommendation_utils.py /opt/spark/work-dir/policy_recommendation_utils.py
 COPY plugins/policy-recommendation/antrea_crd.py /opt/spark/work-dir/antrea_crd.py
 COPY plugins/anomaly-detection/anomaly_detection.py /opt/spark/work-dir/anomaly_detection.py
 COPY plugins/anomaly-detection/requirements.txt /opt/spark/work-dir/anomaly_detection_requirements.txt
 
-RUN pip3 install --upgrade pip && \
-    pip3 install pyyaml && \
-    pip3 install kubernetes && \
-    pip3 install -r /opt/spark/work-dir/anomaly_detection_requirements.txt
+RUN pip3 install --upgrade pip --no-cache-dir && \
+    pip3 install pyyaml --no-cache-dir && \
+    pip3 install kubernetes --no-cache-dir && \
+    pip3 install -r /opt/spark/work-dir/anomaly_detection_requirements.txt --no-cache-dir
diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh
@@ -195,6 +195,7 @@ function load_images {
   echo "load images"
   set +e
   for img in $IMAGES; do
+    df -h
     docker image inspect $img > /dev/null 2>&1
     if [[ $? -ne 0 ]]; then
       echoerr "docker image $img not found"
@@ -206,6 +207,8 @@ function load_images {
       continue
     fi
     echo "loaded image $img"
+    rm -rf $img
+    df -h
   done
   set -e
 }

diff --git a/ci/kind/test-upgrade-theia.sh b/ci/kind/test-upgrade-theia.sh
@@ -189,18 +189,25 @@ DOCKER_IMAGES=("registry.k8s.io/e2e-test-images/agnhost:2.29" \
                 "projects.registry.vmware.com/antrea/antrea-ubuntu:$ANTREA_FROM_TAG" \
                 "projects.registry.vmware.com/antrea/theia-clickhouse-monitor:$THEIA_FROM_TAG" \
                 "projects.registry.vmware.com/antrea/theia-clickhouse-server:$CLICKHOUSE_FROM_TAG" \
+                "projects.registry.vmware.com/antrea/theia-manager:$THEIA_FROM_TAG" \
                 "antrea/antrea-ubuntu:latest")
 
+# delete old images first
+docker system prune -f --filter "until=1h" || true > /dev/null
+
 for img in "${DOCKER_IMAGES[@]}"; do
     echo "Pulling $img"
     for i in `seq 3`; do
         docker pull $img > /dev/null && break
+        df -h
         sleep 1
     done
 done
 
 DOCKER_IMAGES+=("projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest\
-                 projects.registry.vmware.com/antrea/theia-clickhouse-server:latest")
+                 projects.registry.vmware.com/antrea/theia-clickhouse-server:latest\
+                 projects.registry.vmware.com/antrea/theia-manager:latest\
+                 projects.registry.vmware.com/antrea/theia-spark-jobs:latest")
 
 echo "Creating Kind cluster"
 IMAGES="${DOCKER_IMAGES[@]}"