Main update ci 20240319 (#589)

* update driver ci * update ci * update kmod ci * ci: update 1.7* kmod release artifacts name format
bytedance · Mar 27, 2024 · c037550 · c037550
1 parent 6003e21
commit c037550
Show file tree

Hide file tree

Showing 59 changed files with 947 additions and 405 deletions.
diff --git a/.github/workflows/Elkeid_driver_build.yml b/.github/workflows/Elkeid_driver_build.yml
diff --git a/.github/workflows/Elkeid_driver_release.yml b/.github/workflows/Elkeid_driver_release.yml
diff --git a/driver/build_script/gen_ci_from_dockerfiles.py b/driver/build_script/gen_ci_from_dockerfiles.py
@@ -7,7 +7,10 @@
 all_dockers_aarch64 = os.listdir(
     "driver/dockerfiles.aarch64")
 
-black_list = []
+black_list = [
+    "ubuntu1604-k410",
+    "ubuntu1604-k48",
+]
 white_list = []
 
 all_vms = []
@@ -25,12 +28,12 @@ def gen_job(vminfo):
     some_data = OrderedDict(
         {
             "runs-on": runs_on,
-            "continue-on-error": "true",
+            "continue-on-error": True,
             "steps": [
                 OrderedDict({
                     "uses": "actions/checkout@v3",
                     "with": {
-                        "submodules": "false"
+                        "submodules": False
                     }
                 }),
                 OrderedDict({
@@ -43,19 +46,19 @@ def gen_job(vminfo):
                 }),
                 OrderedDict({
                     "name": "Set up Docker Buildx "+vmname,
-                    "uses": "docker/setup-buildx-action@v2",
+                    "uses": "docker/setup-buildx-action@v3",
                     "with": {
                         "config": "/etc/buildkitd.toml",
                     }
                 }) if aarch.endswith("aarch64") else OrderedDict({
                     "name": "Set up Docker Buildx "+vmname,
-                    "uses": "docker/setup-buildx-action@v2"
+                    "uses": "docker/setup-buildx-action@v3"
                 }),
 
                 OrderedDict({
                     "name": "Build "+vmname,
                     "uses": "docker/build-push-action@v3",
-                    "timeout-minutes": "300",
+                    "timeout-minutes": 420,
                     "with": {
                         "context": ".",
                         "file": dockerpath + "/Dockerfile."+vmname,
@@ -118,19 +121,19 @@ def gen_job(vminfo):
         "on": {
             "push": {
                 "paths":[
-                    "'.github/workflows/Elkeid_driver_build.yml'",
-                    "'.github/workflows/Elkeid_driver_release.yml'",
-                    "'driver/LKM/**'",
-                    "'driver/build_script/aarch64/**'",
-                    "'driver/build_script/x86_64/**'",
-                    "'driver/dockerfiles.aarch64/**'",
-                    "'driver/dockerfiles.x86_64/**'",
+                    ".github/workflows/Elkeid_driver_build.yml",
+                    ".github/workflows/Elkeid_driver_release.yml",
+                    "driver/LKM/**",
+                    "driver/build_script/aarch64/**",
+                    "driver/build_script/x86_64/**",
+                    "driver/dockerfiles.aarch64/**",
+                    "driver/dockerfiles.x86_64/**",
                 ],
                 "branches": [
                     "main",
                 ]
             },
-            "schedule": ["cron : '0 3 1 * *'"]
+            "schedule": [{"cron":"0 3 1 * *"}]
         }
     }
 )
@@ -141,7 +144,7 @@ def gen_job(vminfo):
         "on": {
             "push": {
                 "tags": [
-                    "'v*'"
+                    "v*"
                 ]
             },
         }
@@ -168,6 +171,24 @@ def gen_job(vminfo):
                     "prerelease": False,
                 }
             }),
+            OrderedDict({
+                    "uses": "actions/checkout@v3",
+                    "with": {
+                        "submodules": False
+                    }
+                }),
+            OrderedDict({
+                "name": "Setup Version",
+                "run": 'echo "KMOD_VERSION=$(cat driver/LKM/src/init.c | grep MODULE_VERSION | awk -F \'\"\' \'{print $2}\')" >> "$GITHUB_ENV"'
+            }),
+            OrderedDict({
+                "name": "Setup output Version format",
+                "run": 'echo "KMOD_RELEASE_PREFIX=$(echo $KMOD_VERSION | sed -e "s|\\.|\\_|g")" >> "$GITHUB_ENV"'
+            }),
+            OrderedDict({
+                "name": "Setup output Version format",
+                "run": 'echo "KO_TAR_XZ=\"$KMOD_RELEASE_PREFIX\"_elkeid_driver_ko_$(date +\"%Y%m%d\").tar.xz" >> "$GITHUB_ENV"'
+            }),
             OrderedDict({
                 "uses": "actions/download-artifact@v3",
                 "with": {
@@ -182,22 +203,22 @@ def gen_job(vminfo):
 
             OrderedDict({
                 "name": "Prepare artifact 2-1 ko",
-                "run": "BUILD_VERSION=$(cat LKM/src/init.c | grep MODULE_VERSION | awk -F '\"' '{print $2}') mv -f ~/all_elkeid_drivers/*/*$BUILD_VERSION*.ko elkeid_driver/ko || true"
+                "run": "mv -f ~/all_elkeid_drivers/*/*$KMOD_VERSION*.ko elkeid_driver/ko || true"
             }),
 
             OrderedDict({
                 "name": "Prepare artifact 2-2 sign",
-                "run": "BUILD_VERSION=$(cat LKM/src/init.c | grep MODULE_VERSION | awk -F '\"' '{print $2}') mv -f ~/all_elkeid_drivers/*/*$BUILD_VERSION*.sign elkeid_driver/ko || true"
+                "run": "mv -f ~/all_elkeid_drivers/*/*$KMOD_VERSION*.sign elkeid_driver/ko || true"
             }),
 
             OrderedDict({
                 "name": "Prepare artifact 2-3 log",
-                "run": "BUILD_VERSION=$(cat LKM/src/init.c | grep MODULE_VERSION | awk -F '\"' '{print $2}') mv -f ~/all_elkeid_drivers/*/*$BUILD_VERSION*.log elkeid_driver/log || true"
+                "run": "mv -f ~/all_elkeid_drivers/*/*$KMOD_VERSION*.log elkeid_driver/log || true"
             }),
 
             OrderedDict({
                 "name": "Pack artifact",
-                "run": "tar -C elkeid_driver -cJf elkeid_driver_ko.tar.xz ko"
+                "run": "tar -C elkeid_driver -cJf \"$KO_TAR_XZ\" ko"
             }),
 
             OrderedDict({
@@ -221,16 +242,16 @@ def gen_job(vminfo):
             }),
 
             OrderedDict({
-                "name": "Upload Release Asset ",
+                "name": "Upload Release Asset",
                 "id": "upload-release-asset",
                 "uses": "actions/upload-release-asset@v1",
                 "env": {
                         "GITHUB_TOKEN": "${{secrets.GITHUB_TOKEN}}"
                 },
                 "with": {
                     "upload_url": "${{steps.create_release.outputs.upload_url}}",
-                    "asset_path": "./elkeid_driver_ko.tar.xz",
-                    "asset_name": "elkeid_driver_ko.tar.xz",
+                    "asset_path": "${{env.KO_TAR_XZ}}",
+                    "asset_name": "${{env.KO_TAR_XZ}}",
                     "asset_content_type": "application/x-tar"
                 },
             })
@@ -280,11 +301,17 @@ def setup_yaml():
 setup_yaml()
 
 with open(".github/workflows/Elkeid_driver_build.yml", "w") as f:
-    config_data = yaml.dump(yaml_cfg_build, default_flow_style=False)
-    config_data = config_data.replace("'", "")
+    config_data = yaml.dump(yaml_cfg_build, 
+    default_style=None,
+    default_flow_style=False)
+    config_data = config_data.replace("'on'", "on")
+    config_data = config_data.replace("'[self-hosted,linux,ARM64]'", "[self-hosted,linux,ARM64]")
     f.write(config_data)
 
 with open(".github/workflows/Elkeid_driver_release.yml", "w") as f:
-    config_data = yaml.dump(yaml_cfg_release, default_flow_style=False)
-    config_data = config_data.replace("'", "")
+    config_data = yaml.dump(yaml_cfg_release, 
+    default_style=None,
+    default_flow_style=False)
+    config_data = config_data.replace("'on'", "on")
+    config_data = config_data.replace("'[self-hosted,linux,ARM64]'", "[self-hosted,linux,ARM64]")
     f.write(config_data)
diff --git a/driver/dockerfiles.aarch64/Dockerfile.rhel8 b/driver/dockerfiles.aarch64/Dockerfile.rhel8
@@ -1,13 +1,15 @@
 FROM arm64v8/oraclelinux:8 AS rhel8
 
 
-RUN dnf install -y wget perl gcc make tree elfutils-libelf-devel yumdownloader gnutls-utils pciutils-libs; 
+RUN dnf install -y wget perl gcc make tree elfutils-libelf-devel openssl-devel openssl; 
 RUN dnf groupinstall -y  "Development Tools"; 
+
 RUN dnf install -y centos-release-scl
 RUN dnf install -y devtoolset-8
+
 RUN rm -rf /root/headers || true
 RUN mkdir /root/headers
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/aarch64/batch_compile_el7.sh
+RUN bash ./build_script/aarch64/batch_compile_el8.sh
diff --git a/driver/dockerfiles.x86_64/Dockerfile.aliyun_linux2 b/driver/dockerfiles.x86_64/Dockerfile.aliyun_linux2
@@ -12,4 +12,9 @@ RUN rpm --force -i /root/headers/*.rpm  || true
 ADD . /elkeid
 WORKDIR /elkeid/driver
 RUN bash ./build_script/x86_64/batch_compile.sh
+RUN dnf remove -y kernel-devel || true
 
+
+FROM kulukami/aliyun_linux2:latest
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.aliyun_linux3 b/driver/dockerfiles.x86_64/Dockerfile.aliyun_linux3
@@ -13,4 +13,9 @@ RUN rpm --force -i /root/headers/*.rpm  || true
 ADD . /elkeid
 WORKDIR /elkeid/driver
 RUN bash ./build_script/x86_64/batch_compile.sh
+RUN dnf remove -y kernel-devel || true
 
+
+FROM kulukami/aliyun_linux3:latest
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.almalinux8 b/driver/dockerfiles.x86_64/Dockerfile.almalinux8
@@ -25,3 +25,9 @@ ADD . /elkeid
 WORKDIR /elkeid/driver
 RUN bash ./build_script/x86_64/batch_compile.sh
 RUN rm -rf /root/headers/*.rpm 
+RUN dnf remove -y kernel-devel || true
+
+
+FROM almalinux:8
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.almalinux9 b/driver/dockerfiles.x86_64/Dockerfile.almalinux9
@@ -23,4 +23,9 @@ ADD . /elkeid
 WORKDIR /elkeid/driver
 RUN bash ./build_script/x86_64/batch_compile.sh
 RUN rm -rf /root/headers/*.rpm 
+RUN dnf remove -y kernel-devel || true
 
+
+FROM almalinux/9-base
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.amazonlinux1 b/driver/dockerfiles.x86_64/Dockerfile.amazonlinux1
@@ -11,4 +11,10 @@ RUN rm -f  /root/kernel-devel-*amzn1.x86_64.rpm
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN yum remove -y kernel-devel || true
+
+
+FROM amazonlinux:1
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.amazonlinux2 b/driver/dockerfiles.x86_64/Dockerfile.amazonlinux2
@@ -11,4 +11,10 @@ RUN rm -f  /root/kernel-devel-*amzn2.x86_64.rpm
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN yum remove -y kernel-devel || true
+
+
+FROM amazonlinux:2
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.amazonlinux2_510 b/driver/dockerfiles.x86_64/Dockerfile.amazonlinux2_510
@@ -12,4 +12,10 @@ RUN rm -f  /root/kernel-devel-*amzn2.x86_64.rpm
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN yum remove -y kernel-devel || true
+
+
+FROM amazonlinux:1
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.amazonlinux2_54 b/driver/dockerfiles.x86_64/Dockerfile.amazonlinux2_54
@@ -12,4 +12,10 @@ RUN rm -f  /root/kernel-devel-*amzn2.x86_64.rpm
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN yum remove -y kernel-devel || true
+
+
+FROM amazonlinux:2
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.anolis8 b/driver/dockerfiles.x86_64/Dockerfile.anolis8
@@ -25,3 +25,9 @@ ADD . /elkeid
 WORKDIR /elkeid/driver
 RUN bash ./build_script/x86_64/batch_compile.sh
 RUN rm -rf /root/headers/*.rpm 
+RUN dnf remove -y kernel-devel || true
+
+
+FROM almalinux:8
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.debian10 b/driver/dockerfiles.x86_64/Dockerfile.debian10
@@ -16,4 +16,10 @@ RUN apt clean all
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN apt-get remove -y linux-headers* || true
+
+
+FROM debian:buster
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.debian11 b/driver/dockerfiles.x86_64/Dockerfile.debian11
@@ -12,4 +12,10 @@ RUN apt clean all
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN apt-get remove -y linux-headers* || true
+
+
+FROM debian:bullseye
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.debian8 b/driver/dockerfiles.x86_64/Dockerfile.debian8
@@ -13,15 +13,21 @@ RUN apt install --yes --force-yes gcc build-essential libelf-dev;
 RUN apt install --yes --force-yes linux-compiler-gcc* || true
 RUN apt install --yes --force-yes linux-kbuild*;
 RUN apt install --yes --force-yes \
-linux-headers-4.9.0-0.bpo.12-all-amd64 \
-linux-headers-4.9.0-0.bpo.11-all-amd64 \
-linux-headers-4.9-amd64 \
-linux-headers-3.16.0-6-all-amd64 \
-linux-headers-3.16.0-11-amd64 \
-linux-headers-amd64 || true
+    linux-headers-4.9.0-0.bpo.12-all-amd64 \
+    linux-headers-4.9.0-0.bpo.11-all-amd64 \
+    linux-headers-4.9-amd64 \
+    linux-headers-3.16.0-6-all-amd64 \
+    linux-headers-3.16.0-11-amd64 \
+    linux-headers-amd64 || true
 
 
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN apt-get remove -y linux-headers* || true
+
+
+FROM debian:jessie
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]
diff --git a/driver/dockerfiles.x86_64/Dockerfile.debian9 b/driver/dockerfiles.x86_64/Dockerfile.debian9
@@ -7,8 +7,8 @@ RUN sed -i '/deb.debian.org/s/^/#/g' /etc/apt/sources.list
 
 RUN echo "deb http://snapshot.debian.org/archive/debian/20220622T000000Z stretch-backports main" >> /etc/apt/sources.list
 
-
-RUN apt-get -o Acquire::Check-Valid-Until=false update
+RUN rm -rf /var/lib/apt/lists/partial
+RUN apt-get -o Acquire::Check-Valid-Until=false -o Acquire::CompressionTypes::Order::=gz update
 RUN apt-get -o Acquire::Check-Valid-Until=false install -y apt-utils apt-transport-https ca-certificates debian-archive-keyring wget curl
 RUN apt-get -o Acquire::Check-Valid-Until=false install -y gcc build-essential libelf-dev; 
 RUN apt-get -o Acquire::Check-Valid-Until=false install -y linux-compiler-gcc* || true
@@ -25,4 +25,9 @@ RUN apt clean all
 
 ADD . /elkeid
 WORKDIR /elkeid/driver
-RUN bash ./build_script/x86_64/batch_compile.sh
+RUN bash ./build_script/x86_64/batch_compile.sh
+RUN apt-get remove -y linux-headers* || true
+
+FROM debian:stretch
+COPY --from=0 /ko_output /ko_output
+CMD [ "sh", "-c", "echo start" ]