From a68b6296f31c2abb63ba6f227d694f1328ccc6c8 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Wed, 15 May 2024 19:58:40 -0600 Subject: [PATCH 1/2] Fix ttys for 6.6 kernel: For some reason the 6.6 kernel has issues with ttys when they are not explicitly set in the linuxkit yaml. Signed-off-by: Jacob Weinstock --- linuxkit-templates/hook.template.yaml | 51 ++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/linuxkit-templates/hook.template.yaml b/linuxkit-templates/hook.template.yaml index 315c958d..8d74ba83 100644 --- a/linuxkit-templates/hook.template.yaml +++ b/linuxkit-templates/hook.template.yaml @@ -91,8 +91,49 @@ services: - /etc/motd:/etc/motd - /etc/os-release:/etc/os-release - /:/host_root + - /dev:/dev + - /dev/console:/dev/console + - /usr/bin/nerdctl:/usr/bin/nerdctl env: - INSECURE=true + devices: + - path: all + type: b + - path: "/dev/console" + type: c + major: 5 + minor: 1 + mode: "0666" + - path: "/dev/tty0" + type: c + major: 4 + minor: 0 + mode: "0666" + - path: "/dev/tty1" + type: c + major: 4 + minor: 1 + mode: "0666" + - path: "/dev/ttyS0" + type: c + major: 4 + minor: 64 + mode: "0666" + - path: "/dev/ttyS1" + type: c + major: 4 + minor: 65 + mode: "0666" + - path: "/dev/ttyAMA0" + type: c + major: 204 + minor: 64 + mode: "0666" + - path: "/dev/ttyAMA1" + type: c + major: 204 + minor: 65 + mode: "0666" - name: hook-docker image: "${HOOK_CONTAINER_DOCKER_IMAGE}" @@ -105,7 +146,7 @@ services: options: [ "rw", "nosuid", "noexec", "nodev", "relatime" ] destination: /sys/fs/cgroup binds.add: - #- /dev/console:/dev/console + - /dev/console:/dev/console - /dev:/dev - /etc/resolv.conf:/etc/resolv.conf - /lib/modules:/lib/modules @@ -117,6 +158,9 @@ services: - /var/run/images - /var/run/docker - /var/run/worker + devices: + - path: all + type: b - name: hook-bootkit image: "${HOOK_CONTAINER_BOOTKIT_IMAGE}" @@ -155,12 +199,11 @@ services: files: - path: etc/profile.d/local.sh contents: | - alias docker='ctr -n services.linuxkit tasks exec --tty --exec-id cmd hook-docker docker' - alias docker-shell='ctr -n services.linuxkit tasks exec --tty --exec-id shell hook-docker sh' + alias docker='nerdctl -n services.linuxkit exec -it hook-docker docker' + alias docker-shell='nerdctl -n services.linuxkit exec -it hook-docker sh' export PS1='HookOS ${HOOK_VERSION}:\w\$ ' # only print WARNING or higher kernel messages to console echo 4 > /proc/sys/kernel/printk - resize mode: "0644" - path: etc/motd From a96f105e3486b891c8761ff6e512432c0ec4f545 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Wed, 15 May 2024 19:59:47 -0600 Subject: [PATCH 2/2] Add nerdctl, update containerd config: The ctr command with the 6.6 kernel was having issues with character encoding or something. I was getting a lot of weird character when using it. Moving to nerdctl resolves this issue. Nerdctl is also a lot more user friendly in general. Signed-off-by: Jacob Weinstock --- images/containerd/Dockerfile | 12 + images/containerd/etc/containerd/config.toml | 294 ++++++++++++++++++- 2 files changed, 299 insertions(+), 7 deletions(-) diff --git a/images/containerd/Dockerfile b/images/containerd/Dockerfile index 080c5bcc..755514e6 100644 --- a/images/containerd/Dockerfile +++ b/images/containerd/Dockerfile @@ -1,5 +1,6 @@ FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e as builder +ARG TARGETPLATFORM # checkout and compile containerd # Update `FROM` in `pkg/containerd/Dockerfile`, `pkg/init/Dockerfile` and @@ -7,6 +8,7 @@ FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e as builder ENV CONTAINERD_REPO=https://github.com/containerd/containerd.git ENV CONTAINERD_COMMIT=v1.7.15 +ENV NERDCTL_VERSION=1.7.6 ENV GOPATH=/go RUN apk add go git RUN mkdir -p $GOPATH/src/github.com/containerd && \ @@ -18,14 +20,22 @@ RUN apk add --no-cache btrfs-progs-dev gcc libc-dev linux-headers make libseccom WORKDIR $GOPATH/src/github.com/containerd/containerd RUN make binaries EXTRA_FLAGS="-buildmode pie" EXTRA_LDFLAGS='-extldflags "-fno-PIC -static"' BUILDTAGS="static_build no_devmapper" +# install nerdctl +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then ARCHITECTURE=amd64; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCHITECTURE=arm64; else ARCHITECTURE=amd64; fi \ + && wget https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${ARCHITECTURE}.tar.gz \ + && tar -zxvf nerdctl-${NERDCTL_VERSION}-linux-${ARCHITECTURE}.tar.gz -C /usr/local/bin/ + RUN cp bin/containerd bin/ctr bin/containerd-shim bin/containerd-shim-runc-v2 /usr/bin/ RUN strip /usr/bin/containerd /usr/bin/ctr /usr/bin/containerd-shim /usr/bin/containerd-shim-runc-v2 +RUN mkdir -p /opt/containerd FROM scratch as containerd-dev ENTRYPOINT [] WORKDIR / COPY --from=builder /usr/bin/containerd /usr/bin/ctr /usr/bin/containerd-shim /usr/bin/containerd-shim-runc-v2 /usr/bin/ COPY --from=builder /go/src/github.com/containerd/containerd /go/src/github.com/containerd/containerd +COPY --from=builder /usr/local/bin/nerdctl /usr/bin/ +COPY --from=builder /opt/containerd/ /opt/containerd/ # Dockerfile to build linuxkit/containerd for linuxkit FROM linuxkit/alpine:146f540f25cd92ec8ff0c5b0c98342a9a95e479e as alpine @@ -39,6 +49,8 @@ FROM scratch ENTRYPOINT [] WORKDIR / COPY --from=containerd-dev /usr/bin/containerd /usr/bin/ctr /usr/bin/containerd-shim /usr/bin/containerd-shim-runc-v2 /usr/bin/ +COPY --from=containerd-dev /usr/bin/nerdctl /usr/bin/ +COPY --from=containerd-dev /opt/containerd/ /opt/containerd/ COPY --from=alpine /usr/share/zoneinfo/UTC /etc/localtime COPY --from=alpine /etc/init.d/ /etc/init.d/ COPY etc etc/ diff --git a/images/containerd/etc/containerd/config.toml b/images/containerd/etc/containerd/config.toml index fc6df22e..1cbd93d2 100644 --- a/images/containerd/etc/containerd/config.toml +++ b/images/containerd/etc/containerd/config.toml @@ -1,15 +1,295 @@ -state = "/run/containerd" +# default containerd configuration file, generated via `containerd config default` +disabled_plugins = [] +imports = [] +oom_score = 0 +plugin_dir = "" +required_plugins = [] root = "/var/lib/containerd" -disabled_plugins = ["cri"] +state = "/run/containerd" +temp = "" +version = 2 + +[cgroup] + path = "" + +[debug] + address = "" + format = "" + gid = 0 + level = "" + uid = 0 [grpc] address = "/run/containerd/containerd.sock" - uid = 0 gid = 0 - -[debug] - address = "/run/containerd/debug.sock" - level = "info" + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + tcp_address = "" + tcp_tls_ca = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 [metrics] address = "" + grpc_histogram = false + +[plugins] + + [plugins."io.containerd.gc.v1.scheduler"] + deletion_threshold = 0 + mutation_threshold = 100 + pause_threshold = 0.02 + schedule_delay = "0s" + startup_delay = "100ms" + + [plugins."io.containerd.grpc.v1.cri"] + cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] + device_ownership_from_security_context = false + disable_apparmor = false + disable_cgroup = false + disable_hugetlb_controller = true + disable_proc_mount = false + disable_tcp_service = true + drain_exec_sync_io_timeout = "0s" + enable_cdi = false + enable_selinux = false + enable_tls_streaming = false + enable_unprivileged_icmp = false + enable_unprivileged_ports = false + ignore_image_defined_volumes = false + image_pull_progress_timeout = "5m0s" + max_concurrent_downloads = 3 + max_container_log_line_size = 16384 + netns_mounts_under_state_dir = false + restrict_oom_score_adj = false + sandbox_image = "registry.k8s.io/pause:3.8" + selinux_category_range = 1024 + stats_collect_period = 10 + stream_idle_timeout = "4h0m0s" + stream_server_address = "127.0.0.1" + stream_server_port = "0" + systemd_cgroup = false + tolerate_missing_hugetlb_controller = true + unset_seccomp_profile = "" + + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + ip_pref = "" + max_conf_num = 1 + setup_serially = false + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + disable_snapshot_annotations = true + discard_unpacked_layers = false + ignore_blockio_not_enabled_errors = false + ignore_rdt_not_enabled_errors = false + no_pivot = false + snapshotter = "overlayfs" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + sandbox_mode = "" + snapshotter = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + sandbox_mode = "podsandbox" + snapshotter = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "" + CriuImagePath = "" + CriuPath = "" + CriuWorkPath = "" + IoGid = 0 + IoUid = 0 + NoNewKeyring = false + NoPivotRoot = false + Root = "" + ShimCgroup = "" + SystemdCgroup = false + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + sandbox_mode = "" + snapshotter = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".image_decryption] + key_model = "node" + + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "" + + [plugins."io.containerd.grpc.v1.cri".registry.auths] + + [plugins."io.containerd.grpc.v1.cri".registry.configs] + + [plugins."io.containerd.grpc.v1.cri".registry.headers] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + + [plugins."io.containerd.internal.v1.tracing"] + sampling_ratio = 1.0 + service_name = "containerd" + + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + + [plugins."io.containerd.nri.v1.nri"] + disable = true + disable_connections = false + plugin_config_path = "/etc/nri/conf.d" + plugin_path = "/opt/nri/plugins" + plugin_registration_timeout = "5s" + plugin_request_timeout = "2s" + socket_path = "/var/run/nri/nri.sock" + + [plugins."io.containerd.runtime.v1.linux"] + no_shim = false + runtime = "runc" + runtime_root = "" + shim = "containerd-shim" + shim_debug = false + + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + sched_core = false + + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + + [plugins."io.containerd.service.v1.tasks-service"] + blockio_config_file = "" + rdt_config_file = "" + + [plugins."io.containerd.snapshotter.v1.aufs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.blockfile"] + fs_type = "" + mount_options = [] + root_path = "" + scratch_file = "" + + [plugins."io.containerd.snapshotter.v1.btrfs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.devmapper"] + async_remove = false + base_image_size = "" + discard_blocks = false + fs_options = "" + fs_type = "" + pool_name = "" + root_path = "" + + [plugins."io.containerd.snapshotter.v1.native"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.overlayfs"] + mount_options = [] + root_path = "" + sync_remove = false + upperdir_label = false + + [plugins."io.containerd.snapshotter.v1.zfs"] + root_path = "" + + [plugins."io.containerd.tracing.processor.v1.otlp"] + endpoint = "" + insecure = false + protocol = "" + + [plugins."io.containerd.transfer.v1.local"] + config_path = "" + max_concurrent_downloads = 3 + max_concurrent_uploaded_layers = 3 + + [[plugins."io.containerd.transfer.v1.local".unpack_config]] + differ = "" + platform = "linux/amd64" + snapshotter = "overlayfs" + +[proxy_plugins] + +[stream_processors] + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"] + accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar" + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"] + accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar+gzip" + +[timeouts] + "io.containerd.timeout.bolt.open" = "0s" + "io.containerd.timeout.metrics.shimstats" = "2s" + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[ttrpc] + address = "" + gid = 0 + uid = 0