Github runner on k8s

cOS configuration

cOS config to have k3s + some persistency layer (to place it inside /oem/config.yaml in a cOS VM):

name: "Default user"
stages:
   boot:
     - name: "Hostname and setup"
       hostname: "cos-node-1"
       commands:
       - echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6
       dns:
        nameservers:
        - X
#       commands:
#       - passwd -d root
   network:
     - name: "Setup SSH keys"
       authorized_keys:
         admincos:
         - github:mudler
         root:
         - github:mudler
     - if: '[ -z "$(blkid -L COS_SYSTEM || true)" ]'
       name: "Load persisted ssh fingerprint"
       commands:
       - |
            # load ssh fingerprint
            if [ ! -d /usr/local/etc/ssh ]; then
            systemctl start sshd
            mkdir /usr/local/etc/ssh || true
            for i in /etc/ssh/*.pub; do cp -rf $i /usr/local/etc/ssh; done
            fi
     - name: "Setup k3s"
       if: '[ -z "$(blkid -L COS_SYSTEM || true)" ]'
       directories:
       - path: "/usr/local/bin"
         permissions: 0755
         owner: 0
         group: 0
       commands:
       - |
            curl -sfL https://get.k3s.io | \
            INSTALL_K3S_VERSION="v1.20.4+k3s1" \
            INSTALL_K3S_EXEC="--tls-san additional-outside-ip" \
            INSTALL_K3S_SELINUX_WARN="true" \
            sh -
   initramfs:
     - if: '[ -z "$(blkid -L COS_SYSTEM || true)" ]'
       name: "Persist"
       commands:
       - |
            target=/usr/local/.cos-state

            # Always want the latest update of systemd conf from the image
            mkdir -p ${target}/etc/systemd/
            rsync -av /etc/systemd/ ${target}/etc/systemd/
            # Only populate ssh conf once
            if [ ! -e ${target}/etc/ssh ]; then
            mkdir -p ${target}/etc/ssh/
            rsync -av /etc/ssh/ ${target}/etc/ssh/
            fi
            # make /tmp tmpfs
            cp -f /usr/share/systemd/tmp.mount ${target}/etc/systemd/system/
            # undo /home /opt mount from cos immutable-rootfs module
            sed -i '/overlay \/home /d' /etc/fstab
            sed -i '/overlay \/opt /d' /etc/fstab
            umount /home
            umount /opt
            # setup directories as persistent
            for i in root opt home var/lib/rancher var/lib/kubelet etc/systemd etc/rancher etc/ssh usr/libexec; do
            mkdir -p ${target}/$i /$i
            mount ${target}/$i /$i -t none -o bind
            done
            # This is hidden so that if you run some selinux label checking or relabeling the bind
            # mount won't screw up things.  If you have two files at different paths they will get
            # labeled with two different labels.
            mkdir -p ${target}/empty
            mount ${target}/empty ${target} -o bind,ro
            # persist machine-id
            if [ -s /usr/local/etc/machine-id ]; then
            cat /usr/local/etc/machine-id > /etc/machine-id
            else
            mkdir -p /usr/local/etc
            cp /etc/machine-id /usr/local/etc
            fi
            # ensure /var/log/journal exists so it's labeled correctly
            mkdir -p /var/log/journal
     - name: "Setup users"
       users: 
          admincos: 
            homedir: "/home/admincos"
     - name: "groups"
       ensure_entities: 
       - entity: |
                 kind: "group"
                 group_name: "wheel"
                 password: "x"
                 gid: 1020
                 users: "admincos"
       files: 
       - path: "/etc/sudoers.d/wheel"
         owner: 0
         group: 0
         permission: 0600   
         content: |
                   %wheel ALL=(ALL) NOPASSWD: ALL
       - path: "/etc/modprobe.d/ipv6.conf"
         owner: 0
         group: 0
         permission: 0664  
         content: |
                    alias net-pf-10 off
                    alias ipv6 off
                    options ipv6 disable_ipv6=1

GH runners deployment

This is a deployment that can be used to deploy github runners in a k8s cluster, mind to replace TOKEN and OWNER accordingly, and the resources allocated to the docker daemon as well.

---
apiVersion: v1
kind: Namespace
metadata:
  name: github-runner
---
apiVersion: v1
kind: Secret
metadata:
  name: github-token
  namespace: github-runner
stringData:
  token: TOKEN
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: github-runner
  namespace: github-runner
  labels:
    app: github-runner
spec:
  replicas: 1
  selector:
    matchLabels:
      app: github-runner
  template:
    metadata:
      labels:
        app: github-runner
    spec:
#      nodeSelector:
#        kubernetes.io/hostname: 
      automountServiceAccountToken: false
      volumes:
      - name: modules
        hostPath:
          path: /lib/modules
          type: Directory
      - name: cgroup
        hostPath:
          path: /sys/fs/cgroup
          type: Directory
      - name: dind-storage
        emptyDir: {}
      - name: dind-certs
        emptyDir: {}
      containers:
      - name: dind
        image: docker:19.03-dind
        imagePullPolicy: Always
        env:
          - name: DOCKER_TLS_CERTDIR
            value: "/certs"
        resources:
          requests:
            cpu: 1m
            memory: 50Mi
        securityContext:
          privileged: true
        volumeMounts:
          - mountPath: /lib/modules
            name: modules
            readOnly: true
          - mountPath: /sys/fs/cgroup
            name: cgroup
          - name: dind-storage
            mountPath: /var/lib/docker
          - name: dind-certs
            mountPath: /certs
      - name: github-runner
        image: quay.io/mudler/github-runner:latest
        securityContext:
          privileged: true
        env:
        - name: DOCKER_CERT_PATH
          value: "/certs/client"
        - name: DOCKER_TLS_VERIFY
          value: "1"
        - name: KUBEHOST
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: DOCKER_HOST
          value: "tcp://$(KUBEHOST):2376"
        - name: GITHUB_OWNER
          value: OWNER
        #- name: GITHUB_REPOSITORY
        #  value: your-repository # ooptional
        - name: RUNNER_TOKEN # stored separately in a Kubernetes secret
          valueFrom:
            secretKeyRef:
              name: github-token
              key: token
        volumeMounts:
        - name: dind-certs
          mountPath: /certs

For autoscaling, see also: https://github.com/actions-runner-controller/actions-runner-controller