Add automated installation instruction tests

.github/workflows/install-instructions-test.yml

@@ -0,0 +1,28 @@
+name: Install Instructions
+on: workflow_dispatch
+
+jobs:
+  install-instructions:
+    env:
+      BASE_URL: https://docs.konghq.com
+    strategy:
+      fail-fast: false
+      matrix:
+        distro: [ubuntu, rhel, amazon-linux, debian, centos]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Run Install tests
+        env:
+          DISTRO: ${{ matrix.distro }}
+          CONTINUE_ON_ERROR: 1
+          IGNORE_SKIPS: 1
+          EXPECTED_FAILURES_EXIT_CODE: 0
+        run: |
+          cd tools/install-tester
+          npm ci
+          node index.js

tools/install-tester/.gitignore

@@ -0,0 +1,133 @@
+output
+
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*

tools/install-tester/README.md

@@ -0,0 +1,59 @@
+# Install Tester
+
+This is a tool that scrapes our installation documentation and uses `docker` to ensure that the packages install as expected.
+
+## Why does it exist?
+
+Our install docs should always work. That's where everyone starts on the docs.
+
+Why now? I needed to test the following matrix of combinations whilst moving to a new package hosting platform:
+
+(2.6, 2.7, 2.8, 3.0, 3.1, 3.2, 3.3) x (Ubuntu, Debian, RHEL, Amazon Linux, CentOS[2.x only]) x (OSS, EE) x (Package, Repository) = 140 combinations
+
+## How it works
+
+1. Read `kong_versions.yml` to build a list of versions to test
+2. Use `ce-version` and `ee-version` to set expected output (the test runs `kong version` to ensure the package installed correctly)
+3. For each version, loop through each OS
+4. Fetch the docs URL for that OS. Extract the codeblocks for (OSS, EE) x (Package, Repository)
+5. Create a docker container for each OS
+6. Set up the initial environment (install `sudo`, create a non-root user etc)
+7. Run the commands that were fetched earlier
+8. Run `kong version`
+9. Assert that the message matches the expected version from `kong_versions.yml`
+
+## How to run it
+
+Run `make run` in another terminal.
+
+In the current directory:
+
+> This will run a LOT of tests (all 140, in fact). There is a way to run fewer tests that which will be shown later
+
+```bash
+npm ci
+node index.js
+```
+
+You can run a specific `DISTRO`, `METHOD`, `VERSION` or `PACKAGE` e.g.:
+
+* `DISTRO=ubuntu,rhel METHOD=package VERSION=2.8.x,3.3.x PACKAGE=enterprise node index.js` runs a specific set of tests
+* `DISTRO=amazon-linux node index.js` runs all versions, both EE and OSS, via package and repository install
+
+You may also want to add `IGNORE_SKIPS=1 CONTINUE_ON_ERROR=1` if running multiple tests.
+
+## Debugging
+
+When a test runs, it writes a file in `./output` in the format `version-distro-package-method.txt`. This contains the exact command run at the top, plus any output. You can run the command then attach to the running docker image to explore the environment
+
+If you see a failure, you can run that specific combination using the `ONLY` parameter:
+
+```bash
+ONLY=2.8.x/ubuntu/oss/package node index.js
+```
+
+## False positives
+
+Some of the failures are due to external hosting issues and not the docs site. The `expected-failures.yaml` file allows you to mark specific tests as expected to fail. 
+
+By default expected failures do not fail the build. Set `EXPECTED_FAILURES_EXIT_CODE=1` to make it a failure.

tools/install-tester/config/expected-failures.yaml

@@ -0,0 +1,3 @@
+"no": "failures expected"
+# Example format:
+# "2.8.x/ubuntu/enterprise/package": "Cloudsmith package contains OS version"

tools/install-tester/config/jobs.yaml

@@ -0,0 +1,22 @@
+- match: "3\\..*"
+  distros:
+    - ubuntu
+    - rhel
+    - amazon-linux
+    - debian
+  outputs:
+    enterprise: "Kong Enterprise {{ version }}"
+    oss: "{{ version }}"
+- match: "2\\..*"
+  distros:
+    - ubuntu
+    - rhel
+    - amazon-linux
+    - debian
+    - centos
+  skip:
+    - 2.8.x/centos/oss/package # No OSS package for 2.8
+    - 2.8.x/centos/oss/repository # No OSS package for 2.8
+  outputs:
+    enterprise: "Kong Enterprise {{ version }}"
+    oss: "{{ version }}"

tools/install-tester/config/setup.yaml

@@ -0,0 +1,34 @@
+ubuntu:
+  image: "ubuntu:jammy"
+  setup:
+    - apt-get update -y
+    - apt-get install -y curl gpg sudo lsb-release
+    - useradd tester -m -p password
+    - usermod -aG sudo tester
+    - "echo 'ALL            ALL = (ALL) NOPASSWD: ALL' > /etc/sudoers.d/tester"
+debian:
+  image: "debian:11"
+  setup:
+    - apt-get update -y
+    - apt-get install -y curl gpg sudo lsb-release
+    - useradd tester -m -p password
+    - usermod -aG sudo tester
+    - "echo 'ALL            ALL = (ALL) NOPASSWD: ALL' > /etc/sudoers.d/tester"
+rhel:
+  image: "registry.access.redhat.com/ubi7/ubi-init:latest"
+  setup:
+    - yum install -y curl gpg sudo
+    - useradd tester -m -p password
+    - "echo 'ALL            ALL = (ALL) NOPASSWD: ALL' > /etc/sudoers.d/tester"
+amazon-linux:
+  image: "amazonlinux:2"
+  setup:
+    - yum install -y curl gpg sudo shadow-utils util-linux
+    - useradd tester -m -p password
+    - "echo 'ALL            ALL = (ALL) NOPASSWD: ALL' > /etc/sudoers.d/tester"
+centos:
+  image: "centos:centos7"
+  setup:
+    - yum install -y curl gpg sudo
+    - useradd tester -m -p password
+    - "echo 'ALL            ALL = (ALL) NOPASSWD: ALL' > /etc/sudoers.d/tester"

tools/install-tester/execute-in-docker.js

@@ -0,0 +1,76 @@
+module.exports = async function (distro, steps) {
+  const Dockerode = require("dockerode");
+  const streams = require("memory-streams");
+  const fs = require("fs");
+  const yaml = require("js-yaml");
+
+  const config = yaml.load(fs.readFileSync("./config/setup.yaml", "utf8"));
+
+  const stdout = new streams.WritableStream();
+  const stderr = new streams.WritableStream();
+
+  const docker = new Dockerode({ socketPath: "/var/run/docker.sock" });
+
+  let setup = config[distro].setup;
+  if (!setup) {
+    throw new Error(`No setup found for ${distro}`);
+  }
+  setup = setup.join(" && ");
+
+  const asUser = `su tester -c 'cd ~ && ${steps
+    .join(" && ")
+    .replace("\n", " && ")} && kong version'`;
+
+  const completeString = `${setup} && ${asUser}`;
+
+  // Pull the image
+  await new Promise((resolve, reject) => {
+    docker.pull(
+      config[distro].image,
+      { platform: "linux/amd64" },
+      (err, stream) => {
+        if (err) {
+          return reject(err);
+        }
+
+        docker.modem.followProgress(stream, onFinished, onProgress);
+
+        function onFinished(err, output) {
+          if (err) {
+            return reject(err);
+          }
+          return resolve(output);
+        }
+        function onProgress(event) {}
+      },
+    );
+  });
+
+  return new Promise((resolve, reject) => {
+    docker.run(
+      config[distro].image,
+      ["bash", "-c", completeString],
+      [stdout, stderr],
+      { Tty: false, HostConfig: { AutoRemove: true }, platform: "linux/amd64" },
+      function (err, data, container) {
+        if (err) {
+          return reject(err);
+        }
+        const lines = stdout
+          .toString()
+          .split("\n")
+          .filter((l) => l);
+        const version = lines[lines.length - 1];
+        return resolve({
+          version,
+          stdout: stdout.toString(),
+          stderr: stderr.toString(),
+          jobConfig: {
+            image: config[distro].image,
+            commands: completeString,
+          },
+        });
+      },
+    );
+  });
+};