From ced706a91936101759cc4a57208d1e880f75edb3 Mon Sep 17 00:00:00 2001
From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com>
Date: Mon, 10 Jun 2024 23:22:07 -0500
Subject: [PATCH 1/6] initial osquery docker sidecar and osquery local builds

---
 .../terraform/aws-tf-module/docker/.gitignore |  2 +
 .../terraform/aws-tf-module/docker/main.tf    | 48 +++++++++++---
 .../docker/osquery-docker.patch.tmpl          | 28 +++++++++
 .../terraform/aws-tf-module/free-ecs-hosts.tf | 36 ++++++-----
 .../dogfood/terraform/aws-tf-module/main.tf   | 63 +++++++++++++++++++
 terraform/byo-vpc/byo-db/byo-ecs/main.tf      |  1 +
 terraform/byo-vpc/byo-db/byo-ecs/variables.tf |  2 +
 terraform/byo-vpc/byo-db/variables.tf         |  2 +
 terraform/byo-vpc/variables.tf                |  2 +
 terraform/variables.tf                        |  2 +
 10 files changed, 160 insertions(+), 26 deletions(-)
 create mode 100644 infrastructure/dogfood/terraform/aws-tf-module/docker/.gitignore
 create mode 100644 infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl

diff --git a/infrastructure/dogfood/terraform/aws-tf-module/docker/.gitignore b/infrastructure/dogfood/terraform/aws-tf-module/docker/.gitignore
new file mode 100644
index 000000000000..b0bcff9fe7ed
--- /dev/null
+++ b/infrastructure/dogfood/terraform/aws-tf-module/docker/.gitignore
@@ -0,0 +1,2 @@
+osquery
+osquery-docker.patch
diff --git a/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf
index 46e50389574e..3989722968b1 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf
@@ -11,31 +11,61 @@ terraform {
   }
 }
 
-variable "osquery_tag" {
-  description = "The osquery tag to take from dockerhub to your ecr repo."
+variable "osquery_version" {
+  description = "The osquery version to push to your ecr repo."
   type        = string
 }
 
+variable "osquery_tags" {
+  description = "The tags that you wish to push among the built images"
+  type        = list(string)
+}
+
 variable "ecr_repo" {
   description = "The ecr repo to push to"
   type        = string
 }
 
-resource "docker_image" "dockerhub" {
-  name = "osquery/osquery:${var.osquery_tag}"
+resource "local_file" "osquery_patch" {
+  content         = templatefile("${path.module}/osquery-docker.patch.tmpl", { osquery_version = var.osquery_version })
+  filename        = "${path.module}/osuqery-docker.patch"
+  file_permission = "0644"
+}
+
+resource "null_resource" "build_osquery" {
+  triggers = {
+    osquery_version_changed = var.osquery_version
+    osquery_tags_changed    = sha256(jsonencode(var.osquery_tags))
+  }
+  provisioner "local-exec" {
+    working_dir = "${path.module}/osquery"
+    command     = <<-EOT
+      if [ "$(git remote -vvv | head -n1 | awk '{ print $2 }')" = "https://github.com/osquery/osquery.git" ]; then
+        git reset --hard
+        git pull
+      else
+        git clone https://github.com/osquery/osquery.git .
+      fi
+      git patch ${path.module}/osquery-docker.patch
+      cd tools/docker
+      ./build.sh
+    EOT
+  }
 }
 
 resource "docker_tag" "osquery" {
-  source_image = docker_image.dockerhub.name
+  for_each     = toset(var.osquery_tags)
+  source_image = "osquery/osquery:${each.key}"
   # We can't include the sha256 when pushing even if they match
-  target_image = "${var.ecr_repo}:${split("@sha256", var.osquery_tag)[0]}"
+  target_image = "${var.ecr_repo}:${each.key}"
 }
 
 resource "docker_registry_image" "osquery" {
-  name          = docker_tag.osquery.target_image
+  for_each      = toset(var.osquery_tags)
+  name          = docker_tag.osquery[each.key].target_image
   keep_remotely = true
 }
 
-output "ecr_image" {
-  value = docker_tag.osquery.target_image
+output "ecr_images" {
+  value = { for docker_tag in docker_tag.osquery : split(":", docker_tag.target_image)[1] => docker_tag.target_image }
 }
diff --git a/infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl b/infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl
new file mode 100644
index 000000000000..f3103e36ade5
--- /dev/null
+++ b/infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl
@@ -0,0 +1,28 @@
+diff --git a/tools/docker/build.sh b/tools/docker/build.sh
+index 9efba34f6..34ecd8a4e 100755
+--- a/tools/docker/build.sh
++++ b/tools/docker/build.sh
+@@ -6,7 +6,7 @@ build_deb() {
+ 
+     TAG=$(echo $OS | sed 's/://g')
+ 
+-    docker build -f deb-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/deb/osquery_$${VERSION}-1.linux_amd64.deb --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
++    docker build --platform=linux/amd64 -f deb-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/deb/osquery_$${VERSION}-1.linux_amd64.deb --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
+ }
+ 
+ build_rpm() {
+@@ -15,11 +15,11 @@ build_rpm() {
+ 
+     TAG=$(echo $OS | sed 's/://g')
+ 
+-    docker build -f rpm-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/rpm/osquery-$${VERSION}-1.linux.x86_64.rpm --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
++    docker build --platform=linux/amd64 -f rpm-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/rpm/osquery-$${VERSION}-1.linux.x86_64.rpm --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
+ }
+ 
+-versions='5.2.3'
+-deb_platforms='ubuntu:16.04 ubuntu:18.04 ubuntu:20.04 ubuntu:22.04 debian:10 debian:9 debian:8 debian:7'
++versions='${osquery_version}'
++deb_platforms='ubuntu:16.04 ubuntu:18.04 ubuntu:20.04 ubuntu:22.04 ubuntu:24.04 debian:10 debian:9 debian:8 debian:7'
+ rpm_platforms='centos:6 centos:7 centos:8'
+ 
+ for v in $$versions
diff --git a/infrastructure/dogfood/terraform/aws-tf-module/free-ecs-hosts.tf b/infrastructure/dogfood/terraform/aws-tf-module/free-ecs-hosts.tf
index 8021f5892ca9..0e3ebac81dcb 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/free-ecs-hosts.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/free-ecs-hosts.tf
@@ -1,16 +1,18 @@
 ## Linux hosts in ECS
 
 locals {
+  osquery_version = "5.12.2"
   osquery_hosts = {
-    "5.8.2-ubuntu22.04@sha256:b77c7b06c4d7f2a3c58cc3a34e51fffc480e97795fb3c75cb1dc1cf3709e3dc6" = "Skys-laptop"
-    "5.8.2-ubuntu20.04@sha256:3496ffd0ad570c88a9f405e6ef517079cfeed6ce405b9d22db4dc5ef6ed3faac" = "Cloud-City-server"
-    "5.8.2-ubuntu18.04@sha256:372575e876c218dde3c5c0e24fd240d193800fca9b314e94b4ad4e6e22006c9b" = "Mists-laptop"
-    "5.8.2-ubuntu16.04@sha256:112655c42951960d8858c116529fb4c64951e4cf2e34cb7c08cd599a009025bb" = "Ethers-laptop"
-    "5.8.2-debian10@sha256:de29337896aac89b2b03c7642805859d3fb6d52e5dc08230f987bbab4eeba9c5"    = "Breezes-laptop"
-    "5.8.2-debian9@sha256:47e46c19cebdf0dc704dd0061328856bda7e1e86b8c0fefdd6f78bd092c6200e"     = "Aero-server"
-    "5.8.2-centos8@sha256:88a8adde80bd3b1b257e098bc6e41b6afea840f60033653dcb9fe984f36b0f97"     = "Stratuss-laptop"
-    "5.8.2-centos7@sha256:ff251de4935b80a91c5fc1ac352aebdab9a6bbbf5bda1aaada8e26d22b50202d"     = "Zephyrs-Laptop"
-    "5.8.2-centos6@sha256:b56736be8436288d3fbd2549ec6165e0588cd7197e91600de4a2f00f1df28617"     = "Halo-server"
+    "${local.osquery_version}-ubuntu24.04" = "Atmosphere-database"
+    "${local.osquery_version}-ubuntu22.04" = "Skys-laptop"
+    "${local.osquery_version}-ubuntu20.04" = "Cloud-City-server"
+    "${local.osquery_version}-ubuntu18.04" = "Mists-laptop"
+    "${local.osquery_version}-ubuntu16.04" = "Ethers-laptop"
+    "${local.osquery_version}-debian10"    = "Breezes-laptop"
+    "${local.osquery_version}-debian9"     = "Aero-server"
+    "${local.osquery_version}-centos8"     = "Stratuss-laptop"
+    "${local.osquery_version}-centos7"     = "Zephyrs-Laptop"
+    "${local.osquery_version}-centos6"     = "Halo-server"
   }
 
 }
@@ -123,10 +125,10 @@ provider "docker" {
 }
 
 module "osquery_docker" {
-  for_each    = local.osquery_hosts
-  source      = "./docker"
-  ecr_repo    = aws_ecr_repository.osquery.repository_url
-  osquery_tag = each.key
+  source          = "./docker"
+  ecr_repo        = aws_ecr_repository.osquery.repository_url
+  osquery_version = local.osquery_version
+  osquery_tags    = keys(local.osquery_hosts)
 }
 
 resource "random_uuid" "osquery" {
@@ -135,7 +137,7 @@ resource "random_uuid" "osquery" {
 
 resource "aws_ecs_task_definition" "osquery" {
   for_each = local.osquery_hosts
-  // e.g. 5-8-2-ubuntu22-04 to match naming requirements 
+  // e.g. ${osquery_version}-ubuntu22-04 to match naming requirements
   family                   = "osquery-${replace(split("@sha256", each.key)[0], ".", "-")}"
   network_mode             = "awsvpc"
   requires_compatibilities = ["FARGATE"]
@@ -147,7 +149,7 @@ resource "aws_ecs_task_definition" "osquery" {
     [
       {
         name        = "osquery"
-        image       = module.osquery_docker[each.key].ecr_image
+        image       = module.osquery_docker.ecr_images[each.key]
         cpu         = 256
         memory      = 512
         mountPoints = []
@@ -215,8 +217,8 @@ resource "aws_ecs_task_definition" "osquery" {
 
 resource "aws_ecs_service" "osquery" {
   for_each = local.osquery_hosts
-  # Name must match ^[A-Za-z-_]+$ e.g. 5-8-2-ubuntu22-04
-  name            = "osquery_${replace(split("@sha256", each.key)[0], ".", "-")}"
+  # Name must match ^[A-Za-z-_]+$ e.g. 5.12.2-ubuntu22-04
+  name            = "osquery_${replace(each.key, ".", "-")}"
   launch_type     = "FARGATE"
   cluster         = module.free.byo-db.byo-ecs.service.cluster
   task_definition = aws_ecs_task_definition.osquery[each.key].arn
diff --git a/infrastructure/dogfood/terraform/aws-tf-module/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
index f8a055fff29f..5a68426e5e6c 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/main.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
@@ -101,6 +101,7 @@ module "main" {
     family = local.customer
     cpu    = 1024
     mem    = 4096
+    pid_mode = "task"
     autoscaling = {
       min_capacity = 2
       max_capacity = 5
@@ -137,6 +138,68 @@ module "main" {
     #   container_name   = "fleet"
     #   container_port   = 8080
     # }]
+    sidecars = [
+      {
+        name        = "osquery"
+        image       = module.osquery_docker.ecr_images["${local.osquery_version}-ubuntu24.04"]
+        cpu         = 256
+        memory      = 512
+        mountPoints = []
+        volumesFrom = []
+        essential   = true
+        ulimits = [
+          {
+            softLimit = 999999,
+            hardLimit = 999999,
+            name      = "nofile"
+          }
+        ]
+        networkMode = "awsvpc"
+        logConfiguration = {
+          logDriver = "awslogs"
+          options = {
+            awslogs-group         = local.customer
+            awslogs-region        = "us-east-2"
+            awslogs-stream-prefix = "osquery"
+          }
+        }
+        secrets = [
+          {
+            name      = "ENROLL_SECRET"
+            valueFrom = aws_secretsmanager_secret.osquery_enroll.arn
+          }
+        ]
+        workingDirectory = "/",
+        command = [
+          "osqueryd",
+          "--tls_hostname=dogfood.fleetdm.com",
+          "--force=true",
+          # Ensure that the host identifier remains the same between invocations
+          # "--host_identifier=specified",
+          # "--specified_identifier=${random_uuid.osquery[each.key].result}",
+          "--verbose=true",
+          "--tls_dump=true",
+          "--enroll_secret_env=ENROLL_SECRET",
+          "--enroll_tls_endpoint=/api/osquery/enroll",
+          "--config_plugin=tls",
+          "--config_tls_endpoint=/api/osquery/config",
+          "--config_refresh=10",
+          "--disable_distributed=false",
+          "--distributed_plugin=tls",
+          "--distributed_interval=10",
+          "--distributed_tls_max_attempts=3",
+          "--distributed_tls_read_endpoint=/api/osquery/distributed/read",
+          "--distributed_tls_write_endpoint=/api/osquery/distributed/write",
+          "--logger_plugin=tls",
+          "--logger_tls_endpoint=/api/osquery/log",
+          "--logger_tls_period=10",
+          "--disable_carver=false",
+          "--carver_start_endpoint=/api/osquery/carve/begin",
+          "--carver_continue_endpoint=/api/osquery/carve/block",
+          "--carver_block_size=8000000",
+        ]
+      }
+    ]
   }
   alb_config = {
     name = local.customer
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/main.tf b/terraform/byo-vpc/byo-db/byo-ecs/main.tf
index fabf20b413d1..d5950ad452c7 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/main.tf
+++ b/terraform/byo-vpc/byo-db/byo-ecs/main.tf
@@ -60,6 +60,7 @@ resource "aws_ecs_task_definition" "backend" {
   execution_role_arn       = aws_iam_role.execution.arn
   cpu                      = var.fleet_config.cpu
   memory                   = var.fleet_config.mem
+  pid_mode                 = var.fleet_config.pid_mode
   container_definitions = jsonencode(
     concat([
       {
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
index 47a8f4a61b6a..ea474195ea3b 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
+++ b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
@@ -13,6 +13,7 @@ variable "fleet_config" {
   type = object({
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
+    pid_mode                     = optional(string, null)
     image                        = optional(string, "fleetdm/fleet:v4.51.0")
     family                       = optional(string, "fleet")
     sidecars                     = optional(list(any), [])
@@ -97,6 +98,7 @@ variable "fleet_config" {
   default = {
     mem                          = 512
     cpu                          = 256
+    pid_mode                     = null
     image                        = "fleetdm/fleet:v4.51.0"
     family                       = "fleet"
     sidecars                     = []
diff --git a/terraform/byo-vpc/byo-db/variables.tf b/terraform/byo-vpc/byo-db/variables.tf
index db2132225d9d..2f9af039079d 100644
--- a/terraform/byo-vpc/byo-db/variables.tf
+++ b/terraform/byo-vpc/byo-db/variables.tf
@@ -74,6 +74,7 @@ variable "fleet_config" {
   type = object({
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
+    pid_mode                     = optional(string, null)
     image                        = optional(string, "fleetdm/fleet:v4.51.0")
     family                       = optional(string, "fleet")
     sidecars                     = optional(list(any), [])
@@ -172,6 +173,7 @@ variable "fleet_config" {
   default = {
     mem                          = 512
     cpu                          = 256
+    pid_mode                     = null
     image                        = "fleetdm/fleet:v4.51.0"
     family                       = "fleet"
     sidecars                     = []
diff --git a/terraform/byo-vpc/variables.tf b/terraform/byo-vpc/variables.tf
index 6fd1789b2391..8c114b1684a3 100644
--- a/terraform/byo-vpc/variables.tf
+++ b/terraform/byo-vpc/variables.tf
@@ -167,6 +167,7 @@ variable "fleet_config" {
   type = object({
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
+    pid_mode                     = optional(string, null)
     image                        = optional(string, "fleetdm/fleet:v4.51.0")
     family                       = optional(string, "fleet")
     sidecars                     = optional(list(any), [])
@@ -265,6 +266,7 @@ variable "fleet_config" {
   default = {
     mem                          = 512
     cpu                          = 256
+    pid_mode                     = null
     image                        = "fleetdm/fleet:v4.51.0"
     family                       = "fleet"
     sidecars                     = []
diff --git a/terraform/variables.tf b/terraform/variables.tf
index 7b58e7fbbfb9..8a49af91b9b2 100644
--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@@ -215,6 +215,7 @@ variable "fleet_config" {
   type = object({
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
+    pid_mode                     = optional(string, null)
     image                        = optional(string, "fleetdm/fleet:v4.51.0")
     family                       = optional(string, "fleet")
     sidecars                     = optional(list(any), [])
@@ -313,6 +314,7 @@ variable "fleet_config" {
   default = {
     mem                          = 512
     cpu                          = 256
+    pid_mode                     = null
     image                        = "fleetdm/fleet:v4.51.0"
     family                       = "fleet"
     sidecars                     = []

From c108ac7541dbe91e5ace104598175b131ca3c5dc Mon Sep 17 00:00:00 2001
From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com>
Date: Mon, 10 Jun 2024 23:34:56 -0500
Subject: [PATCH 2/6] fix

---
 .../terraform/aws-tf-module/docker/osquery-docker.patch.tmpl    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl b/infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl
index f3103e36ade5..2ba4208e4725 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl
+++ b/infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl
@@ -25,4 +25,4 @@ index 9efba34f6..34ecd8a4e 100755
 +deb_platforms='ubuntu:16.04 ubuntu:18.04 ubuntu:20.04 ubuntu:22.04 ubuntu:24.04 debian:10 debian:9 debian:8 debian:7'
  rpm_platforms='centos:6 centos:7 centos:8'
  
- for v in $$versions
+ for v in $versions

From 5675052709e87f547d5036b997fb575b7b72af8d Mon Sep 17 00:00:00 2001
From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com>
Date: Wed, 12 Jun 2024 01:39:36 -0500
Subject: [PATCH 3/6] fix osquery builds

---
 .../dogfood/terraform/aws-tf-module/docker/main.tf     | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf
index 3989722968b1..a2d1655ed2e7 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf
@@ -28,25 +28,28 @@ variable "ecr_repo" {
 
 resource "local_file" "osquery_patch" {
   content         = templatefile("${path.module}/osquery-docker.patch.tmpl", { osquery_version = var.osquery_version })
-  filename        = "${path.module}/osuqery-docker.patch"
+  filename        = "${path.module}/osquery-docker.patch"
   file_permission = "0644"
 }
 
 resource "null_resource" "build_osquery" {
+  depends_on = [local_file.osquery_patch]
   triggers = {
     osquery_version_changed = var.osquery_version
     osquery_tags_changed    = sha256(jsonencode(var.osquery_tags))
   }
   provisioner "local-exec" {
-    working_dir = "${path.module}/osquery"
+    working_dir = "${path.module}"
     command     = <<-EOT
+      mkdir -p osquery
+      cd osquery
       if [ "$(git remote -vvv | head -n1 | awk '{ print $2 }')" = "https://github.com/osquery/osquery.git" ]; then
         git reset --hard
         git pull
       else
         git clone https://github.com/osquery/osquery.git .
       fi
-      git patch ${path.module}/osquery-docker.patch
+      git apply ../osquery-docker.patch
       cd tools/docker
       ./build.sh
     EOT
@@ -54,6 +57,7 @@ resource "null_resource" "build_osquery" {
 }
 
 resource "docker_tag" "osquery" {
+  depends_on   = [null_resource.build_osquery]
   for_each     = toset(var.osquery_tags)
   source_image = "osquery/osquery:${each.key}"
   # We can't include the sha256 when pushing even if they match

From 199b6aa6f99e34e9f58f9df9cbc02bb20fbb7e7c Mon Sep 17 00:00:00 2001
From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com>
Date: Wed, 12 Jun 2024 02:52:39 -0500
Subject: [PATCH 4/6] Enable osquery sidecar for dogfood

---
 .github/workflows/dogfood-deploy.yml          |  1 +
 .../dogfood/terraform/aws-tf-module/main.tf   | 66 ++++++++++++++++---
 terraform/byo-vpc/byo-db/byo-ecs/iam.tf       |  4 +-
 terraform/byo-vpc/byo-db/byo-ecs/main.tf      |  4 +-
 terraform/byo-vpc/byo-db/byo-ecs/variables.tf |  4 ++
 terraform/byo-vpc/byo-db/variables.tf         |  4 ++
 terraform/byo-vpc/variables.tf                |  4 ++
 terraform/variables.tf                        |  4 ++
 8 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/dogfood-deploy.yml b/.github/workflows/dogfood-deploy.yml
index d13d2f4761fa..39f6983824eb 100644
--- a/.github/workflows/dogfood-deploy.yml
+++ b/.github/workflows/dogfood-deploy.yml
@@ -31,6 +31,7 @@ env:
   TF_VAR_elastic_url: ${{ secrets.ELASTIC_APM_SERVER_URL }}
   TF_VAR_elastic_token: ${{ secrets.ELASTIC_APM_SECRET_TOKEN }}
   TF_VAR_geolite2_license: ${{ secrets.MAXMIND_LICENSE }}
+  TF_VAR_dogfood_sidecar_enroll_secret: ${{ secrets.DOGFOOD_SERVERS_CANARY_ENROLL_SECRET }}
 
 permissions:
   id-token: write
diff --git a/infrastructure/dogfood/terraform/aws-tf-module/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
index 5a68426e5e6c..f7678d8ba2d4 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/main.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
@@ -40,6 +40,7 @@ variable "fleet_calendar_periodicity" {
   default     = "30s"
   description = "The refresh period for the calendar integration."
 }
+variable "dogfood_sidecar_enroll_secret" {}
 
 data "aws_caller_identity" "current" {}
 
@@ -68,7 +69,8 @@ locals {
 }
 
 module "main" {
-  source          = "github.com/fleetdm/fleet//terraform?ref=tf-mod-root-v1.8.0"
+  # source = "github.com/fleetdm/fleet//terraform?ref=tf-mod-root-v1.8.0"
+  source          = "../../../../terraform"
   certificate_arn = module.acm.acm_certificate_arn
   vpc = {
     name = local.customer
@@ -97,10 +99,12 @@ module "main" {
     cluster_name = local.customer
   }
   fleet_config = {
-    image  = local.geolite2_image
-    family = local.customer
-    cpu    = 1024
-    mem    = 4096
+    image    = local.geolite2_image
+    family   = local.customer
+    task_cpu = 2048
+    task_mem = 5120
+    cpu      = 1024
+    mem      = 4096
     pid_mode = "task"
     autoscaling = {
       min_capacity = 2
@@ -121,7 +125,7 @@ module "main" {
       }
     }
     extra_iam_policies           = concat(module.firehose-logging.fleet_extra_iam_policies, module.osquery-carve.fleet_extra_iam_policies, module.ses.fleet_extra_iam_policies)
-    extra_execution_iam_policies = concat(module.mdm.extra_execution_iam_policies, [aws_iam_policy.sentry.arn]) #, module.saml_auth_proxy.fleet_extra_execution_policies)
+    extra_execution_iam_policies = concat(module.mdm.extra_execution_iam_policies, [aws_iam_policy.sentry.arn, aws_iam_policy.osquery_sidecar.arn]) #, module.saml_auth_proxy.fleet_extra_execution_policies)
     extra_environment_variables = merge(
       module.mdm.extra_environment_variables,
       module.firehose-logging.fleet_extra_environment_variables,
@@ -142,8 +146,8 @@ module "main" {
       {
         name        = "osquery"
         image       = module.osquery_docker.ecr_images["${local.osquery_version}-ubuntu24.04"]
-        cpu         = 256
-        memory      = 512
+        cpu         = 1024
+        memory      = 1024
         mountPoints = []
         volumesFrom = []
         essential   = true
@@ -166,7 +170,7 @@ module "main" {
         secrets = [
           {
             name      = "ENROLL_SECRET"
-            valueFrom = aws_secretsmanager_secret.osquery_enroll.arn
+            valueFrom = aws_secretsmanager_secret.dogfood_sidecar_enroll_secret.arn
           }
         ]
         workingDirectory = "/",
@@ -532,3 +536,47 @@ module "vuln-processing" {
     prefix = module.main.byo-vpc.byo-db.byo-ecs.fleet_config.awslogs.prefix
   }
 }
+
+resource "aws_secretsmanager_secret" "dogfood_sidecar_enroll_secret" {
+  name = "dogfood-sidecar-enroll-secret"
+}
+
+resource "aws_secretsmanager_secret_version" "dogfood_sidecar_enroll_secret" {
+  secret_id     = aws_secretsmanager_secret.dogfood_sidecar_enroll_secret.id
+  secret_string = var.dogfood_sidecar_enroll_secret
+}
+
+data "aws_iam_policy_document" "osquery_sidecar" {
+  statement {
+    actions = [
+      "ecr:BatchCheckLayerAvailability",
+      "ecr:BatchGetImage",
+      "ecr:GetDownloadUrlForLayer",
+      "ecr:GetAuthorizationToken"
+    ]
+    resources = ["*"]
+  }
+  statement {
+    actions = [ #tfsec:ignore:aws-iam-no-policy-wildcards
+      "kms:Encrypt*",
+      "kms:Decrypt*",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:Describe*"
+    ]
+    resources = [aws_kms_key.osquery.arn]
+  }
+  statement {
+    actions = [ #tfsec:ignore:aws-iam-no-policy-wildcards
+      "secretsmanager:GetSecretValue"
+    ]
+    resources = [aws_secretsmanager_secret.dogfood_sidecar_enroll_secret.arn]
+
+  }
+}
+
+resource "aws_iam_policy" "osquery_sidecar" {
+  name        = "osquery-sidecar-policy"
+  description = "IAM policy that Osquery sidecar containers use to define access to AWS resources"
+  policy      = data.aws_iam_policy_document.osquery_sidecar.json
+}
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/iam.tf b/terraform/byo-vpc/byo-db/byo-ecs/iam.tf
index d1f00302e261..974dd8766fbe 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/iam.tf
+++ b/terraform/byo-vpc/byo-db/byo-ecs/iam.tf
@@ -21,8 +21,8 @@ data "aws_iam_policy_document" "assume_role" {
 data "aws_iam_policy_document" "fleet-execution" {
   // allow fleet application to obtain the database password from secrets manager
   statement {
-    effect    = "Allow"
-    actions   = ["secretsmanager:GetSecretValue"]
+    effect  = "Allow"
+    actions = ["secretsmanager:GetSecretValue"]
     resources = [
       var.fleet_config.database.password_secret_arn,
       aws_secretsmanager_secret.fleet_server_private_key.arn
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/main.tf b/terraform/byo-vpc/byo-db/byo-ecs/main.tf
index d5950ad452c7..4e466398f9c8 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/main.tf
+++ b/terraform/byo-vpc/byo-db/byo-ecs/main.tf
@@ -58,8 +58,8 @@ resource "aws_ecs_task_definition" "backend" {
   requires_compatibilities = ["FARGATE"]
   task_role_arn            = var.fleet_config.iam_role_arn == null ? aws_iam_role.main[0].arn : var.fleet_config.iam_role_arn
   execution_role_arn       = aws_iam_role.execution.arn
-  cpu                      = var.fleet_config.cpu
-  memory                   = var.fleet_config.mem
+  cpu                      = var.fleet_config.task_cpu == null ? var.fleet_config.cpu : var.fleet_config.task_cpu
+  memory                   = var.fleet_config.task_mem == null ? var.fleet_config.mem : var.fleet_config.task_mem
   pid_mode                 = var.fleet_config.pid_mode
   container_definitions = jsonencode(
     concat([
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
index ea474195ea3b..8986ac15e07b 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
+++ b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf
@@ -11,6 +11,8 @@ variable "vpc_id" {
 
 variable "fleet_config" {
   type = object({
+    task_mem                     = optional(number, null)
+    task_cpu                     = optional(number, null)
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
     pid_mode                     = optional(string, null)
@@ -96,6 +98,8 @@ variable "fleet_config" {
     })
   })
   default = {
+    task_mem                     = null
+    task_cpu                     = null
     mem                          = 512
     cpu                          = 256
     pid_mode                     = null
diff --git a/terraform/byo-vpc/byo-db/variables.tf b/terraform/byo-vpc/byo-db/variables.tf
index 2f9af039079d..f7daea241b92 100644
--- a/terraform/byo-vpc/byo-db/variables.tf
+++ b/terraform/byo-vpc/byo-db/variables.tf
@@ -72,6 +72,8 @@ variable "ecs_cluster" {
 
 variable "fleet_config" {
   type = object({
+    task_mem                     = optional(number, null)
+    task_cpu                     = optional(number, null)
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
     pid_mode                     = optional(string, null)
@@ -171,6 +173,8 @@ variable "fleet_config" {
     })
   })
   default = {
+    task_mem                     = null
+    task_cpu                     = null
     mem                          = 512
     cpu                          = 256
     pid_mode                     = null
diff --git a/terraform/byo-vpc/variables.tf b/terraform/byo-vpc/variables.tf
index 8c114b1684a3..f1459f152ce0 100644
--- a/terraform/byo-vpc/variables.tf
+++ b/terraform/byo-vpc/variables.tf
@@ -165,6 +165,8 @@ variable "ecs_cluster" {
 
 variable "fleet_config" {
   type = object({
+    task_mem                     = optional(number, null)
+    task_cpu                     = optional(number, null)
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
     pid_mode                     = optional(string, null)
@@ -264,6 +266,8 @@ variable "fleet_config" {
     })
   })
   default = {
+    task_mem                     = null
+    task_cpu                     = null
     mem                          = 512
     cpu                          = 256
     pid_mode                     = null
diff --git a/terraform/variables.tf b/terraform/variables.tf
index 8a49af91b9b2..be7d389b70ad 100644
--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@@ -213,6 +213,8 @@ variable "ecs_cluster" {
 
 variable "fleet_config" {
   type = object({
+    task_mem                     = optional(number, null)
+    task_cpu                     = optional(number, null)
     mem                          = optional(number, 4096)
     cpu                          = optional(number, 512)
     pid_mode                     = optional(string, null)
@@ -312,6 +314,8 @@ variable "fleet_config" {
     })
   })
   default = {
+    task_mem                     = null
+    task_cpu                     = null
     mem                          = 512
     cpu                          = 256
     pid_mode                     = null

From ae398d2d18348e7c6922c237e8bff17c7c9efca2 Mon Sep 17 00:00:00 2001
From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com>
Date: Wed, 12 Jun 2024 02:54:06 -0500
Subject: [PATCH 5/6] Pre-populate a future version of the root module

---
 infrastructure/dogfood/terraform/aws-tf-module/main.tf | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/infrastructure/dogfood/terraform/aws-tf-module/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
index f7678d8ba2d4..18f6b1129b0e 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/main.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
@@ -69,8 +69,7 @@ locals {
 }
 
 module "main" {
-  # source = "github.com/fleetdm/fleet//terraform?ref=tf-mod-root-v1.8.0"
-  source          = "../../../../terraform"
+  source = "github.com/fleetdm/fleet//terraform?ref=tf-mod-root-v1.9.0"
   certificate_arn = module.acm.acm_certificate_arn
   vpc = {
     name = local.customer

From c0c98f95d59df47fe99d130031bc9da18dd5e40d Mon Sep 17 00:00:00 2001
From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com>
Date: Wed, 12 Jun 2024 11:24:38 -0500
Subject: [PATCH 6/6] Improve sidecar support for vuln processing

---
 .../dogfood/terraform/aws-tf-module/main.tf         |  4 +++-
 .../addons/external-vuln-scans/.terraform.lock.hcl  |  1 +
 terraform/addons/external-vuln-scans/README.md      |  4 ++++
 terraform/addons/external-vuln-scans/main.tf        |  7 +++++--
 terraform/addons/external-vuln-scans/outputs.tf     |  2 +-
 terraform/addons/external-vuln-scans/variables.tf   | 13 +++++++++++++
 6 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/infrastructure/dogfood/terraform/aws-tf-module/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
index 18f6b1129b0e..f6dd44b68bcd 100644
--- a/infrastructure/dogfood/terraform/aws-tf-module/main.tf
+++ b/infrastructure/dogfood/terraform/aws-tf-module/main.tf
@@ -521,7 +521,7 @@ module "geolite2" {
 }
 
 module "vuln-processing" {
-  source                              = "github.com/fleetdm/fleet//terraform/addons/external-vuln-scans?ref=tf-mod-addon-external-vuln-scans-v2.1.0"
+  source                              = "github.com/fleetdm/fleet//terraform/addons/external-vuln-scans?ref=tf-mod-addon-external-vuln-scans-v2.2.0"
   ecs_cluster                         = module.main.byo-vpc.byo-db.byo-ecs.service.cluster
   execution_iam_role_arn              = module.main.byo-vpc.byo-db.byo-ecs.execution_iam_role_arn
   subnets                             = module.main.byo-vpc.byo-db.byo-ecs.service.network_configuration[0].subnets
@@ -529,6 +529,8 @@ module "vuln-processing" {
   fleet_config                        = module.main.byo-vpc.byo-db.byo-ecs.fleet_config
   task_role_arn                       = module.main.byo-vpc.byo-db.byo-ecs.iam_role_arn
   fleet_server_private_key_secret_arn = module.main.byo-vpc.byo-db.byo-ecs.fleet_server_private_key_secret_arn
+  vuln_processing_task_memory         = 5120
+  vuln_processing_task_cpu            = 2048
   awslogs_config = {
     group  = module.main.byo-vpc.byo-db.byo-ecs.fleet_config.awslogs.name
     region = module.main.byo-vpc.byo-db.byo-ecs.fleet_config.awslogs.region
diff --git a/terraform/addons/external-vuln-scans/.terraform.lock.hcl b/terraform/addons/external-vuln-scans/.terraform.lock.hcl
index 99ef55563470..f284c8030c37 100644
--- a/terraform/addons/external-vuln-scans/.terraform.lock.hcl
+++ b/terraform/addons/external-vuln-scans/.terraform.lock.hcl
@@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/aws" {
   version = "5.11.0"
   hashes = [
     "h1:OyEBhYcTPChBb0gooSlLIcrxakh72qAN+Sd8Oo12uoc=",
+    "h1:Wo6WCPXNnbyeRp57Jvlp7VBm9acVAAg6jVmFRU2IWjk=",
     "zh:2913af44f9b584f756e5548d5ddc5a251c6d68a7fcd7c41d1418a800a94ef113",
     "zh:31d2bfa84608b74ff5896f41b09e5927d7c37d18875277a51dcd75a1fea3f909",
     "zh:8538ff18e3b4822178e793f06764efdbb84c62227c1051af7d2409ab7be37bfc",
diff --git a/terraform/addons/external-vuln-scans/README.md b/terraform/addons/external-vuln-scans/README.md
index 5c0e75535922..c45dd143361f 100644
--- a/terraform/addons/external-vuln-scans/README.md
+++ b/terraform/addons/external-vuln-scans/README.md
@@ -39,14 +39,18 @@ No modules.
 | <a name="input_ecs_cluster"></a> [ecs\_cluster](#input\_ecs\_cluster) | The ecs cluster module that is created by the byo-db module | `any` | n/a | yes |
 | <a name="input_execution_iam_role_arn"></a> [execution\_iam\_role\_arn](#input\_execution\_iam\_role\_arn) | The ARN of the fleet execution role, this is necessary to pass role from ecs events | `any` | n/a | yes |
 | <a name="input_fleet_config"></a> [fleet\_config](#input\_fleet\_config) | The root Fleet config object | `any` | n/a | yes |
+| <a name="input_fleet_server_private_key_secret_arn"></a> [fleet\_server\_private\_key\_secret\_arn](#input\_fleet\_server\_private\_key\_secret\_arn) | The ARN of the secret that stores the Fleet private key | `string` | n/a | yes |
 | <a name="input_security_groups"></a> [security\_groups](#input\_security\_groups) | n/a | `list(string)` | n/a | yes |
 | <a name="input_subnets"></a> [subnets](#input\_subnets) | n/a | `list(string)` | n/a | yes |
 | <a name="input_task_role_arn"></a> [task\_role\_arn](#input\_task\_role\_arn) | The ARN of the fleet task role, this is necessary to pass role from ecs events | `any` | n/a | yes |
 | <a name="input_vuln_processing_cpu"></a> [vuln\_processing\_cpu](#input\_vuln\_processing\_cpu) | The amount of CPU to dedicate to the vuln processing command | `number` | `1024` | no |
 | <a name="input_vuln_processing_memory"></a> [vuln\_processing\_memory](#input\_vuln\_processing\_memory) | The amount of memory to dedicate to the vuln processing command | `number` | `4096` | no |
+| <a name="input_vuln_processing_task_cpu"></a> [vuln\_processing\_task\_cpu](#input\_vuln\_processing\_task\_cpu) | The amount of CPU to dedicate to the vuln processing task including sidecars | `number` | `1024` | no |
+| <a name="input_vuln_processing_task_memory"></a> [vuln\_processing\_task\_memory](#input\_vuln\_processing\_task\_memory) | The amount of memory to dedicate to the vuln processing task including sidecars | `number` | `4096` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_extra_environment_variables"></a> [extra\_environment\_variables](#output\_extra\_environment\_variables) | n/a |
+| <a name="output_vuln_service_arn"></a> [vuln\_service\_arn](#output\_vuln\_service\_arn) | n/a |
diff --git a/terraform/addons/external-vuln-scans/main.tf b/terraform/addons/external-vuln-scans/main.tf
index 210531823c32..90b182eb05c0 100644
--- a/terraform/addons/external-vuln-scans/main.tf
+++ b/terraform/addons/external-vuln-scans/main.tf
@@ -50,17 +50,20 @@ resource "aws_ecs_service" "fleet" {
 
 resource "aws_ecs_task_definition" "vuln-processing" {
   family                   = "${var.fleet_config.family}-vuln-processing"
-  cpu                      = var.vuln_processing_cpu
-  memory                   = var.vuln_processing_memory
+  cpu                      = var.vuln_processing_task_cpu
+  memory                   = var.vuln_processing_task_memory
   execution_role_arn       = var.execution_iam_role_arn
   task_role_arn            = var.task_role_arn
   network_mode             = "awsvpc"
+  pid_mode                 = var.fleet_config.pid_mode
   requires_compatibilities = ["FARGATE"]
 
   container_definitions = jsonencode(concat([
     {
       name                  = "fleet-vuln-processing"
       image                 = var.fleet_config.image
+      cpu                   = var.vuln_processing_cpu
+      memory                = var.vuln_processing_memory
       essential             = true
       networkMode           = "awsvpc"
       secrets               = local.secrets
diff --git a/terraform/addons/external-vuln-scans/outputs.tf b/terraform/addons/external-vuln-scans/outputs.tf
index 69e8da7bef13..913a55da7225 100644
--- a/terraform/addons/external-vuln-scans/outputs.tf
+++ b/terraform/addons/external-vuln-scans/outputs.tf
@@ -1,6 +1,6 @@
 output "extra_environment_variables" {
   value = {
-    FLEET_VULNERABILITIES_DISABLE_SCHEDULE  = "true"
+    FLEET_VULNERABILITIES_DISABLE_SCHEDULE = "true"
   }
 }
 
diff --git a/terraform/addons/external-vuln-scans/variables.tf b/terraform/addons/external-vuln-scans/variables.tf
index fbb14d86d7cb..0882fb5ee1bf 100644
--- a/terraform/addons/external-vuln-scans/variables.tf
+++ b/terraform/addons/external-vuln-scans/variables.tf
@@ -45,6 +45,19 @@ variable "task_role_arn" {
   description = "The ARN of the fleet task role, this is necessary to pass role from ecs events"
 }
 
+variable "vuln_processing_task_memory" {
+  // note must conform to FARGATE breakpoints https://docs.aws.amazon.com/AmazonECS/latest/userguide/fargate-task-defs.html
+  default     = 4096
+  description = "The amount of memory to dedicate to the vuln processing task including sidecars"
+}
+
+variable "vuln_processing_task_cpu" {
+  // note must conform to FARGETE breakpoints https://docs.aws.amazon.com/AmazonECS/latest/userguide/fargate-task-defs.html
+  default     = 1024
+  description = "The amount of CPU to dedicate to the vuln processing task including sidecars"
+}
+
+
 variable "vuln_processing_memory" {
   // note must conform to FARGATE breakpoints https://docs.aws.amazon.com/AmazonECS/latest/userguide/fargate-task-defs.html
   default     = 4096