initial osquery docker sidecar and osquery local builds

.github/workflows/dogfood-deploy.yml

@@ -31,6 +31,7 @@ env:
   TF_VAR_elastic_url: ${{ secrets.ELASTIC_APM_SERVER_URL }}
   TF_VAR_elastic_token: ${{ secrets.ELASTIC_APM_SECRET_TOKEN }}
   TF_VAR_geolite2_license: ${{ secrets.MAXMIND_LICENSE }}
+  TF_VAR_dogfood_sidecar_enroll_secret: ${{ secrets.DOGFOOD_SERVERS_CANARY_ENROLL_SECRET }}
 
 permissions:
   id-token: write

infrastructure/dogfood/terraform/aws-tf-module/docker/.gitignore

@@ -0,0 +1,2 @@
+osquery
+osquery-docker.patch

infrastructure/dogfood/terraform/aws-tf-module/docker/main.tf

@@ -11,31 +11,65 @@ terraform {
   }
 }
 
-variable "osquery_tag" {
-  description = "The osquery tag to take from dockerhub to your ecr repo."
+variable "osquery_version" {
+  description = "The osquery version to push to your ecr repo."
   type        = string
 }
 
+variable "osquery_tags" {
+  description = "The tags that you wish to push among the built images"
+  type        = list(string)
+}
+
 variable "ecr_repo" {
   description = "The ecr repo to push to"
   type        = string
 }
 
-resource "docker_image" "dockerhub" {
-  name = "osquery/osquery:${var.osquery_tag}"
+resource "local_file" "osquery_patch" {
+  content         = templatefile("${path.module}/osquery-docker.patch.tmpl", { osquery_version = var.osquery_version })
+  filename        = "${path.module}/osquery-docker.patch"
+  file_permission = "0644"
+}
+
+resource "null_resource" "build_osquery" {
+  depends_on = [local_file.osquery_patch]
+  triggers = {
+    osquery_version_changed = var.osquery_version
+    osquery_tags_changed    = sha256(jsonencode(var.osquery_tags))
+  }
+  provisioner "local-exec" {
+    working_dir = "${path.module}"
+    command     = <<-EOT
+      mkdir -p osquery
+      cd osquery
+      if [ "$(git remote -vvv | head -n1 | awk '{ print $2 }')" = "https://github.com/osquery/osquery.git" ]; then
+        git reset --hard
+        git pull
+      else
+        git clone https://github.com/osquery/osquery.git .
+      fi
+      git apply ../osquery-docker.patch
+      cd tools/docker
+      ./build.sh
+    EOT
+  }
 }
 
 resource "docker_tag" "osquery" {
-  source_image = docker_image.dockerhub.name
+  depends_on   = [null_resource.build_osquery]
+  for_each     = toset(var.osquery_tags)
+  source_image = "osquery/osquery:${each.key}"
   # We can't include the sha256 when pushing even if they match
-  target_image = "${var.ecr_repo}:${split("@sha256", var.osquery_tag)[0]}"
+  target_image = "${var.ecr_repo}:${each.key}"
 }
 
 resource "docker_registry_image" "osquery" {
-  name          = docker_tag.osquery.target_image
+  for_each      = toset(var.osquery_tags)
+  name          = docker_tag.osquery[each.key].target_image
   keep_remotely = true
 }
 
-output "ecr_image" {
-  value = docker_tag.osquery.target_image
+output "ecr_images" {
+  value = { for docker_tag in docker_tag.osquery : split(":", docker_tag.target_image)[1] => docker_tag.target_image }
 }

infrastructure/dogfood/terraform/aws-tf-module/docker/osquery-docker.patch.tmpl

@@ -0,0 +1,28 @@
+diff --git a/tools/docker/build.sh b/tools/docker/build.sh
+index 9efba34f6..34ecd8a4e 100755
+--- a/tools/docker/build.sh
++++ b/tools/docker/build.sh
+@@ -6,7 +6,7 @@ build_deb() {
+
+     TAG=$(echo $OS | sed 's/://g')
+
+-    docker build -f deb-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/deb/osquery_$${VERSION}-1.linux_amd64.deb --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
++    docker build --platform=linux/amd64 -f deb-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/deb/osquery_$${VERSION}-1.linux_amd64.deb --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
+ }
+
+ build_rpm() {
+@@ -15,11 +15,11 @@ build_rpm() {
+
+     TAG=$(echo $OS | sed 's/://g')
+
+-    docker build -f rpm-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/rpm/osquery-$${VERSION}-1.linux.x86_64.rpm --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
++    docker build --platform=linux/amd64 -f rpm-dockerfile . --build-arg OSQUERY_URL=https://pkg.osquery.io/rpm/osquery-$${VERSION}-1.linux.x86_64.rpm --build-arg OS_IMAGE=$OS -t osquery/osquery:$${VERSION}-$${TAG}
+ }
+
+-versions='5.2.3'
+-deb_platforms='ubuntu:16.04 ubuntu:18.04 ubuntu:20.04 ubuntu:22.04 debian:10 debian:9 debian:8 debian:7'
++versions='${osquery_version}'
++deb_platforms='ubuntu:16.04 ubuntu:18.04 ubuntu:20.04 ubuntu:22.04 ubuntu:24.04 debian:10 debian:9 debian:8 debian:7'
+ rpm_platforms='centos:6 centos:7 centos:8'
+
+ for v in $versions

infrastructure/dogfood/terraform/aws-tf-module/free-ecs-hosts.tf

@@ -1,16 +1,18 @@
 ## Linux hosts in ECS
 
 locals {
+  osquery_version = "5.12.2"
   osquery_hosts = {
-    "5.8.2-ubuntu22.04@sha256:b77c7b06c4d7f2a3c58cc3a34e51fffc480e97795fb3c75cb1dc1cf3709e3dc6" = "Skys-laptop"
-    "5.8.2-ubuntu20.04@sha256:3496ffd0ad570c88a9f405e6ef517079cfeed6ce405b9d22db4dc5ef6ed3faac" = "Cloud-City-server"
-    "5.8.2-ubuntu18.04@sha256:372575e876c218dde3c5c0e24fd240d193800fca9b314e94b4ad4e6e22006c9b" = "Mists-laptop"
-    "5.8.2-ubuntu16.04@sha256:112655c42951960d8858c116529fb4c64951e4cf2e34cb7c08cd599a009025bb" = "Ethers-laptop"
-    "5.8.2-debian10@sha256:de29337896aac89b2b03c7642805859d3fb6d52e5dc08230f987bbab4eeba9c5"    = "Breezes-laptop"
-    "5.8.2-debian9@sha256:47e46c19cebdf0dc704dd0061328856bda7e1e86b8c0fefdd6f78bd092c6200e"     = "Aero-server"
-    "5.8.2-centos8@sha256:88a8adde80bd3b1b257e098bc6e41b6afea840f60033653dcb9fe984f36b0f97"     = "Stratuss-laptop"
-    "5.8.2-centos7@sha256:ff251de4935b80a91c5fc1ac352aebdab9a6bbbf5bda1aaada8e26d22b50202d"     = "Zephyrs-Laptop"
-    "5.8.2-centos6@sha256:b56736be8436288d3fbd2549ec6165e0588cd7197e91600de4a2f00f1df28617"     = "Halo-server"
+    "${local.osquery_version}-ubuntu24.04" = "Atmosphere-database"
+    "${local.osquery_version}-ubuntu22.04" = "Skys-laptop"
+    "${local.osquery_version}-ubuntu20.04" = "Cloud-City-server"
+    "${local.osquery_version}-ubuntu18.04" = "Mists-laptop"
+    "${local.osquery_version}-ubuntu16.04" = "Ethers-laptop"
+    "${local.osquery_version}-debian10"    = "Breezes-laptop"
+    "${local.osquery_version}-debian9"     = "Aero-server"
+    "${local.osquery_version}-centos8"     = "Stratuss-laptop"
+    "${local.osquery_version}-centos7"     = "Zephyrs-Laptop"
+    "${local.osquery_version}-centos6"     = "Halo-server"
   }
 
 }
@@ -123,10 +125,10 @@ provider "docker" {
 }
 
 module "osquery_docker" {
-  for_each    = local.osquery_hosts
-  source      = "./docker"
-  ecr_repo    = aws_ecr_repository.osquery.repository_url
-  osquery_tag = each.key
+  source          = "./docker"
+  ecr_repo        = aws_ecr_repository.osquery.repository_url
+  osquery_version = local.osquery_version
+  osquery_tags    = keys(local.osquery_hosts)
 }
 
 resource "random_uuid" "osquery" {
@@ -135,7 +137,7 @@ resource "random_uuid" "osquery" {
 
 resource "aws_ecs_task_definition" "osquery" {
   for_each = local.osquery_hosts
-  // e.g. 5-8-2-ubuntu22-04 to match naming requirements 
+  // e.g. ${osquery_version}-ubuntu22-04 to match naming requirements
   family                   = "osquery-${replace(split("@sha256", each.key)[0], ".", "-")}"
   network_mode             = "awsvpc"
   requires_compatibilities = ["FARGATE"]
@@ -147,7 +149,7 @@ resource "aws_ecs_task_definition" "osquery" {
     [
       {
         name        = "osquery"
-        image       = module.osquery_docker[each.key].ecr_image
+        image       = module.osquery_docker.ecr_images[each.key]
         cpu         = 256
         memory      = 512
         mountPoints = []
@@ -215,8 +217,8 @@ resource "aws_ecs_task_definition" "osquery" {
 
 resource "aws_ecs_service" "osquery" {
   for_each = local.osquery_hosts
-  # Name must match ^[A-Za-z-_]+$ e.g. 5-8-2-ubuntu22-04
-  name            = "osquery_${replace(split("@sha256", each.key)[0], ".", "-")}"
+  # Name must match ^[A-Za-z-_]+$ e.g. 5.12.2-ubuntu22-04
+  name            = "osquery_${replace(each.key, ".", "-")}"
   launch_type     = "FARGATE"
   cluster         = module.free.byo-db.byo-ecs.service.cluster
   task_definition = aws_ecs_task_definition.osquery[each.key].arn

infrastructure/dogfood/terraform/aws-tf-module/main.tf

@@ -40,6 +40,7 @@ variable "fleet_calendar_periodicity" {
   default     = "30s"
   description = "The refresh period for the calendar integration."
 }
+variable "dogfood_sidecar_enroll_secret" {}
 
 data "aws_caller_identity" "current" {}
 
@@ -68,7 +69,7 @@ locals {
 }
 
 module "main" {
-  source          = "github.com/fleetdm/fleet//terraform?ref=tf-mod-root-v1.8.0"
+  source = "github.com/fleetdm/fleet//terraform?ref=tf-mod-root-v1.9.0"
   certificate_arn = module.acm.acm_certificate_arn
   vpc = {
     name = local.customer
@@ -97,10 +98,13 @@ module "main" {
     cluster_name = local.customer
   }
   fleet_config = {
-    image  = local.geolite2_image
-    family = local.customer
-    cpu    = 1024
-    mem    = 4096
+    image    = local.geolite2_image
+    family   = local.customer
+    task_cpu = 2048
+    task_mem = 5120
+    cpu      = 1024
+    mem      = 4096
+    pid_mode = "task"
     autoscaling = {
       min_capacity = 2
       max_capacity = 5
@@ -120,7 +124,7 @@ module "main" {
       }
     }
     extra_iam_policies           = concat(module.firehose-logging.fleet_extra_iam_policies, module.osquery-carve.fleet_extra_iam_policies, module.ses.fleet_extra_iam_policies)
-    extra_execution_iam_policies = concat(module.mdm.extra_execution_iam_policies, [aws_iam_policy.sentry.arn]) #, module.saml_auth_proxy.fleet_extra_execution_policies)
+    extra_execution_iam_policies = concat(module.mdm.extra_execution_iam_policies, [aws_iam_policy.sentry.arn, aws_iam_policy.osquery_sidecar.arn]) #, module.saml_auth_proxy.fleet_extra_execution_policies)
     extra_environment_variables = merge(
       module.mdm.extra_environment_variables,
       module.firehose-logging.fleet_extra_environment_variables,
@@ -137,6 +141,68 @@ module "main" {
     #   container_name   = "fleet"
     #   container_port   = 8080
     # }]
+    sidecars = [
+      {
+        name        = "osquery"
+        image       = module.osquery_docker.ecr_images["${local.osquery_version}-ubuntu24.04"]
+        cpu         = 1024
+        memory      = 1024
+        mountPoints = []
+        volumesFrom = []
+        essential   = true
+        ulimits = [
+          {
+            softLimit = 999999,
+            hardLimit = 999999,
+            name      = "nofile"
+          }
+        ]
+        networkMode = "awsvpc"
+        logConfiguration = {
+          logDriver = "awslogs"
+          options = {
+            awslogs-group         = local.customer
+            awslogs-region        = "us-east-2"
+            awslogs-stream-prefix = "osquery"
+          }
+        }
+        secrets = [
+          {
+            name      = "ENROLL_SECRET"
+            valueFrom = aws_secretsmanager_secret.dogfood_sidecar_enroll_secret.arn
+          }
+        ]
+        workingDirectory = "/",
+        command = [
+          "osqueryd",
+          "--tls_hostname=dogfood.fleetdm.com",
+          "--force=true",
+          # Ensure that the host identifier remains the same between invocations
+          # "--host_identifier=specified",
+          # "--specified_identifier=${random_uuid.osquery[each.key].result}",
+          "--verbose=true",
+          "--tls_dump=true",
+          "--enroll_secret_env=ENROLL_SECRET",
+          "--enroll_tls_endpoint=/api/osquery/enroll",
+          "--config_plugin=tls",
+          "--config_tls_endpoint=/api/osquery/config",
+          "--config_refresh=10",
+          "--disable_distributed=false",
+          "--distributed_plugin=tls",
+          "--distributed_interval=10",
+          "--distributed_tls_max_attempts=3",
+          "--distributed_tls_read_endpoint=/api/osquery/distributed/read",
+          "--distributed_tls_write_endpoint=/api/osquery/distributed/write",
+          "--logger_plugin=tls",
+          "--logger_tls_endpoint=/api/osquery/log",
+          "--logger_tls_period=10",
+          "--disable_carver=false",
+          "--carver_start_endpoint=/api/osquery/carve/begin",
+          "--carver_continue_endpoint=/api/osquery/carve/block",
+          "--carver_block_size=8000000",
+        ]
+      }
+    ]
   }
   alb_config = {
     name = local.customer
@@ -455,17 +521,63 @@ module "geolite2" {
 }
 
 module "vuln-processing" {
-  source                              = "github.com/fleetdm/fleet//terraform/addons/external-vuln-scans?ref=tf-mod-addon-external-vuln-scans-v2.1.0"
+  source                              = "github.com/fleetdm/fleet//terraform/addons/external-vuln-scans?ref=tf-mod-addon-external-vuln-scans-v2.2.0"
   ecs_cluster                         = module.main.byo-vpc.byo-db.byo-ecs.service.cluster
   execution_iam_role_arn              = module.main.byo-vpc.byo-db.byo-ecs.execution_iam_role_arn
   subnets                             = module.main.byo-vpc.byo-db.byo-ecs.service.network_configuration[0].subnets
   security_groups                     = module.main.byo-vpc.byo-db.byo-ecs.service.network_configuration[0].security_groups
   fleet_config                        = module.main.byo-vpc.byo-db.byo-ecs.fleet_config
   task_role_arn                       = module.main.byo-vpc.byo-db.byo-ecs.iam_role_arn
   fleet_server_private_key_secret_arn = module.main.byo-vpc.byo-db.byo-ecs.fleet_server_private_key_secret_arn
+  vuln_processing_task_memory         = 5120
+  vuln_processing_task_cpu            = 2048
   awslogs_config = {
     group  = module.main.byo-vpc.byo-db.byo-ecs.fleet_config.awslogs.name
     region = module.main.byo-vpc.byo-db.byo-ecs.fleet_config.awslogs.region
     prefix = module.main.byo-vpc.byo-db.byo-ecs.fleet_config.awslogs.prefix
   }
 }
+
+resource "aws_secretsmanager_secret" "dogfood_sidecar_enroll_secret" {
+  name = "dogfood-sidecar-enroll-secret"
+}
+
+resource "aws_secretsmanager_secret_version" "dogfood_sidecar_enroll_secret" {
+  secret_id     = aws_secretsmanager_secret.dogfood_sidecar_enroll_secret.id
+  secret_string = var.dogfood_sidecar_enroll_secret
+}
+
+data "aws_iam_policy_document" "osquery_sidecar" {
+  statement {
+    actions = [
+      "ecr:BatchCheckLayerAvailability",
+      "ecr:BatchGetImage",
+      "ecr:GetDownloadUrlForLayer",
+      "ecr:GetAuthorizationToken"
+    ]
+    resources = ["*"]
+  }
+  statement {
+    actions = [ #tfsec:ignore:aws-iam-no-policy-wildcards
+      "kms:Encrypt*",
+      "kms:Decrypt*",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:Describe*"
+    ]
+    resources = [aws_kms_key.osquery.arn]
+  }
+  statement {
+    actions = [ #tfsec:ignore:aws-iam-no-policy-wildcards
+      "secretsmanager:GetSecretValue"
+    ]
+    resources = [aws_secretsmanager_secret.dogfood_sidecar_enroll_secret.arn]
+
+  }
+}
+
+resource "aws_iam_policy" "osquery_sidecar" {
+  name        = "osquery-sidecar-policy"
+  description = "IAM policy that Osquery sidecar containers use to define access to AWS resources"
+  policy      = data.aws_iam_policy_document.osquery_sidecar.json
+}

terraform/addons/external-vuln-scans/README.md

@@ -39,14 +39,18 @@ No modules.
 | <a name="input_ecs_cluster"></a> [ecs\_cluster](#input\_ecs\_cluster) | The ecs cluster module that is created by the byo-db module | `any` | n/a | yes |
 | <a name="input_execution_iam_role_arn"></a> [execution\_iam\_role\_arn](#input\_execution\_iam\_role\_arn) | The ARN of the fleet execution role, this is necessary to pass role from ecs events | `any` | n/a | yes |
 | <a name="input_fleet_config"></a> [fleet\_config](#input\_fleet\_config) | The root Fleet config object | `any` | n/a | yes |
+| <a name="input_fleet_server_private_key_secret_arn"></a> [fleet\_server\_private\_key\_secret\_arn](#input\_fleet\_server\_private\_key\_secret\_arn) | The ARN of the secret that stores the Fleet private key | `string` | n/a | yes |
 | <a name="input_security_groups"></a> [security\_groups](#input\_security\_groups) | n/a | `list(string)` | n/a | yes |
 | <a name="input_subnets"></a> [subnets](#input\_subnets) | n/a | `list(string)` | n/a | yes |
 | <a name="input_task_role_arn"></a> [task\_role\_arn](#input\_task\_role\_arn) | The ARN of the fleet task role, this is necessary to pass role from ecs events | `any` | n/a | yes |
 | <a name="input_vuln_processing_cpu"></a> [vuln\_processing\_cpu](#input\_vuln\_processing\_cpu) | The amount of CPU to dedicate to the vuln processing command | `number` | `1024` | no |
 | <a name="input_vuln_processing_memory"></a> [vuln\_processing\_memory](#input\_vuln\_processing\_memory) | The amount of memory to dedicate to the vuln processing command | `number` | `4096` | no |
+| <a name="input_vuln_processing_task_cpu"></a> [vuln\_processing\_task\_cpu](#input\_vuln\_processing\_task\_cpu) | The amount of CPU to dedicate to the vuln processing task including sidecars | `number` | `1024` | no |
+| <a name="input_vuln_processing_task_memory"></a> [vuln\_processing\_task\_memory](#input\_vuln\_processing\_task\_memory) | The amount of memory to dedicate to the vuln processing task including sidecars | `number` | `4096` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_extra_environment_variables"></a> [extra\_environment\_variables](#output\_extra\_environment\_variables) | n/a |
+| <a name="output_vuln_service_arn"></a> [vuln\_service\_arn](#output\_vuln\_service\_arn) | n/a |