osquery-perf changes needed for load testing with simulated Windows…

… hosts (#12754)

Changes in osquery-perf to allow for testing of Windows hosts in
loadtest environments.

cmd/osquery-perf/agent.go

@@ -9,14 +9,10 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"math/rand"
 	"net/http"
 	"os"
-	"path"
-	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -32,33 +28,77 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-//go:embed *.tmpl
-var templatesFS embed.FS
+var (
+	//go:embed *.tmpl
+	templatesFS embed.FS
 
-//go:embed *.software
-var softwareFS embed.FS
+	//go:embed *.software
+	macOSVulnerableSoftwareFS embed.FS
 
-var vulnerableSoftware []fleet.Software
+	//go:embed ubuntu_2204-software.json.bz2
+	ubuntuSoftwareFS embed.FS
+	//go:embed windows_11-software.json.bz2
+	windowsSoftwareFS embed.FS
 
-func init() {
-	vulnerableSoftwareData, err := softwareFS.ReadFile("vulnerable.software")
+	macosVulnerableSoftware []fleet.Software
+	windowsSoftware         []map[string]string
+	ubuntuSoftware          []map[string]string
+)
+
+func loadMacOSVulnerableSoftware() {
+	macOSVulnerableSoftwareData, err := macOSVulnerableSoftwareFS.ReadFile("macos_vulnerable.software")
 	if err != nil {
-		log.Fatal("reading vulnerable software file: ", err)
+		log.Fatal("reading vulnerable macOS software file: ", err)
 	}
-	lines := bytes.Split(vulnerableSoftwareData, []byte("\n"))
+	lines := bytes.Split(macOSVulnerableSoftwareData, []byte("\n"))
 	for _, line := range lines {
 		parts := bytes.Split(line, []byte("##"))
 		if len(parts) < 2 {
 			fmt.Println("skipping", string(line))
 			continue
 		}
-		vulnerableSoftware = append(vulnerableSoftware, fleet.Software{
+		macosVulnerableSoftware = append(macosVulnerableSoftware, fleet.Software{
 			Name:    strings.TrimSpace(string(parts[0])),
 			Version: strings.TrimSpace(string(parts[1])),
 			Source:  "apps",
 		})
 	}
-	log.Printf("Loaded %d vulnerable software\n", len(vulnerableSoftware))
+	log.Printf("Loaded %d vulnerable macOS software\n", len(macosVulnerableSoftware))
+}
+
+func loadSoftwareItems(fs embed.FS, path string) []map[string]string {
+	bz2, err := fs.Open(path)
+	if err != nil {
+		panic(err)
+	}
+
+	type softwareJSON struct {
+		Name    string `json:"name"`
+		Version string `json:"version"`
+		Release string `json:"release,omitempty"`
+		Arch    string `json:"arch,omitempty"`
+	}
+	var softwareList []softwareJSON
+	// ignoring "G110: Potential DoS vulnerability via decompression bomb", as this is test code.
+	if err := json.NewDecoder(bzip2.NewReader(bz2)).Decode(&softwareList); err != nil { //nolint:gosec
+		panic(err)
+	}
+
+	softwareRows := make([]map[string]string, 0, len(softwareList))
+	for _, s := range softwareList {
+		softwareRows = append(softwareRows, map[string]string{
+			"name":    s.Name,
+			"version": s.Version,
+			"source":  "programs",
+		})
+	}
+	return softwareRows
+}
+
+func init() {
+	loadMacOSVulnerableSoftware()
+	windowsSoftware = loadSoftwareItems(windowsSoftwareFS, "windows_11-software.json.bz2")
+	ubuntuSoftware = loadSoftwareItems(ubuntuSoftwareFS, "ubuntu_2204-software.json.bz2")
 }
 
 type Stats struct {
@@ -748,99 +788,6 @@ func (a *agent) hostUsers() []map[string]string {
 	return users
 }
 
-func extract(src, dst string) {
-	srcF, err := os.Open(src)
-	if err != nil {
-		panic(err)
-	}
-	defer srcF.Close()
-
-	dstF, err := os.Create(dst)
-	if err != nil {
-		panic(err)
-	}
-	defer dstF.Close()
-
-	r := bzip2.NewReader(srcF)
-	// ignoring "G110: Potential DoS vulnerability via decompression bomb", as this is test code.
-	_, err = io.Copy(dstF, r) //nolint:gosec
-	if err != nil {
-		panic(err)
-	}
-}
-
-func loadSoftware(platform string, ver string) []map[string]string {
-	_, exFilename, _, ok := runtime.Caller(0)
-	if !ok {
-		panic("No caller information")
-	}
-	exDir := path.Dir(exFilename)
-
-	srcPath := filepath.Join(
-		exDir,
-		"..",
-		"..",
-		"server",
-		"vulnerabilities",
-		"testdata",
-		platform,
-		"software",
-		fmt.Sprintf("%s_%s-software.json.bz2", platform, ver),
-	)
-
-	tmpDir, err := os.MkdirTemp("", "osquery-perf")
-	if err != nil {
-		panic(err)
-	}
-	defer os.RemoveAll(tmpDir)
-	dstPath := filepath.Join(tmpDir, fmt.Sprintf("%s-software.json", ver))
-
-	extract(srcPath, dstPath)
-
-	type softwareJSON struct {
-		Name    string `json:"name"`
-		Version string `json:"version"`
-		Release string `json:"release,omitempty"`
-		Arch    string `json:"arch,omitempty"`
-	}
-
-	var software []softwareJSON
-	contents, err := os.ReadFile(dstPath)
-	if err != nil {
-		log.Printf("reading vuln software for %s %s: %s\n", platform, ver, err)
-		return nil
-	}
-
-	err = json.Unmarshal(contents, &software)
-	if err != nil {
-		log.Printf("unmarshalling vuln software for %s %s:%s", platform, ver, err)
-		return nil
-	}
-
-	var r []map[string]string
-	for i, fi := range software {
-		installedPath := ""
-		if i%2 == 0 {
-			installedPath = fmt.Sprintf("/some/path/%s", fi.Name)
-		}
-		r = append(r, map[string]string{
-			"name":           fi.Name,
-			"version":        fi.Version,
-			"source":         "osquery-perf",
-			"installed_path": installedPath,
-		})
-	}
-	return r
-}
-
-func (a *agent) softwareWindows11() []map[string]string {
-	return loadSoftware("windows", "11")
-}
-
-func (a *agent) softwareUbuntu2204() []map[string]string {
-	return loadSoftware("ubuntu", "2204")
-}
-
 func (a *agent) softwareMacOS() []map[string]string {
 	var lastOpenedCount int
 	commonSoftware := make([]map[string]string, a.softwareCount.common)
@@ -887,7 +834,7 @@ func (a *agent) softwareMacOS() []map[string]string {
 	}
 	randomVulnerableSoftware := make([]map[string]string, a.softwareCount.vulnerable)
 	for i := 0; i < len(randomVulnerableSoftware); i++ {
-		sw := vulnerableSoftware[rand.Intn(len(vulnerableSoftware))]
+		sw := macosVulnerableSoftware[rand.Intn(len(macosVulnerableSoftware))]
 		var lastOpenedAt string
 		if l := a.genLastOpenedAt(&lastOpenedCount); l != nil {
 			lastOpenedAt = l.Format(time.UnixDate)
@@ -1245,15 +1192,15 @@ func (a *agent) processQuery(name, query string) (handled bool, results []map[st
 	case name == hostDetailQueryPrefix+"software_windows":
 		ss := fleet.OsqueryStatus(rand.Intn(2))
 		if ss == fleet.StatusOK {
-			results = a.softwareWindows11()
+			results = windowsSoftware
 		}
 		return true, results, &ss, nil
 	case name == hostDetailQueryPrefix+"software_linux":
 		ss := fleet.OsqueryStatus(rand.Intn(2))
 		if ss == fleet.StatusOK {
 			switch a.os {
 			case "ubuntu_22.04":
-				results = a.softwareUbuntu2204()
+				results = ubuntuSoftware
 			}
 		}
 		return true, results, &ss, nil

infrastructure/loadtesting/terraform/loadtesting.tf

@@ -55,6 +55,7 @@ resource "aws_ecs_task_definition" "loadtest" {
           "-server_url", "http://${aws_lb.internal.dns_name}",
           "--policy_pass_prob", "0.5",
           "--start_period", "5m",
+          "--orbit_prob", "0.0"
         ]
       }
   ])