refact acquisition: build profiles (optionally exclude datasources from final binary)

.github/workflows/go-tests.yml

@@ -142,6 +142,11 @@ jobs:
         make build BUILD_STATIC=1
         make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
+    # check if some component stubs are missing
+    - name: "Build profile: minimal"
+      run: |
+        make build BUILD_PROFILE=minimal
+
     - name: Run tests again, dynamic
       run: |
         make clean build

Makefile

@@ -115,6 +115,68 @@ STRIP_SYMBOLS := -s -w
 DISABLE_OPTIMIZATION :=
 endif
 
+#--------------------------------------
+
+# Handle optional components and build profiles, to save space on the final binaries.
+
+# Keep it safe for now until we decide how to expand on the idea. Either choose a profile or exclude components manually.
+# For example if we want to disable some component by default, or have opt-in components (INCLUDE?).
+
+ifeq ($(and $(BUILD_PROFILE),$(EXCLUDE)),1)
+$(error "Cannot specify both BUILD_PROFILE and EXCLUDE")
+endif
+
+COMPONENTS := \
+	datasource_appsec \
+	datasource_cloudwatch \
+	datasource_docker \
+	datasource_file \
+	datasource_k8saudit \
+	datasource_kafka \
+	datasource_journalctl \
+	datasource_kinesis \
+	datasource_loki \
+	datasource_s3 \
+	datasource_syslog \
+	datasource_wineventlog
+
+comma := ,
+space := $(empty) $(empty)
+
+# Predefined profiles
+
+# keep only datasource-file
+EXCLUDE_MINIMAL := $(subst $(space),$(comma),$(filter-out datasource_file,,$(COMPONENTS)))
+
+# example
+# EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3
+
+BUILD_PROFILE ?= default
+
+# Set the EXCLUDE_LIST based on the chosen profile, unless EXCLUDE is already set
+ifeq ($(BUILD_PROFILE),minimal)
+EXCLUDE ?= $(EXCLUDE_MINIMAL)
+else ifneq ($(BUILD_PROFILE),default)
+$(error Invalid build profile specified: $(BUILD_PROFILE). Valid profiles are: minimal, default)
+endif
+
+# Create list of excluded components from the EXCLUDE variable
+EXCLUDE_LIST := $(subst $(comma),$(space),$(EXCLUDE))
+
+INVALID_COMPONENTS := $(filter-out $(COMPONENTS),$(EXCLUDE_LIST))
+ifneq ($(INVALID_COMPONENTS),)
+$(error Invalid optional components specified in EXCLUDE: $(INVALID_COMPONENTS). Valid components are: $(COMPONENTS))
+endif
+
+# Convert the excluded components to "no_<component>" form
+COMPONENT_TAGS := $(foreach component,$(EXCLUDE_LIST),no_$(component))
+
+ifneq ($(COMPONENT_TAGS),)
+GO_TAGS := $(GO_TAGS),$(subst $(space),$(comma),$(COMPONENT_TAGS))
+endif
+
+#--------------------------------------
+
 export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
 	-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
 
@@ -130,6 +192,7 @@ build: build-info crowdsec cscli plugins  ## Build crowdsec, cscli and plugins
 .PHONY: build-info
 build-info:  ## Print build information
 	$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
+	$(info Excluded components: $(EXCLUDE_LIST))
 
 ifneq (,$(RE2_FAIL))
 	$(error $(RE2_FAIL))

cmd/crowdsec/appsec.go

@@ -0,0 +1,18 @@
+// +build !no_datasource_appsec
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func LoadAppsecRules(hub *cwhub.Hub) error {
+	if err := appsec.LoadAppsecRules(hub); err != nil {
+		return fmt.Errorf("while loading appsec rules: %w", err)
+	}
+
+	return nil
+}

cmd/crowdsec/appsec_stub.go

@@ -0,0 +1,11 @@
+//go:build no_datasource_appsec
+
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func LoadAppsecRules(hub *cwhub.Hub) error {
+	return nil
+}

cmd/crowdsec/crowdsec.go

@@ -14,7 +14,6 @@
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
-	"github.com/crowdsecurity/crowdsec/pkg/appsec"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
@@ -43,12 +42,13 @@
 		return nil, nil, fmt.Errorf("while loading parsers: %w", err)
 	}
 
-	if err := LoadBuckets(cConfig, hub); err != nil {
+	if err = LoadBuckets(cConfig, hub); err != nil {
 		return nil, nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 
-	if err := appsec.LoadAppsecRules(hub); err != nil {
-		return nil, nil, fmt.Errorf("while loading appsec rules: %w", err)
+	// can be nerfed by a build flag
+	if err = LoadAppsecRules(hub); err != nil {
+		return nil, nil, err
 	}
 
 	datasources, err := LoadAcquisition(cConfig)
@@ -82,6 +82,7 @@
 				return nil
 			})
 		}
+
 		parserWg.Done()
 
 		return nil
@@ -108,6 +109,7 @@
 				return runPour(inputEventChan, holders, buckets, cConfig)
 			})
 		}
+
 		bucketWg.Done()
 
 		return nil
@@ -134,6 +136,7 @@
 				return runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, apiClient)
 			})
 		}
+
 		outputWg.Done()
 
 		return nil