refact acquisition: build profiles (optionally exclude datasources from final binary)

.github/workflows/go-tests.yml

@@ -142,6 +142,11 @@ jobs:
         make build BUILD_STATIC=1
         make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
+    # check if some component stubs are missing
+    - name: "Build profile: minimal"
+      run: |
+        make build BUILD_PROFILE=minimal
+
     - name: Run tests again, dynamic
       run: |
         make clean build

Makefile

@@ -115,6 +115,68 @@ STRIP_SYMBOLS := -s -w
 DISABLE_OPTIMIZATION :=
 endif
 
+#--------------------------------------
+
+# Handle optional components and build profiles, to save space on the final binaries.
+
+# Keep it safe for now until we decide how to expand on the idea. Either choose a profile or exclude components manually.
+# For example if we want to disable some component by default, or have opt-in components (INCLUDE?).
+
+ifeq ($(and $(BUILD_PROFILE),$(EXCLUDE)),1)
+$(error "Cannot specify both BUILD_PROFILE and EXCLUDE")
+endif
+
+COMPONENTS := \
+	datasource_appsec \
+	datasource_cloudwatch \
+	datasource_docker \
+	datasource_file \
+	datasource_k8saudit \
+	datasource_kafka \
+	datasource_journalctl \
+	datasource_kinesis \
+	datasource_loki \
+	datasource_s3 \
+	datasource_syslog \
+	datasource_wineventlog
+
+comma := ,
+space := $(empty) $(empty)
+
+# Predefined profiles
+
+# keep only datasource-file
+EXCLUDE_MINIMAL := $(subst $(space),$(comma),$(filter-out datasource_file,,$(COMPONENTS)))
+
+# example
+# EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3
+
+BUILD_PROFILE ?= default
+
+# Set the EXCLUDE_LIST based on the chosen profile, unless EXCLUDE is already set
+ifeq ($(BUILD_PROFILE),minimal)
+EXCLUDE ?= $(EXCLUDE_MINIMAL)
+else ifneq ($(BUILD_PROFILE),default)
+$(error Invalid build profile specified: $(BUILD_PROFILE). Valid profiles are: minimal, default)
+endif
+
+# Create list of excluded components from the EXCLUDE variable
+EXCLUDE_LIST := $(subst $(comma),$(space),$(EXCLUDE))
+
+INVALID_COMPONENTS := $(filter-out $(COMPONENTS),$(EXCLUDE_LIST))
+ifneq ($(INVALID_COMPONENTS),)
+$(error Invalid optional components specified in EXCLUDE: $(INVALID_COMPONENTS). Valid components are: $(COMPONENTS))
+endif
+
+# Convert the excluded components to "no_<component>" form
+COMPONENT_TAGS := $(foreach component,$(EXCLUDE_LIST),no_$(component))
+
+ifneq ($(COMPONENT_TAGS),)
+GO_TAGS := $(GO_TAGS),$(subst $(space),$(comma),$(COMPONENT_TAGS))
+endif
+
+#--------------------------------------
+
 export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
 	-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
 
@@ -130,6 +192,7 @@ build: build-info crowdsec cscli plugins  ## Build crowdsec, cscli and plugins
 .PHONY: build-info
 build-info:  ## Print build information
 	$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
+	$(info Excluded components: $(EXCLUDE_LIST))
 
 ifneq (,$(RE2_FAIL))
 	$(error $(RE2_FAIL))

cmd/crowdsec/appsec.go

@@ -0,0 +1,18 @@
+// +build !no_datasource_appsec
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func LoadAppsecRules(hub *cwhub.Hub) error {
+	if err := appsec.LoadAppsecRules(hub); err != nil {
+		return fmt.Errorf("while loading appsec rules: %w", err)
+	}
+
+	return nil
+}

cmd/crowdsec/appsec_stub.go

@@ -0,0 +1,11 @@
+// +build no_datasource_appsec
+
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func LoadAppsecRules(hub *cwhub.Hub) error {
+	return nil
+}

cmd/crowdsec/crowdsec.go

@@ -14,7 +14,6 @@
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
-	"github.com/crowdsecurity/crowdsec/pkg/appsec"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
@@ -47,8 +46,9 @@
 		return nil, nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 
-	if err := appsec.LoadAppsecRules(hub); err != nil {
-		return nil, nil, fmt.Errorf("while loading appsec rules: %w", err)
+	// can be nerfed by a build flag
+	if err := LoadAppsecRules(hub); err != nil {
+		return nil, nil, err
 	}
 
 	datasources, err := LoadAcquisition(cConfig)

pkg/acquisition/acquisition.go

@@ -18,18 +18,6 @@
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
-	appsecacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec"
-	cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch"
-	dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker"
-	fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
-	journalctlacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/journalctl"
-	kafkaacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kafka"
-	kinesisacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kinesis"
-	k8sauditacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kubernetesaudit"
-	lokiacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki"
-	s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3"
-	syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog"
-	wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -64,29 +52,66 @@
 	Dump() interface{}
 }
 
-var AcquisitionSources = map[string]func() DataSource{
-	"file":        func() DataSource { return &fileacquisition.FileSource{} },
-	"journalctl":  func() DataSource { return &journalctlacquisition.JournalCtlSource{} },
-	"cloudwatch":  func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} },
-	"syslog":      func() DataSource { return &syslogacquisition.SyslogSource{} },
-	"docker":      func() DataSource { return &dockeracquisition.DockerSource{} },
-	"kinesis":     func() DataSource { return &kinesisacquisition.KinesisSource{} },
-	"wineventlog": func() DataSource { return &wineventlogacquisition.WinEventLogSource{} },
-	"kafka":       func() DataSource { return &kafkaacquisition.KafkaSource{} },
-	"k8s-audit":   func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} },
-	"loki":        func() DataSource { return &lokiacquisition.LokiSource{} },
-	"s3":          func() DataSource { return &s3acquisition.S3Source{} },
-	"appsec":      func() DataSource { return &appsecacquisition.AppsecSource{} },
+var (
+	// We declare everything here so we can tell if they are unsupported, or excluded from the build
+	AcquisitionSources = map[string]func() DataSource{
+		"appsec": nil,
+		"cloudwatch": nil,
+		"docker": nil,
+		"file": nil,
+		"journalctl": nil,
+		"k8s-audit": nil,
+		"kafka": nil,
+		"kinesis": nil,
+		"loki": nil,
+		"s3": nil,
+		"syslog": nil,
+		"wineventlog": nil,
+	}
+	transformRuntimes = map[string]*vm.Program{}
+)
+
+func GetDataSourceIface(dataSourceType string) (DataSource, error) {
+	source, ok := AcquisitionSources[dataSourceType]
+	if !ok {
+		return nil, fmt.Errorf("unknown data source %s", dataSourceType)
+	}
+	if source == nil {
+		return nil, fmt.Errorf("data source %s is not built in this version of crowdsec", dataSourceType)
+	}
+	return source(), nil
 }
 
-var transformRuntimes = map[string]*vm.Program{}
+// registerDataSource registers a datasource in the AcquisitionSources map.
+// It must be called in the init() function of the datasource package, and the datasource name
+// must be declared with a nil value in the map, to allow for conditional compilation.
+func registerDataSource(dataSourceType string, dsGetter func() DataSource) {
+	_, ok := AcquisitionSources[dataSourceType]
+	if !ok {
+		panic("datasource must be declared in the map: " + dataSourceType)
+	}
+	AcquisitionSources[dataSourceType] = dsGetter
+}
 
-func GetDataSourceIface(dataSourceType string) DataSource {
-	source := AcquisitionSources[dataSourceType]
-	if source == nil {
-		return nil
+
+// setupLogger creates a logger for the datasource to use at runtime.
+func setupLogger(source, name string, level *log.Level) (*log.Entry, error) {
+	clog := log.New()
+	if err := types.ConfigureLogger(clog); err != nil {
+		return nil, fmt.Errorf("while configuring datasource logger: %w", err)
+	}
+	if level != nil {
+		clog.SetLevel(*level)
+	}
+	fields := log.Fields{
+		"type": source,
 	}
-	return source()
+	if name != "" {
+		fields["name"] = name
+	}
+	subLogger := clog.WithFields(fields)
+
+	return subLogger, nil
 }
 
 // DataSourceConfigure creates and returns a DataSource object from a configuration,
@@ -100,33 +125,25 @@
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal back interface: %w", err)
 	}
-	if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil {
-		/* this logger will then be used by the datasource at runtime */
-		clog := log.New()
-		if err := types.ConfigureLogger(clog); err != nil {
-			return nil, fmt.Errorf("while configuring datasource logger: %w", err)
-		}
-		if commonConfig.LogLevel != nil {
-			clog.SetLevel(*commonConfig.LogLevel)
-		}
-		customLog := log.Fields{
-			"type": commonConfig.Source,
-		}
-		if commonConfig.Name != "" {
-			customLog["name"] = commonConfig.Name
-		}
-		subLogger := clog.WithFields(customLog)
-		/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
-		if err := dataSrc.CanRun(); err != nil {
-			return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
-		}
-		/* configure the actual datasource */
-		if err := dataSrc.Configure(yamlConfig, subLogger, metricsLevel); err != nil {
-			return nil, fmt.Errorf("failed to configure datasource %s: %w", commonConfig.Source, err)
-		}
-		return &dataSrc, nil
+	dataSrc, err := GetDataSourceIface(commonConfig.Source)
+	if err != nil {
+		return nil, err
 	}
-	return nil, fmt.Errorf("cannot find source %s", commonConfig.Source)
+
+	subLogger, err := setupLogger(commonConfig.Source, commonConfig.Name, commonConfig.LogLevel)
+	if err != nil {
+		return nil, err
+	}
+
+	/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
+	if err := dataSrc.CanRun(); err != nil {
+		return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
+	}
+	/* configure the actual datasource */
+	if err := dataSrc.Configure(yamlConfig, subLogger, metricsLevel); err != nil {
+		return nil, fmt.Errorf("failed to configure datasource %s: %w", commonConfig.Source, err)
+	}
+	return &dataSrc, nil
 }
 
 // detectBackwardCompatAcquis: try to magically detect the type for backward compat (type was not mandatory then)
@@ -150,16 +167,17 @@
 	if len(frags) == 1 {
 		return nil, fmt.Errorf("%s isn't valid dsn (no protocol)", dsn)
 	}
-	dataSrc := GetDataSourceIface(frags[0])
-	if dataSrc == nil {
-		return nil, fmt.Errorf("no acquisition for protocol %s://", frags[0])
+
+	dataSrc, err := GetDataSourceIface(frags[0])
+	if err != nil {
+		return nil, fmt.Errorf("no acquisition for protocol %s:// - %w", frags[0], err)
 	}
-	/* this logger will then be used by the datasource at runtime */
-	clog := log.New()
-	if err := types.ConfigureLogger(clog); err != nil {
-		return nil, fmt.Errorf("while configuring datasource logger: %w", err)
+
+	subLogger, err := setupLogger(dsn, "", nil)
+	if err != nil {
+		return nil, err
 	}
-	subLogger := clog.WithField("type", dsn)
+
 	uniqueId := uuid.NewString()
 	if transformExpr != "" {
 		vm, err := expr.Compile(transformExpr, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
@@ -168,7 +186,7 @@
 		}
 		transformRuntimes[uniqueId] = vm
 	}
-	err := dataSrc.ConfigureByDSN(dsn, labels, subLogger, uniqueId)
+	err = dataSrc.ConfigureByDSN(dsn, labels, subLogger, uniqueId)
 	if err != nil {
 		return nil, fmt.Errorf("while configuration datasource for %s: %w", dsn, err)
 	}
@@ -237,9 +255,13 @@
 			if sub.Source == "" {
 				return nil, fmt.Errorf("data source type is empty ('source') in %s (position: %d)", acquisFile, idx)
 			}
-			if GetDataSourceIface(sub.Source) == nil {
-				return nil, fmt.Errorf("unknown data source %s in %s (position: %d)", sub.Source, acquisFile, idx)
+
+			// pre-check that the source is valid
+			_, err := GetDataSourceIface(sub.Source)
+			if err != nil {
+				return nil, fmt.Errorf("in file %s (position: %d) - %w", acquisFile, idx, err)
 			}
+
 			uniqueId := uuid.NewString()
 			sub.UniqueId = uniqueId
 			src, err := DataSourceConfigure(sub, metrics_level)

pkg/acquisition/acquisition_test.go

@@ -79,13 +79,8 @@ func (f *MockSourceCantRun) GetName() string { return "mock_cant_run" }
 
 // appendMockSource is only used to add mock source for tests
 func appendMockSource() {
-	if GetDataSourceIface("mock") == nil {
-		AcquisitionSources["mock"] = func() DataSource { return &MockSource{} }
-	}
-
-	if GetDataSourceIface("mock_cant_run") == nil {
-		AcquisitionSources["mock_cant_run"] = func() DataSource { return &MockSourceCantRun{} }
-	}
+	AcquisitionSources["mock"] = func() DataSource { return &MockSource{} }
+	AcquisitionSources["mock_cant_run"] = func() DataSource { return &MockSourceCantRun{} }
 }
 
 func TestDataSourceConfigure(t *testing.T) {
@@ -150,7 +145,7 @@ labels:
 log_level: debug
 source: tutu
 `,
-			ExpectedError: "cannot find source tutu",
+			ExpectedError: "unknown data source tutu",
 		},
 		{
 			TestName: "mismatch_config",
@@ -270,7 +265,7 @@ func TestLoadAcquisitionFromFile(t *testing.T) {
 			Config: csconfig.CrowdsecServiceCfg{
 				AcquisitionFiles: []string{"test_files/bad_source.yaml"},
 			},
-			ExpectedError: "unknown data source does_not_exist in test_files/bad_source.yaml",
+			ExpectedError: "in file test_files/bad_source.yaml (position: 0) - unknown data source does_not_exist",
 		},
 		{
 			TestName: "invalid_filetype_config",
@@ -542,9 +537,7 @@ func TestConfigureByDSN(t *testing.T) {
 		},
 	}
 
-	if GetDataSourceIface("mockdsn") == nil {
-		AcquisitionSources["mockdsn"] = func() DataSource { return &MockSourceByDSN{} }
-	}
+	AcquisitionSources["mockdsn"] = func() DataSource { return &MockSourceByDSN{} }
 
 	for _, tc := range tests {
 		t.Run(tc.dsn, func(t *testing.T) {