From 20ee29b39a36893bf49cbfce6639cd0751bbe98c Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 6 Sep 2024 16:44:30 +0200 Subject: [PATCH 01/12] refact: extract function setupLogger() --- pkg/acquisition/acquisition.go | 50 ++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 634557021f1..537ec9a658e 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -89,6 +89,26 @@ func GetDataSourceIface(dataSourceType string) DataSource { return source() } +// setupLogger creates a logger for the datasource to use at runtime. +func setupLogger(source, name string, level *log.Level) (*log.Entry, error) { + clog := log.New() + if err := types.ConfigureLogger(clog); err != nil { + return nil, fmt.Errorf("while configuring datasource logger: %w", err) + } + if level != nil { + clog.SetLevel(*level) + } + fields := log.Fields{ + "type": source, + } + if name != "" { + fields["name"] = name + } + subLogger := clog.WithFields(fields) + + return subLogger, nil +} + // DataSourceConfigure creates and returns a DataSource object from a configuration, // if the configuration is not valid it returns an error. // If the datasource can't be run (eg. journalctl not available), it still returns an error which @@ -101,21 +121,11 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg, metrics return nil, fmt.Errorf("unable to marshal back interface: %w", err) } if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil { - /* this logger will then be used by the datasource at runtime */ - clog := log.New() - if err := types.ConfigureLogger(clog); err != nil { - return nil, fmt.Errorf("while configuring datasource logger: %w", err) - } - if commonConfig.LogLevel != nil { - clog.SetLevel(*commonConfig.LogLevel) - } - customLog := log.Fields{ - "type": commonConfig.Source, - } - if commonConfig.Name != "" { - customLog["name"] = commonConfig.Name + subLogger, err := setupLogger(commonConfig.Source, commonConfig.Name, commonConfig.LogLevel) + if err != nil { + return nil, err } - subLogger := clog.WithFields(customLog) + /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} @@ -154,12 +164,12 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string, transformExpr if dataSrc == nil { return nil, fmt.Errorf("no acquisition for protocol %s://", frags[0]) } - /* this logger will then be used by the datasource at runtime */ - clog := log.New() - if err := types.ConfigureLogger(clog); err != nil { - return nil, fmt.Errorf("while configuring datasource logger: %w", err) + + subLogger, err := setupLogger(dsn, "", nil) + if err != nil { + return nil, err } - subLogger := clog.WithField("type", dsn) + uniqueId := uuid.NewString() if transformExpr != "" { vm, err := expr.Compile(transformExpr, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) @@ -168,7 +178,7 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string, transformExpr } transformRuntimes[uniqueId] = vm } - err := dataSrc.ConfigureByDSN(dsn, labels, subLogger, uniqueId) + err = dataSrc.ConfigureByDSN(dsn, labels, subLogger, uniqueId) if err != nil { return nil, fmt.Errorf("while configuration datasource for %s: %w", dsn, err) } From 49cb2d72f5b09e7f3f8e1aa29e84a7ad27f89106 Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 6 Sep 2024 17:19:14 +0200 Subject: [PATCH 02/12] build: optionally exclude datasources from the binaries --- Makefile | 7 +++++++ pkg/acquisition/acquisition.go | 27 +-------------------------- pkg/acquisition/appsec.go | 12 ++++++++++++ pkg/acquisition/cloudwatch.go | 12 ++++++++++++ pkg/acquisition/docker.go | 12 ++++++++++++ pkg/acquisition/file.go | 12 ++++++++++++ pkg/acquisition/journalctl.go | 12 ++++++++++++ pkg/acquisition/k8s.go | 12 ++++++++++++ pkg/acquisition/kafka.go | 12 ++++++++++++ pkg/acquisition/kinesis.go | 12 ++++++++++++ pkg/acquisition/loki.go | 12 ++++++++++++ pkg/acquisition/s3.go | 12 ++++++++++++ pkg/acquisition/syslog.go | 12 ++++++++++++ pkg/acquisition/wineventlog.go | 12 ++++++++++++ 14 files changed, 152 insertions(+), 26 deletions(-) create mode 100644 pkg/acquisition/appsec.go create mode 100644 pkg/acquisition/cloudwatch.go create mode 100644 pkg/acquisition/docker.go create mode 100644 pkg/acquisition/file.go create mode 100644 pkg/acquisition/journalctl.go create mode 100644 pkg/acquisition/k8s.go create mode 100644 pkg/acquisition/kafka.go create mode 100644 pkg/acquisition/kinesis.go create mode 100644 pkg/acquisition/loki.go create mode 100644 pkg/acquisition/s3.go create mode 100644 pkg/acquisition/syslog.go create mode 100644 pkg/acquisition/wineventlog.go diff --git a/Makefile b/Makefile index 207b5d610f0..f3c656d7e16 100644 --- a/Makefile +++ b/Makefile @@ -115,6 +115,13 @@ STRIP_SYMBOLS := -s -w DISABLE_OPTIMIZATION := endif +# optionally exclude datasources from the build + +# Add the build tags if MINIMAL is enabled +ifeq ($(call bool,$(MINIMAL)),1) + GO_TAGS := $(GO_TAGS),no_datasource_appsec,no_datasource_cloudwatch,no_datasource_docker,no_datasource_k8saudit,no_datasource_kafka,no_datasource_kinesis,no_datasource_loki,no_datasource_s3 +endif + export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \ -trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 537ec9a658e..e2e0635193e 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -18,18 +18,6 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - appsecacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec" - cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" - dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" - fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" - journalctlacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/journalctl" - kafkaacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kafka" - kinesisacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kinesis" - k8sauditacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kubernetesaudit" - lokiacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki" - s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3" - syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog" - wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -64,20 +52,7 @@ type DataSource interface { Dump() interface{} } -var AcquisitionSources = map[string]func() DataSource{ - "file": func() DataSource { return &fileacquisition.FileSource{} }, - "journalctl": func() DataSource { return &journalctlacquisition.JournalCtlSource{} }, - "cloudwatch": func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} }, - "syslog": func() DataSource { return &syslogacquisition.SyslogSource{} }, - "docker": func() DataSource { return &dockeracquisition.DockerSource{} }, - "kinesis": func() DataSource { return &kinesisacquisition.KinesisSource{} }, - "wineventlog": func() DataSource { return &wineventlogacquisition.WinEventLogSource{} }, - "kafka": func() DataSource { return &kafkaacquisition.KafkaSource{} }, - "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, - "loki": func() DataSource { return &lokiacquisition.LokiSource{} }, - "s3": func() DataSource { return &s3acquisition.S3Source{} }, - "appsec": func() DataSource { return &appsecacquisition.AppsecSource{} }, -} +var AcquisitionSources = map[string]func() DataSource{} var transformRuntimes = map[string]*vm.Program{} diff --git a/pkg/acquisition/appsec.go b/pkg/acquisition/appsec.go new file mode 100644 index 00000000000..0b5a340bebd --- /dev/null +++ b/pkg/acquisition/appsec.go @@ -0,0 +1,12 @@ +// +build !no_datasource_appsec + +package acquisition + +import ( + appsecacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["appsec"] = func() DataSource { return &appsecacquisition.AppsecSource{} } +} diff --git a/pkg/acquisition/cloudwatch.go b/pkg/acquisition/cloudwatch.go new file mode 100644 index 00000000000..8e4ba1fa76c --- /dev/null +++ b/pkg/acquisition/cloudwatch.go @@ -0,0 +1,12 @@ +// +build !no_datasource_cloudwatch + +package acquisition + +import ( + cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["cloudwatch"] = func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} } +} diff --git a/pkg/acquisition/docker.go b/pkg/acquisition/docker.go new file mode 100644 index 00000000000..9cc6629bcb8 --- /dev/null +++ b/pkg/acquisition/docker.go @@ -0,0 +1,12 @@ +// +build !no_datasource_docker + +package acquisition + +import ( + dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["docker"] = func() DataSource { return &dockeracquisition.DockerSource{} } +} diff --git a/pkg/acquisition/file.go b/pkg/acquisition/file.go new file mode 100644 index 00000000000..e3748ec6e75 --- /dev/null +++ b/pkg/acquisition/file.go @@ -0,0 +1,12 @@ +// +build !no_datasource_file + +package acquisition + +import ( + fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["file"] = func() DataSource { return &fileacquisition.FileSource{} } +} diff --git a/pkg/acquisition/journalctl.go b/pkg/acquisition/journalctl.go new file mode 100644 index 00000000000..7cd91dfa843 --- /dev/null +++ b/pkg/acquisition/journalctl.go @@ -0,0 +1,12 @@ +// +build !no_datasource_journalctl + +package acquisition + +import ( + journalctlacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/journalctl" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["journalctl"] = func() DataSource { return &journalctlacquisition.JournalCtlSource{} } +} diff --git a/pkg/acquisition/k8s.go b/pkg/acquisition/k8s.go new file mode 100644 index 00000000000..e1bfd8a87d5 --- /dev/null +++ b/pkg/acquisition/k8s.go @@ -0,0 +1,12 @@ +// +build !no_datasource_k8saudit + +package acquisition + +import ( + k8sauditacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kubernetesaudit" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["k8s-audit"] = func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} } +} diff --git a/pkg/acquisition/kafka.go b/pkg/acquisition/kafka.go new file mode 100644 index 00000000000..5a1035dee78 --- /dev/null +++ b/pkg/acquisition/kafka.go @@ -0,0 +1,12 @@ +// +build !no_datasource_kafka + +package acquisition + +import ( + kafkaacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kafka" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["kafka"] = func() DataSource { return &kafkaacquisition.KafkaSource{} } +} diff --git a/pkg/acquisition/kinesis.go b/pkg/acquisition/kinesis.go new file mode 100644 index 00000000000..5610f3b5d37 --- /dev/null +++ b/pkg/acquisition/kinesis.go @@ -0,0 +1,12 @@ +// +build !no_datasource_kinesis + +package acquisition + +import ( + kinesisacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kinesis" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["kinesis"] = func() DataSource { return &kinesisacquisition.KinesisSource{} } +} diff --git a/pkg/acquisition/loki.go b/pkg/acquisition/loki.go new file mode 100644 index 00000000000..baf7af721e6 --- /dev/null +++ b/pkg/acquisition/loki.go @@ -0,0 +1,12 @@ +// +build !no_datasource_loki + +package acquisition + +import ( + "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["loki"] = func() DataSource { return &loki.LokiSource{} } +} diff --git a/pkg/acquisition/s3.go b/pkg/acquisition/s3.go new file mode 100644 index 00000000000..d1c3d875c0d --- /dev/null +++ b/pkg/acquisition/s3.go @@ -0,0 +1,12 @@ +// +build !no_datasource_s3 + +package acquisition + +import ( + s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["s3"] = func() DataSource { return &s3acquisition.S3Source{} } +} diff --git a/pkg/acquisition/syslog.go b/pkg/acquisition/syslog.go new file mode 100644 index 00000000000..d878489d0f8 --- /dev/null +++ b/pkg/acquisition/syslog.go @@ -0,0 +1,12 @@ +// +build !no_datasource_syslog + +package acquisition + +import ( + syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["syslog"] = func() DataSource { return &syslogacquisition.SyslogSource{} } +} diff --git a/pkg/acquisition/wineventlog.go b/pkg/acquisition/wineventlog.go new file mode 100644 index 00000000000..f7759e8148d --- /dev/null +++ b/pkg/acquisition/wineventlog.go @@ -0,0 +1,12 @@ +// +build !no_datasource_wineventlog + +package acquisition + +import ( + wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" +) + +//nolint:gochecknoinits +func init() { + AcquisitionSources["wineventlog"] = func() DataSource { return &wineventlogacquisition.WinEventLogSource{} } +} From b72fda2189470dfe62e9dc47ab3e97d67dffd9c0 Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 9 Sep 2024 12:09:03 +0200 Subject: [PATCH 03/12] list built data sources with "cscli version" --- pkg/cwversion/constraint/constraint.go | 32 +++++++++++++++++++++ pkg/cwversion/version.go | 39 ++++++-------------------- pkg/leakybucket/manager_load.go | 6 ++-- pkg/parser/stage.go | 6 ++-- 4 files changed, 46 insertions(+), 37 deletions(-) create mode 100644 pkg/cwversion/constraint/constraint.go diff --git a/pkg/cwversion/constraint/constraint.go b/pkg/cwversion/constraint/constraint.go new file mode 100644 index 00000000000..67593f9ebbc --- /dev/null +++ b/pkg/cwversion/constraint/constraint.go @@ -0,0 +1,32 @@ +package constraint + +import ( + "fmt" + + goversion "github.com/hashicorp/go-version" +) + +const ( + Parser = ">= 1.0, <= 3.0" + Scenario = ">= 1.0, <= 3.0" + API = "v1" + Acquis = ">= 1.0, < 2.0" +) + +func Satisfies(strvers string, constraint string) (bool, error) { + vers, err := goversion.NewVersion(strvers) + if err != nil { + return false, fmt.Errorf("failed to parse '%s': %w", strvers, err) + } + + constraints, err := goversion.NewConstraint(constraint) + if err != nil { + return false, fmt.Errorf("failed to parse constraint '%s'", constraint) + } + + if !constraints.Check(vers) { + return false, nil + } + + return true, nil +} diff --git a/pkg/cwversion/version.go b/pkg/cwversion/version.go index b208467aef5..c25ba9c00fb 100644 --- a/pkg/cwversion/version.go +++ b/pkg/cwversion/version.go @@ -4,11 +4,12 @@ import ( "fmt" "strings" - goversion "github.com/hashicorp/go-version" - + "github.com/crowdsecurity/go-cs-lib/maptools" "github.com/crowdsecurity/go-cs-lib/version" + "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/apiclient/useragent" + "github.com/crowdsecurity/crowdsec/pkg/cwversion/constraint" ) var ( @@ -16,13 +17,6 @@ var ( Libre2 = "WebAssembly" ) -const ( - Constraint_parser = ">= 1.0, <= 3.0" - Constraint_scenario = ">= 1.0, <= 3.0" - Constraint_api = "v1" - Constraint_acquis = ">= 1.0, < 2.0" -) - func FullString() string { ret := fmt.Sprintf("version: %s\n", version.String()) ret += fmt.Sprintf("Codename: %s\n", Codename) @@ -31,10 +25,11 @@ func FullString() string { ret += fmt.Sprintf("Platform: %s\n", version.System) ret += fmt.Sprintf("libre2: %s\n", Libre2) ret += fmt.Sprintf("User-Agent: %s\n", useragent.Default()) - ret += fmt.Sprintf("Constraint_parser: %s\n", Constraint_parser) - ret += fmt.Sprintf("Constraint_scenario: %s\n", Constraint_scenario) - ret += fmt.Sprintf("Constraint_api: %s\n", Constraint_api) - ret += fmt.Sprintf("Constraint_acquis: %s\n", Constraint_acquis) + ret += fmt.Sprintf("Constraint_parser: %s\n", constraint.Parser) + ret += fmt.Sprintf("Constraint_scenario: %s\n", constraint.Scenario) + ret += fmt.Sprintf("Constraint_api: %s\n", constraint.API) + ret += fmt.Sprintf("Constraint_acquis: %s\n", constraint.Acquis) + ret += fmt.Sprintf("Acquisition data sources: %s\n", strings.Join(maptools.SortedKeys(acquisition.AcquisitionSources), ", ")) return ret } @@ -46,21 +41,3 @@ func VersionStrip() string { return ret[0] } - -func Satisfies(strvers string, constraint string) (bool, error) { - vers, err := goversion.NewVersion(strvers) - if err != nil { - return false, fmt.Errorf("failed to parse '%s': %w", strvers, err) - } - - constraints, err := goversion.NewConstraint(constraint) - if err != nil { - return false, fmt.Errorf("failed to parse constraint '%s'", constraint) - } - - if !constraints.Check(vers) { - return false, nil - } - - return true, nil -} diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 6055a5308b5..1ae70fbfab3 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -22,7 +22,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/crowdsec/pkg/cwversion/constraint" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -292,13 +292,13 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []str bucketFactory.FormatVersion = "1.0" } - ok, err := cwversion.Satisfies(bucketFactory.FormatVersion, cwversion.Constraint_scenario) + ok, err := constraint.Satisfies(bucketFactory.FormatVersion, constraint.Scenario) if err != nil { return nil, nil, fmt.Errorf("failed to check version: %w", err) } if !ok { - log.Errorf("can't load %s : %s doesn't satisfy scenario format %s, skip", bucketFactory.Name, bucketFactory.FormatVersion, cwversion.Constraint_scenario) + log.Errorf("can't load %s : %s doesn't satisfy scenario format %s, skip", bucketFactory.Name, bucketFactory.FormatVersion, constraint.Scenario) continue } diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index fe538023b61..b98db350254 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -21,7 +21,7 @@ import ( log "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/crowdsec/pkg/cwversion/constraint" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) @@ -85,12 +85,12 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ( log.Tracef("no version in %s, assuming '1.0'", node.Name) node.FormatVersion = "1.0" } - ok, err := cwversion.Satisfies(node.FormatVersion, cwversion.Constraint_parser) + ok, err := constraint.Satisfies(node.FormatVersion, constraint.Parser) if err != nil { return nil, fmt.Errorf("failed to check version : %s", err) } if !ok { - log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, cwversion.Constraint_parser) + log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, constraint.Parser) continue } From ed369121a062073d3bcc9cb0cd2bc1d41583cc3f Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 9 Sep 2024 14:19:29 +0200 Subject: [PATCH 04/12] build profiles example $ make BUILD_PROFILE=minimal or $ make EXCLUDE=datasource_s3,datasource_kinesis --- Makefile | 51 ++++++++++++++++++++++++++++++++++++---- cmd/crowdsec/crowdsec.go | 6 ++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index f3c656d7e16..6634301abb2 100644 --- a/Makefile +++ b/Makefile @@ -115,13 +115,55 @@ STRIP_SYMBOLS := -s -w DISABLE_OPTIMIZATION := endif -# optionally exclude datasources from the build +#-------------------------------------- + +# Handle optional components and build profiles, to save space on the final binaries. + +# Keep it safe for now until we decide how to expand on the idea. Either choose a profile or exclude components manually. +# For example if we want to disable some component by default, or have opt-in components (INCLUDE?). + +ifeq ($(and $(BUILD_PROFILE),$(EXCLUDE)),1) +$(error "Cannot specify both BUILD_PROFILE and EXCLUDE") +endif + +COMPONENTS := datasource_appsec datasource_cloudwatch datasource_docker datasource_k8saudit datasource_kafka datasource_kinesis datasource_loki datasource_s3 + +comma := , +space := $(empty) $(empty) + +# Predefined profiles + +EXCLUDE_MINIMAL := $(subst $(space),$(comma),$(COMPONENTS)) +EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3 -# Add the build tags if MINIMAL is enabled -ifeq ($(call bool,$(MINIMAL)),1) - GO_TAGS := $(GO_TAGS),no_datasource_appsec,no_datasource_cloudwatch,no_datasource_docker,no_datasource_k8saudit,no_datasource_kafka,no_datasource_kinesis,no_datasource_loki,no_datasource_s3 +BUILD_PROFILE ?= default + +# Set the EXCLUDE_LIST based on the chosen profile, unless EXCLUDE is already set +ifeq ($(BUILD_PROFILE),minimal) +EXCLUDE ?= $(EXCLUDE_MINIMAL) +else ifeq ($(BUILD_PROFILE),medium) +EXCLUDE ?= $(EXCLUDE_MEDIUM) +else ifneq ($(BUILD_PROFILE),default) +$(error Invalid build profile specified: $(BUILD_PROFILE). Valid profiles are: minimal, medium, default) endif +# Create list of excluded components from the EXCLUDE variable +EXCLUDE_LIST := $(subst $(comma),$(space),$(EXCLUDE)) + +INVALID_COMPONENTS := $(filter-out $(COMPONENTS),$(EXCLUDE_LIST)) +ifneq ($(INVALID_COMPONENTS),) +$(error Invalid optional components specified in EXCLUDE: $(INVALID_COMPONENTS). Valid components are: $(COMPONENTS)) +endif + +# Convert the excluded components to "no_" form +COMPONENT_TAGS := $(foreach component,$(EXCLUDE_LIST),no_$(component)) + +ifneq ($(COMPONENT_TAGS),) +GO_TAGS := $(GO_TAGS),$((subst $(space),$(comma),$(COMPONENT_TAGS))) +endif + +#-------------------------------------- + export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \ -trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION) @@ -137,6 +179,7 @@ build: build-info crowdsec cscli plugins ## Build crowdsec, cscli and plugins .PHONY: build-info build-info: ## Print build information $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH)) + $(info Excluded components: $(EXCLUDE_LIST)) ifneq (,$(RE2_FAIL)) $(error $(RE2_FAIL)) diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 5aafc6b0dfe..6b750397946 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -14,7 +14,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/alertcontext" - "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" @@ -47,8 +46,9 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, [] return nil, nil, fmt.Errorf("while loading scenarios: %w", err) } - if err := appsec.LoadAppsecRules(hub); err != nil { - return nil, nil, fmt.Errorf("while loading appsec rules: %w", err) + // can be nerfed by a build flag + if err := LoadAppsecRules(hub); err != nil { + return nil, nil, err } datasources, err := LoadAcquisition(cConfig) From dbff41d617c1b47b4f7328fbe99f3d4de3207f4a Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 9 Sep 2024 15:29:32 +0200 Subject: [PATCH 05/12] oops --- cmd/crowdsec/appsec.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 cmd/crowdsec/appsec.go diff --git a/cmd/crowdsec/appsec.go b/cmd/crowdsec/appsec.go new file mode 100644 index 00000000000..cb02b137dcd --- /dev/null +++ b/cmd/crowdsec/appsec.go @@ -0,0 +1,18 @@ +// +build !no_datasource_appsec + +package main + +import ( + "fmt" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func LoadAppsecRules(hub *cwhub.Hub) error { + if err := appsec.LoadAppsecRules(hub); err != nil { + return fmt.Errorf("while loading appsec rules: %w", err) + } + + return nil +} From a3d11da0d8d7a69d9571f354a2011502d29a91c4 Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 9 Sep 2024 16:27:50 +0200 Subject: [PATCH 06/12] remove extra parens --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6634301abb2..edf487934a1 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ endif COMPONENT_TAGS := $(foreach component,$(EXCLUDE_LIST),no_$(component)) ifneq ($(COMPONENT_TAGS),) -GO_TAGS := $(GO_TAGS),$((subst $(space),$(comma),$(COMPONENT_TAGS))) +GO_TAGS := $(GO_TAGS),$(subst $(space),$(comma),$(COMPONENT_TAGS)) endif #-------------------------------------- From 8981bec5c315f7ad4ea9584de344b54d974e7d22 Mon Sep 17 00:00:00 2001 From: marco Date: Mon, 9 Sep 2024 16:38:49 +0200 Subject: [PATCH 07/12] keep only file datasource in minimal profile (??) --- Makefile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index edf487934a1..7e177480aeb 100644 --- a/Makefile +++ b/Makefile @@ -126,14 +126,28 @@ ifeq ($(and $(BUILD_PROFILE),$(EXCLUDE)),1) $(error "Cannot specify both BUILD_PROFILE and EXCLUDE") endif -COMPONENTS := datasource_appsec datasource_cloudwatch datasource_docker datasource_k8saudit datasource_kafka datasource_kinesis datasource_loki datasource_s3 +COMPONENTS := \ + datasource_appsec \ + datasource_cloudwatch \ + datasource_docker \ + datasource_file \ + datasource_k8saudit \ + datasource_kafka \ + datasource_journalctl \ + datasource_kinesis \ + datasource_loki \ + datasource_s3 \ + datasource_syslog \ + datasource_wineventlog comma := , space := $(empty) $(empty) # Predefined profiles -EXCLUDE_MINIMAL := $(subst $(space),$(comma),$(COMPONENTS)) +# keep only datasource-file +EXCLUDE_MINIMAL := $(subst $(space),$(comma),$(filter-out datasource_file,,$(COMPONENTS))) + EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3 BUILD_PROFILE ?= default From 717131b755eea8369446a8ee60d73f2a05a192b9 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 10 Sep 2024 09:39:19 +0200 Subject: [PATCH 08/12] CI: build profile/minimal --- .github/workflows/go-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 3709c695231..4efffc679b1 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -148,6 +148,10 @@ jobs: set -o pipefail make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter + - name: "Build profile: minimal" + run: | + make build BUILD_PROFILE=minimal + - name: Upload unit coverage to Codecov uses: codecov/codecov-action@v4 with: From ed771b12152cec9afb064929ff02c61674ad87cd Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 10 Sep 2024 09:55:11 +0200 Subject: [PATCH 09/12] appsec_stub.go --- .github/workflows/go-tests.yml | 9 +++++---- cmd/crowdsec/appsec_stub.go | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 cmd/crowdsec/appsec_stub.go diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 4efffc679b1..df5c0b4fb88 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -142,16 +142,17 @@ jobs: make build BUILD_STATIC=1 make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter + # check if some component stubs are missing + - name: "Build profile: minimal" + run: | + make build BUILD_PROFILE=minimal + - name: Run tests again, dynamic run: | make clean build set -o pipefail make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter - - name: "Build profile: minimal" - run: | - make build BUILD_PROFILE=minimal - - name: Upload unit coverage to Codecov uses: codecov/codecov-action@v4 with: diff --git a/cmd/crowdsec/appsec_stub.go b/cmd/crowdsec/appsec_stub.go new file mode 100644 index 00000000000..5abf317f17c --- /dev/null +++ b/cmd/crowdsec/appsec_stub.go @@ -0,0 +1,11 @@ +// +build no_datasource_appsec + +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func LoadAppsecRules(hub *cwhub.Hub) error { + return nil +} From 65726834ca02a7737431b08d298a90a18aabb036 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 10 Sep 2024 12:16:00 +0200 Subject: [PATCH 10/12] Explicit errors for excluded data sources --- Makefile | 7 ++- pkg/acquisition/acquisition.go | 91 ++++++++++++++++++++++++---------- pkg/acquisition/appsec.go | 2 +- pkg/acquisition/cloudwatch.go | 2 +- pkg/acquisition/docker.go | 2 +- pkg/acquisition/file.go | 2 +- pkg/acquisition/journalctl.go | 2 +- pkg/acquisition/k8s.go | 2 +- pkg/acquisition/kafka.go | 2 +- pkg/acquisition/kinesis.go | 2 +- pkg/acquisition/loki.go | 2 +- pkg/acquisition/s3.go | 2 +- pkg/acquisition/syslog.go | 2 +- pkg/acquisition/wineventlog.go | 2 +- pkg/cwversion/version.go | 20 +++++++- pkg/setup/detect.go | 6 +-- test/bats.mk | 6 +++ test/bats/01_crowdsec.bats | 37 +++++++++++++- 18 files changed, 143 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index 7e177480aeb..6bd3cbb7980 100644 --- a/Makefile +++ b/Makefile @@ -148,17 +148,16 @@ space := $(empty) $(empty) # keep only datasource-file EXCLUDE_MINIMAL := $(subst $(space),$(comma),$(filter-out datasource_file,,$(COMPONENTS))) -EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3 +# example +# EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3 BUILD_PROFILE ?= default # Set the EXCLUDE_LIST based on the chosen profile, unless EXCLUDE is already set ifeq ($(BUILD_PROFILE),minimal) EXCLUDE ?= $(EXCLUDE_MINIMAL) -else ifeq ($(BUILD_PROFILE),medium) -EXCLUDE ?= $(EXCLUDE_MEDIUM) else ifneq ($(BUILD_PROFILE),default) -$(error Invalid build profile specified: $(BUILD_PROFILE). Valid profiles are: minimal, medium, default) +$(error Invalid build profile specified: $(BUILD_PROFILE). Valid profiles are: minimal, default) endif # Create list of excluded components from the EXCLUDE variable diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index e2e0635193e..7a6646ee783 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -52,18 +52,48 @@ type DataSource interface { Dump() interface{} } -var AcquisitionSources = map[string]func() DataSource{} - -var transformRuntimes = map[string]*vm.Program{} +var ( + // We declare everything here so we can tell if they are unsupported, or excluded from the build + AcquisitionSources = map[string]func() DataSource{ + "appsec": nil, + "cloudwatch": nil, + "docker": nil, + "file": nil, + "journalctl": nil, + "k8s-audit": nil, + "kafka": nil, + "kinesis": nil, + "loki": nil, + "s3": nil, + "syslog": nil, + "wineventlog": nil, + } + transformRuntimes = map[string]*vm.Program{} +) -func GetDataSourceIface(dataSourceType string) DataSource { - source := AcquisitionSources[dataSourceType] +func GetDataSourceIface(dataSourceType string) (DataSource, error) { + source, ok := AcquisitionSources[dataSourceType] + if !ok { + return nil, fmt.Errorf("unknown data source %s", dataSourceType) + } if source == nil { - return nil + return nil, fmt.Errorf("data source %s is not built in this version of crowdsec", dataSourceType) + } + return source(), nil +} + +// registerDataSource registers a datasource in the AcquisitionSources map. +// It must be called in the init() function of the datasource package, and the datasource name +// must be declared with a nil value in the map, to allow for conditional compilation. +func registerDataSource(dataSourceType string, dsGetter func() DataSource) { + _, ok := AcquisitionSources[dataSourceType] + if !ok { + panic("datasource must be declared in the map: " + dataSourceType) } - return source() + AcquisitionSources[dataSourceType] = dsGetter } + // setupLogger creates a logger for the datasource to use at runtime. func setupLogger(source, name string, level *log.Level) (*log.Entry, error) { clog := log.New() @@ -95,23 +125,25 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg, metrics if err != nil { return nil, fmt.Errorf("unable to marshal back interface: %w", err) } - if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil { - subLogger, err := setupLogger(commonConfig.Source, commonConfig.Name, commonConfig.LogLevel) - if err != nil { - return nil, err - } + dataSrc, err := GetDataSourceIface(commonConfig.Source) + if err != nil { + return nil, err + } - /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ - if err := dataSrc.CanRun(); err != nil { - return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} - } - /* configure the actual datasource */ - if err := dataSrc.Configure(yamlConfig, subLogger, metricsLevel); err != nil { - return nil, fmt.Errorf("failed to configure datasource %s: %w", commonConfig.Source, err) - } - return &dataSrc, nil + subLogger, err := setupLogger(commonConfig.Source, commonConfig.Name, commonConfig.LogLevel) + if err != nil { + return nil, err + } + + /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ + if err := dataSrc.CanRun(); err != nil { + return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} } - return nil, fmt.Errorf("cannot find source %s", commonConfig.Source) + /* configure the actual datasource */ + if err := dataSrc.Configure(yamlConfig, subLogger, metricsLevel); err != nil { + return nil, fmt.Errorf("failed to configure datasource %s: %w", commonConfig.Source, err) + } + return &dataSrc, nil } // detectBackwardCompatAcquis: try to magically detect the type for backward compat (type was not mandatory then) @@ -135,9 +167,10 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string, transformExpr if len(frags) == 1 { return nil, fmt.Errorf("%s isn't valid dsn (no protocol)", dsn) } - dataSrc := GetDataSourceIface(frags[0]) - if dataSrc == nil { - return nil, fmt.Errorf("no acquisition for protocol %s://", frags[0]) + + dataSrc, err := GetDataSourceIface(frags[0]) + if err != nil { + return nil, fmt.Errorf("no acquisition for protocol %s:// - %w", frags[0], err) } subLogger, err := setupLogger(dsn, "", nil) @@ -222,9 +255,13 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig if sub.Source == "" { return nil, fmt.Errorf("data source type is empty ('source') in %s (position: %d)", acquisFile, idx) } - if GetDataSourceIface(sub.Source) == nil { - return nil, fmt.Errorf("unknown data source %s in %s (position: %d)", sub.Source, acquisFile, idx) + + // pre-check that the source is valid + _, err := GetDataSourceIface(sub.Source) + if err != nil { + return nil, fmt.Errorf("in file %s (position: %d) - %w", acquisFile, idx, err) } + uniqueId := uuid.NewString() sub.UniqueId = uniqueId src, err := DataSourceConfigure(sub, metrics_level) diff --git a/pkg/acquisition/appsec.go b/pkg/acquisition/appsec.go index 0b5a340bebd..112b28b04ee 100644 --- a/pkg/acquisition/appsec.go +++ b/pkg/acquisition/appsec.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["appsec"] = func() DataSource { return &appsecacquisition.AppsecSource{} } + registerDataSource("appsec", func() DataSource { return &appsecacquisition.AppsecSource{}}) } diff --git a/pkg/acquisition/cloudwatch.go b/pkg/acquisition/cloudwatch.go index 8e4ba1fa76c..adf45294dad 100644 --- a/pkg/acquisition/cloudwatch.go +++ b/pkg/acquisition/cloudwatch.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["cloudwatch"] = func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} } + registerDataSource("cloudwatch", func() DataSource { return &cloudwatchacquisition.CloudwatchSource{}}) } diff --git a/pkg/acquisition/docker.go b/pkg/acquisition/docker.go index 9cc6629bcb8..2d6ef2f6beb 100644 --- a/pkg/acquisition/docker.go +++ b/pkg/acquisition/docker.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["docker"] = func() DataSource { return &dockeracquisition.DockerSource{} } + registerDataSource("docker", func() DataSource { return &dockeracquisition.DockerSource{}}) } diff --git a/pkg/acquisition/file.go b/pkg/acquisition/file.go index e3748ec6e75..78f9f6a2b8d 100644 --- a/pkg/acquisition/file.go +++ b/pkg/acquisition/file.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["file"] = func() DataSource { return &fileacquisition.FileSource{} } + registerDataSource("file", func() DataSource { return &fileacquisition.FileSource{}}) } diff --git a/pkg/acquisition/journalctl.go b/pkg/acquisition/journalctl.go index 7cd91dfa843..54cf1f20ec9 100644 --- a/pkg/acquisition/journalctl.go +++ b/pkg/acquisition/journalctl.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["journalctl"] = func() DataSource { return &journalctlacquisition.JournalCtlSource{} } + registerDataSource("journalctl", func() DataSource { return &journalctlacquisition.JournalCtlSource{}}) } diff --git a/pkg/acquisition/k8s.go b/pkg/acquisition/k8s.go index e1bfd8a87d5..83384404ace 100644 --- a/pkg/acquisition/k8s.go +++ b/pkg/acquisition/k8s.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["k8s-audit"] = func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} } + registerDataSource("k8s-audit", func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{}}) } diff --git a/pkg/acquisition/kafka.go b/pkg/acquisition/kafka.go index 5a1035dee78..3cc7f43e0ee 100644 --- a/pkg/acquisition/kafka.go +++ b/pkg/acquisition/kafka.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["kafka"] = func() DataSource { return &kafkaacquisition.KafkaSource{} } + registerDataSource("kafka", func() DataSource { return &kafkaacquisition.KafkaSource{}}) } diff --git a/pkg/acquisition/kinesis.go b/pkg/acquisition/kinesis.go index 5610f3b5d37..22c3d219492 100644 --- a/pkg/acquisition/kinesis.go +++ b/pkg/acquisition/kinesis.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["kinesis"] = func() DataSource { return &kinesisacquisition.KinesisSource{} } + registerDataSource("kinesis", func() DataSource { return &kinesisacquisition.KinesisSource{}}) } diff --git a/pkg/acquisition/loki.go b/pkg/acquisition/loki.go index baf7af721e6..ac19ec9a89e 100644 --- a/pkg/acquisition/loki.go +++ b/pkg/acquisition/loki.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["loki"] = func() DataSource { return &loki.LokiSource{} } + registerDataSource("loki", func() DataSource { return &loki.LokiSource{}}) } diff --git a/pkg/acquisition/s3.go b/pkg/acquisition/s3.go index d1c3d875c0d..1413a6f9f49 100644 --- a/pkg/acquisition/s3.go +++ b/pkg/acquisition/s3.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["s3"] = func() DataSource { return &s3acquisition.S3Source{} } + registerDataSource("s3", func() DataSource { return &s3acquisition.S3Source{}}) } diff --git a/pkg/acquisition/syslog.go b/pkg/acquisition/syslog.go index d878489d0f8..998bb2f7fc7 100644 --- a/pkg/acquisition/syslog.go +++ b/pkg/acquisition/syslog.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["syslog"] = func() DataSource { return &syslogacquisition.SyslogSource{} } + registerDataSource("syslog", func() DataSource { return &syslogacquisition.SyslogSource{}}) } diff --git a/pkg/acquisition/wineventlog.go b/pkg/acquisition/wineventlog.go index f7759e8148d..eb4bb095ffd 100644 --- a/pkg/acquisition/wineventlog.go +++ b/pkg/acquisition/wineventlog.go @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - AcquisitionSources["wineventlog"] = func() DataSource { return &wineventlogacquisition.WinEventLogSource{} } + registerDataSource("wineventlog", func() DataSource { return &wineventlogacquisition.WinEventLogSource{}}) } diff --git a/pkg/cwversion/version.go b/pkg/cwversion/version.go index c25ba9c00fb..9948bed0844 100644 --- a/pkg/cwversion/version.go +++ b/pkg/cwversion/version.go @@ -18,6 +18,17 @@ var ( ) func FullString() string { + dsBuilt := []string{} + dsExcluded := []string{} + + for _, ds := range maptools.SortedKeys(acquisition.AcquisitionSources) { + if acquisition.AcquisitionSources[ds] != nil { + dsBuilt = append(dsBuilt, ds) + continue + } + dsExcluded = append(dsExcluded, ds) + } + ret := fmt.Sprintf("version: %s\n", version.String()) ret += fmt.Sprintf("Codename: %s\n", Codename) ret += fmt.Sprintf("BuildDate: %s\n", version.BuildDate) @@ -29,7 +40,14 @@ func FullString() string { ret += fmt.Sprintf("Constraint_scenario: %s\n", constraint.Scenario) ret += fmt.Sprintf("Constraint_api: %s\n", constraint.API) ret += fmt.Sprintf("Constraint_acquis: %s\n", constraint.Acquis) - ret += fmt.Sprintf("Acquisition data sources: %s\n", strings.Join(maptools.SortedKeys(acquisition.AcquisitionSources), ", ")) + + if len(dsBuilt) > 0 { + ret += fmt.Sprintf("Built data sources: %s\n", strings.Join(dsBuilt, ", ")) + } + + if len(dsExcluded) > 0 { + ret += fmt.Sprintf("Excluded data sources: %s\n", strings.Join(dsExcluded, ", ")) + } return ret } diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go index 55af951bf89..01368091a6b 100644 --- a/pkg/setup/detect.go +++ b/pkg/setup/detect.go @@ -73,9 +73,9 @@ func validateDataSource(opaqueDS DataSourceItem) error { // source must be known - ds := acquisition.GetDataSourceIface(commonDS.Source) - if ds == nil { - return fmt.Errorf("unknown source '%s'", commonDS.Source) + ds, err := acquisition.GetDataSourceIface(commonDS.Source) + if err != nil { + return err } // unmarshal and validate the rest with the specific implementation diff --git a/test/bats.mk b/test/bats.mk index 8f507cb659b..631cc55579b 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -38,6 +38,7 @@ define ENV := export TEST_DIR="$(TEST_DIR)" export LOCAL_DIR="$(LOCAL_DIR)" export BIN_DIR="$(BIN_DIR)" +# append .min to the binary names to use the minimal profile export CROWDSEC="$(CROWDSEC)" export CSCLI="$(CSCLI)" export CONFIG_YAML="$(CONFIG_DIR)/config.yaml" @@ -75,6 +76,11 @@ bats-update-tools: ## Install/update tools required for functional tests # Build and installs crowdsec in a local directory. Rebuilds if already exists. bats-build: bats-environment ## Build binaries for functional tests @$(MKDIR) $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(BATS_PLUGIN_DIR) + # minimal profile + @$(MAKE) build DEBUG=1 TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) BUILD_PROFILE=minimal + @install -m 0755 cmd/crowdsec/crowdsec $(BIN_DIR)/crowdsec.min + @install -m 0755 cmd/crowdsec-cli/cscli $(BIN_DIR)/cscli.min + # default profile @$(MAKE) build DEBUG=1 TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) @install -m 0755 cmd/crowdsec/crowdsec cmd/crowdsec-cli/cscli $(BIN_DIR)/ @install -m 0755 cmd/notification-*/notification-* $(BATS_PLUGIN_DIR)/ diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 83072b0f159..aa5830a6bae 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -199,7 +199,42 @@ teardown() { assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } -@test "crowdsec (disabled datasources)" { +@test "crowdsec (datasource not built)" { + config_set '.common.log_media="stdout"' + + # a datasource cannot run - it's not built in the log processor executable + + ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') + mkdir -p "$ACQUIS_DIR" + cat >"$ACQUIS_DIR"/foo.yaml <<-EOT + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" + labels: + type: syslog + EOT + + #shellcheck disable=SC2016 + rune -1 wait-for \ + --err "crowdsec init: while loading acquisition config: in file $ACQUIS_DIR/foo.yaml (position: 0) - data source journalctl is not built in this version of crowdsec" \ + env PATH='' "$CROWDSEC".min + + # auto-detection of journalctl_filter still works + cat >"$ACQUIS_DIR"/foo.yaml <<-EOT + source: whatever + journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" + labels: + type: syslog + EOT + + #shellcheck disable=SC2016 + rune -1 wait-for \ + --err "crowdsec init: while loading acquisition config: in file $ACQUIS_DIR/foo.yaml (position: 0) - data source journalctl is not built in this version of crowdsec" \ + env PATH='' "$CROWDSEC".min +} + +@test "crowdsec (disabled datasource)" { if is_package_testing; then # we can't hide journalctl in package testing # because crowdsec is run from systemd From 7a14b63d720ad08e92227268163843035a26d4da Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 10 Sep 2024 13:18:59 +0200 Subject: [PATCH 11/12] fix unit tests --- pkg/acquisition/acquisition_test.go | 17 +++++------------ pkg/setup/detect_test.go | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index a5eecbc20ed..494832e4c05 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -79,13 +79,8 @@ func (f *MockSourceCantRun) GetName() string { return "mock_cant_run" } // appendMockSource is only used to add mock source for tests func appendMockSource() { - if GetDataSourceIface("mock") == nil { - AcquisitionSources["mock"] = func() DataSource { return &MockSource{} } - } - - if GetDataSourceIface("mock_cant_run") == nil { - AcquisitionSources["mock_cant_run"] = func() DataSource { return &MockSourceCantRun{} } - } + AcquisitionSources["mock"] = func() DataSource { return &MockSource{} } + AcquisitionSources["mock_cant_run"] = func() DataSource { return &MockSourceCantRun{} } } func TestDataSourceConfigure(t *testing.T) { @@ -150,7 +145,7 @@ labels: log_level: debug source: tutu `, - ExpectedError: "cannot find source tutu", + ExpectedError: "unknown data source tutu", }, { TestName: "mismatch_config", @@ -270,7 +265,7 @@ func TestLoadAcquisitionFromFile(t *testing.T) { Config: csconfig.CrowdsecServiceCfg{ AcquisitionFiles: []string{"test_files/bad_source.yaml"}, }, - ExpectedError: "unknown data source does_not_exist in test_files/bad_source.yaml", + ExpectedError: "in file test_files/bad_source.yaml (position: 0) - unknown data source does_not_exist", }, { TestName: "invalid_filetype_config", @@ -542,9 +537,7 @@ func TestConfigureByDSN(t *testing.T) { }, } - if GetDataSourceIface("mockdsn") == nil { - AcquisitionSources["mockdsn"] = func() DataSource { return &MockSourceByDSN{} } - } + AcquisitionSources["mockdsn"] = func() DataSource { return &MockSourceByDSN{} } for _, tc := range tests { t.Run(tc.dsn, func(t *testing.T) { diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index c744e7d6796..6f61b5dac78 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -871,7 +871,7 @@ func TestDetectDatasourceValidation(t *testing.T) { datasource: source: wombat`, expected: setup.Setup{Setup: []setup.ServiceSetup{}}, - expectedErr: "invalid datasource for foobar: unknown source 'wombat'", + expectedErr: "invalid datasource for foobar: unknown data source wombat", }, { name: "source is misplaced", config: ` From 79cc22a14bebca08d5875cf318cdf0b8fbcc5a7c Mon Sep 17 00:00:00 2001 From: marco Date: Thu, 12 Sep 2024 17:04:18 +0200 Subject: [PATCH 12/12] lint --- cmd/crowdsec/appsec_stub.go | 2 +- cmd/crowdsec/crowdsec.go | 7 ++- pkg/acquisition/acquisition.go | 51 ++++++++++++------- pkg/acquisition/acquisition_test.go | 10 +++- pkg/acquisition/appsec.go | 4 +- pkg/acquisition/cloudwatch.go | 4 +- pkg/acquisition/docker.go | 4 +- pkg/acquisition/file.go | 4 +- pkg/acquisition/journalctl.go | 4 +- pkg/acquisition/k8s.go | 4 +- pkg/acquisition/kafka.go | 4 +- pkg/acquisition/kinesis.go | 4 +- pkg/acquisition/loki.go | 4 +- .../modules/appsec/appsec_lnx_test.go | 2 +- .../modules/appsec/appsec_win_test.go | 1 - pkg/acquisition/s3.go | 4 +- pkg/acquisition/syslog.go | 4 +- pkg/acquisition/wineventlog.go | 4 +- pkg/cwversion/version.go | 1 + 19 files changed, 74 insertions(+), 48 deletions(-) diff --git a/cmd/crowdsec/appsec_stub.go b/cmd/crowdsec/appsec_stub.go index 5abf317f17c..4a65b32a9ad 100644 --- a/cmd/crowdsec/appsec_stub.go +++ b/cmd/crowdsec/appsec_stub.go @@ -1,4 +1,4 @@ -// +build no_datasource_appsec +//go:build no_datasource_appsec package main diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 6b750397946..460e8ab4328 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -42,12 +42,12 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, [] return nil, nil, fmt.Errorf("while loading parsers: %w", err) } - if err := LoadBuckets(cConfig, hub); err != nil { + if err = LoadBuckets(cConfig, hub); err != nil { return nil, nil, fmt.Errorf("while loading scenarios: %w", err) } // can be nerfed by a build flag - if err := LoadAppsecRules(hub); err != nil { + if err = LoadAppsecRules(hub); err != nil { return nil, nil, err } @@ -82,6 +82,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H return nil }) } + parserWg.Done() return nil @@ -108,6 +109,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H return runPour(inputEventChan, holders, buckets, cConfig) }) } + bucketWg.Done() return nil @@ -134,6 +136,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H return runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, apiClient) }) } + outputWg.Done() return nil diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 7a6646ee783..38bf228abbc 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -55,17 +55,17 @@ type DataSource interface { var ( // We declare everything here so we can tell if they are unsupported, or excluded from the build AcquisitionSources = map[string]func() DataSource{ - "appsec": nil, - "cloudwatch": nil, - "docker": nil, - "file": nil, - "journalctl": nil, - "k8s-audit": nil, - "kafka": nil, - "kinesis": nil, - "loki": nil, - "s3": nil, - "syslog": nil, + "appsec": nil, + "cloudwatch": nil, + "docker": nil, + "file": nil, + "journalctl": nil, + "k8s-audit": nil, + "kafka": nil, + "kinesis": nil, + "loki": nil, + "s3": nil, + "syslog": nil, "wineventlog": nil, } transformRuntimes = map[string]*vm.Program{} @@ -76,9 +76,11 @@ func GetDataSourceIface(dataSourceType string) (DataSource, error) { if !ok { return nil, fmt.Errorf("unknown data source %s", dataSourceType) } + if source == nil { return nil, fmt.Errorf("data source %s is not built in this version of crowdsec", dataSourceType) } + return source(), nil } @@ -90,25 +92,29 @@ func registerDataSource(dataSourceType string, dsGetter func() DataSource) { if !ok { panic("datasource must be declared in the map: " + dataSourceType) } + AcquisitionSources[dataSourceType] = dsGetter } - // setupLogger creates a logger for the datasource to use at runtime. func setupLogger(source, name string, level *log.Level) (*log.Entry, error) { clog := log.New() if err := types.ConfigureLogger(clog); err != nil { return nil, fmt.Errorf("while configuring datasource logger: %w", err) } + if level != nil { clog.SetLevel(*level) } + fields := log.Fields{ "type": source, } + if name != "" { fields["name"] = name } + subLogger := clog.WithFields(fields) return subLogger, nil @@ -125,6 +131,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg, metrics if err != nil { return nil, fmt.Errorf("unable to marshal back interface: %w", err) } + dataSrc, err := GetDataSourceIface(commonConfig.Source) if err != nil { return nil, err @@ -143,6 +150,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg, metrics if err := dataSrc.Configure(yamlConfig, subLogger, metricsLevel); err != nil { return nil, fmt.Errorf("failed to configure datasource %s: %w", commonConfig.Source, err) } + return &dataSrc, nil } @@ -151,12 +159,15 @@ func detectBackwardCompatAcquis(sub configuration.DataSourceCommonCfg) string { if _, ok := sub.Config["filename"]; ok { return "file" } + if _, ok := sub.Config["filenames"]; ok { return "file" } + if _, ok := sub.Config["journalctl_filter"]; ok { return "journalctl" } + return "" } @@ -179,18 +190,23 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string, transformExpr } uniqueId := uuid.NewString() + if transformExpr != "" { vm, err := expr.Compile(transformExpr, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { return nil, fmt.Errorf("while compiling transform expression '%s': %w", transformExpr, err) } + transformRuntimes[uniqueId] = vm } + err = dataSrc.ConfigureByDSN(dsn, labels, subLogger, uniqueId) if err != nil { return nil, fmt.Errorf("while configuration datasource for %s: %w", dsn, err) } + sources = append(sources, dataSrc) + return sources, nil } @@ -237,18 +253,18 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig break } - //for backward compat ('type' was not mandatory, detect it) + // for backward compat ('type' was not mandatory, detect it) if guessType := detectBackwardCompatAcquis(sub); guessType != "" { sub.Source = guessType } - //it's an empty item, skip it + // it's an empty item, skip it if len(sub.Labels) == 0 { if sub.Source == "" { log.Debugf("skipping empty item in %s", acquisFile) continue } if sub.Source != "docker" { - //docker is the only source that can be empty + // docker is the only source that can be empty return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx) } } @@ -363,7 +379,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb } for i := range len(sources) { - subsrc := sources[i] //ensure its a copy + subsrc := sources[i] // ensure its a copy log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) AcquisTomb.Go(func() error { @@ -391,7 +407,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb err = subsrc.OneShotAcquisition(outChan, AcquisTomb) } if err != nil { - //if one of the acqusition returns an error, we kill the others to properly shutdown + // if one of the acqusition returns an error, we kill the others to properly shutdown AcquisTomb.Kill(err) } return nil @@ -400,5 +416,6 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb /*return only when acquisition is over (cat) or never (tail)*/ err := AcquisTomb.Wait() + return err } diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index 494832e4c05..e39199f9cdb 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -179,6 +179,7 @@ wowo: ajsajasjas yaml.Unmarshal([]byte(tc.String), &common) ds, err := DataSourceConfigure(common, configuration.METRICS_NONE) cstest.RequireErrorContains(t, err, tc.ExpectedError) + if tc.ExpectedError != "" { return } @@ -279,6 +280,7 @@ func TestLoadAcquisitionFromFile(t *testing.T) { t.Run(tc.TestName, func(t *testing.T) { dss, err := LoadAcquisitionFromFile(&tc.Config, nil) cstest.RequireErrorContains(t, err, tc.ExpectedError) + if tc.ExpectedError != "" { return } @@ -324,6 +326,7 @@ func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) erro return nil } + func (f *MockCat) StreamingAcquisition(chan types.Event, *tomb.Tomb) error { return errors.New("can't run in tail") } @@ -362,12 +365,14 @@ func (f *MockTail) GetMode() string { return "tail" } func (f *MockTail) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error { return errors.New("can't run in cat mode") } + func (f *MockTail) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { for range 10 { evt := types.Event{} evt.Line.Src = "test" out <- evt } + <-t.Dying() return nil @@ -381,7 +386,7 @@ func (f *MockTail) ConfigureByDSN(string, map[string]string, *log.Entry, string) } func (f *MockTail) GetUuid() string { return "" } -//func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { +// func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { func TestStartAcquisitionCat(t *testing.T) { sources := []DataSource{ @@ -451,6 +456,7 @@ func (f *MockTailError) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) evt.Line.Src = "test" out <- evt } + t.Kill(errors.New("got error (tomb)")) return errors.New("got error") @@ -480,7 +486,7 @@ READLOOP: } } assert.Equal(t, 10, count) - //acquisTomb.Kill(nil) + // acquisTomb.Kill(nil) time.Sleep(1 * time.Second) cstest.RequireErrorContains(t, acquisTomb.Err(), "got error (tomb)") } diff --git a/pkg/acquisition/appsec.go b/pkg/acquisition/appsec.go index 112b28b04ee..81616d3d2b8 100644 --- a/pkg/acquisition/appsec.go +++ b/pkg/acquisition/appsec.go @@ -1,4 +1,4 @@ -// +build !no_datasource_appsec +//go:build !no_datasource_appsec package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("appsec", func() DataSource { return &appsecacquisition.AppsecSource{}}) + registerDataSource("appsec", func() DataSource { return &appsecacquisition.AppsecSource{} }) } diff --git a/pkg/acquisition/cloudwatch.go b/pkg/acquisition/cloudwatch.go index adf45294dad..e6b3d3e3e53 100644 --- a/pkg/acquisition/cloudwatch.go +++ b/pkg/acquisition/cloudwatch.go @@ -1,4 +1,4 @@ -// +build !no_datasource_cloudwatch +//go:build !no_datasource_cloudwatch package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("cloudwatch", func() DataSource { return &cloudwatchacquisition.CloudwatchSource{}}) + registerDataSource("cloudwatch", func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} }) } diff --git a/pkg/acquisition/docker.go b/pkg/acquisition/docker.go index 2d6ef2f6beb..3bf792a039a 100644 --- a/pkg/acquisition/docker.go +++ b/pkg/acquisition/docker.go @@ -1,4 +1,4 @@ -// +build !no_datasource_docker +//go:build !no_datasource_docker package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("docker", func() DataSource { return &dockeracquisition.DockerSource{}}) + registerDataSource("docker", func() DataSource { return &dockeracquisition.DockerSource{} }) } diff --git a/pkg/acquisition/file.go b/pkg/acquisition/file.go index 78f9f6a2b8d..1ff2e4a3c0e 100644 --- a/pkg/acquisition/file.go +++ b/pkg/acquisition/file.go @@ -1,4 +1,4 @@ -// +build !no_datasource_file +//go:build !no_datasource_file package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("file", func() DataSource { return &fileacquisition.FileSource{}}) + registerDataSource("file", func() DataSource { return &fileacquisition.FileSource{} }) } diff --git a/pkg/acquisition/journalctl.go b/pkg/acquisition/journalctl.go index 54cf1f20ec9..691f961ae77 100644 --- a/pkg/acquisition/journalctl.go +++ b/pkg/acquisition/journalctl.go @@ -1,4 +1,4 @@ -// +build !no_datasource_journalctl +//go:build !no_datasource_journalctl package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("journalctl", func() DataSource { return &journalctlacquisition.JournalCtlSource{}}) + registerDataSource("journalctl", func() DataSource { return &journalctlacquisition.JournalCtlSource{} }) } diff --git a/pkg/acquisition/k8s.go b/pkg/acquisition/k8s.go index 83384404ace..cb9446be285 100644 --- a/pkg/acquisition/k8s.go +++ b/pkg/acquisition/k8s.go @@ -1,4 +1,4 @@ -// +build !no_datasource_k8saudit +//go:build !no_datasource_k8saudit package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("k8s-audit", func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{}}) + registerDataSource("k8s-audit", func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }) } diff --git a/pkg/acquisition/kafka.go b/pkg/acquisition/kafka.go index 3cc7f43e0ee..7d315d87feb 100644 --- a/pkg/acquisition/kafka.go +++ b/pkg/acquisition/kafka.go @@ -1,4 +1,4 @@ -// +build !no_datasource_kafka +//go:build !no_datasource_kafka package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("kafka", func() DataSource { return &kafkaacquisition.KafkaSource{}}) + registerDataSource("kafka", func() DataSource { return &kafkaacquisition.KafkaSource{} }) } diff --git a/pkg/acquisition/kinesis.go b/pkg/acquisition/kinesis.go index 22c3d219492..b41372e7fb9 100644 --- a/pkg/acquisition/kinesis.go +++ b/pkg/acquisition/kinesis.go @@ -1,4 +1,4 @@ -// +build !no_datasource_kinesis +//go:build !no_datasource_kinesis package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("kinesis", func() DataSource { return &kinesisacquisition.KinesisSource{}}) + registerDataSource("kinesis", func() DataSource { return &kinesisacquisition.KinesisSource{} }) } diff --git a/pkg/acquisition/loki.go b/pkg/acquisition/loki.go index ac19ec9a89e..1eed6686591 100644 --- a/pkg/acquisition/loki.go +++ b/pkg/acquisition/loki.go @@ -1,4 +1,4 @@ -// +build !no_datasource_loki +//go:build !no_datasource_loki package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("loki", func() DataSource { return &loki.LokiSource{}}) + registerDataSource("loki", func() DataSource { return &loki.LokiSource{} }) } diff --git a/pkg/acquisition/modules/appsec/appsec_lnx_test.go b/pkg/acquisition/modules/appsec/appsec_lnx_test.go index 3e40a1f970c..61dfc536f5e 100644 --- a/pkg/acquisition/modules/appsec/appsec_lnx_test.go +++ b/pkg/acquisition/modules/appsec/appsec_lnx_test.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package appsecacquisition @@ -16,6 +15,7 @@ import ( func TestAppsecRuleTransformsOthers(t *testing.T) { log.SetLevel(log.TraceLevel) + tests := []appsecRuleTest{ { name: "normalizepath", diff --git a/pkg/acquisition/modules/appsec/appsec_win_test.go b/pkg/acquisition/modules/appsec/appsec_win_test.go index e85d75df251..a6b8f3a0340 100644 --- a/pkg/acquisition/modules/appsec/appsec_win_test.go +++ b/pkg/acquisition/modules/appsec/appsec_win_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package appsecacquisition diff --git a/pkg/acquisition/s3.go b/pkg/acquisition/s3.go index 1413a6f9f49..73343b0408d 100644 --- a/pkg/acquisition/s3.go +++ b/pkg/acquisition/s3.go @@ -1,4 +1,4 @@ -// +build !no_datasource_s3 +//go:build !no_datasource_s3 package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("s3", func() DataSource { return &s3acquisition.S3Source{}}) + registerDataSource("s3", func() DataSource { return &s3acquisition.S3Source{} }) } diff --git a/pkg/acquisition/syslog.go b/pkg/acquisition/syslog.go index 998bb2f7fc7..f62cc23b916 100644 --- a/pkg/acquisition/syslog.go +++ b/pkg/acquisition/syslog.go @@ -1,4 +1,4 @@ -// +build !no_datasource_syslog +//go:build !no_datasource_syslog package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("syslog", func() DataSource { return &syslogacquisition.SyslogSource{}}) + registerDataSource("syslog", func() DataSource { return &syslogacquisition.SyslogSource{} }) } diff --git a/pkg/acquisition/wineventlog.go b/pkg/acquisition/wineventlog.go index eb4bb095ffd..0c4889a3f5c 100644 --- a/pkg/acquisition/wineventlog.go +++ b/pkg/acquisition/wineventlog.go @@ -1,4 +1,4 @@ -// +build !no_datasource_wineventlog +//go:build !no_datasource_wineventlog package acquisition @@ -8,5 +8,5 @@ import ( //nolint:gochecknoinits func init() { - registerDataSource("wineventlog", func() DataSource { return &wineventlogacquisition.WinEventLogSource{}}) + registerDataSource("wineventlog", func() DataSource { return &wineventlogacquisition.WinEventLogSource{} }) } diff --git a/pkg/cwversion/version.go b/pkg/cwversion/version.go index 9948bed0844..867098e7d5a 100644 --- a/pkg/cwversion/version.go +++ b/pkg/cwversion/version.go @@ -26,6 +26,7 @@ func FullString() string { dsBuilt = append(dsBuilt, ds) continue } + dsExcluded = append(dsExcluded, ds) }