From 93d0beeb94641ee3f33ca78f3cdda229cfd95740 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:46:25 +0200 Subject: [PATCH] non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2310) This on the other hand, gives a new fatal error when there are no valid datasources. In the previous version, crowdsec kept running with just a warning if no acquisition yaml or dir were specified. --- cmd/crowdsec/crowdsec.go | 10 +++--- cmd/crowdsec/main.go | 4 +++ pkg/acquisition/acquisition.go | 31 ++++++++++++++--- pkg/acquisition/acquisition_test.go | 2 +- tests/bats/01_crowdsec.bats | 49 +++++++++++++++++++++++---- tests/bin/assert-crowdsec-not-running | 2 +- 6 files changed, 80 insertions(+), 18 deletions(-) Signed-off-by: Cyril Brulebois diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 84cf0838..a3b095ae 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -23,22 +23,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error // Populate cwhub package tools - if err := cwhub.GetHubIdx(cConfig.Hub); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err) + if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { + return &parser.Parsers{}, fmt.Errorf("while loading hub index : %s", err) } // Start loading configs csParsers := newParsers() if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err) + return nil, fmt.Errorf("while loading parsers: %s", err) } if err := LoadBuckets(cConfig); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err) + return nil, fmt.Errorf("while loading scenarios: %s", err) } if err := LoadAcquisition(cConfig); err != nil { - return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err) + return nil, fmt.Errorf("while loading acquisition config: %s", err) } return csParsers, nil } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 43eb63ec..a25c372f 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -157,6 +157,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error { } } + if len(dataSources) == 0 { + return fmt.Errorf("no datasource enabled") + } + return nil } diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 43093a50..2461064d 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -25,6 +25,20 @@ import ( tomb "gopkg.in/tomb.v2" ) +type DataSourceUnavailableError struct { + Name string + Err error +} + +func (e *DataSourceUnavailableError) Error() string { + return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err) +} + +func (e *DataSourceUnavailableError) Unwrap() error { + return e.Err +} + + // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module @@ -86,8 +100,11 @@ func GetDataSourceIface(dataSourceType string) DataSource { return nil } +// DataSourceConfigure creates and returns a DataSource object from a configuration, +// if the configuration is not valid it returns an error. +// If the datasource can't be run (eg. journalctl not available), it still returns an error which +// can be checked for the appropriate action. func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { - //we dump it back to []byte, because we want to decode the yaml blob twice : //once to DataSourceCommonCfg, and then later to the dedicated type of the datasource yamlConfig, err := yaml.Marshal(commonConfig) @@ -112,7 +129,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS subLogger := clog.WithFields(customLog) /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { - return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source) + return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} } /* configure the actual datasource */ if err := dataSrc.Configure(yamlConfig, subLogger); err != nil { @@ -179,10 +196,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) + idx := -1 for { var sub configuration.DataSourceCommonCfg - var idx int err = dec.Decode(&sub) + idx += 1 if err != nil { if ! errors.Is(err, io.EOF) { return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile) @@ -199,7 +217,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, if len(sub.Labels) == 0 { if sub.Source == "" { log.Debugf("skipping empty item in %s", acquisFile) - idx += 1 continue } return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx) @@ -212,10 +229,14 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } src, err := DataSourceConfigure(sub) if err != nil { + var dserr *DataSourceUnavailableError + if errors.As(err, &dserr) { + log.Error(err) + continue + } return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx) } sources = append(sources, *src) - idx += 1 } } return sources, nil diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index a547970a..a96044ca 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -171,7 +171,7 @@ log_level: debug source: mock_cant_run wowo: ajsajasjas `, - ExpectedError: "datasource mock_cant_run cannot be run: can't run bro", + ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro", }, } diff --git a/tests/bats/01_crowdsec.bats b/tests/bats/01_crowdsec.bats index a60b576d..f8272eb9 100644 --- a/tests/bats/01_crowdsec.bats +++ b/tests/bats/01_crowdsec.bats @@ -148,9 +148,10 @@ teardown() { rm -f "$ACQUIS_DIR" config_set '.common.log_media="stdout"' - run -124 --separate-stderr timeout 2s "${CROWDSEC}" + run -1 --separate-stderr timeout 2s "${CROWDSEC}" # check warning - assert_stderr_line --partial "no acquisition file found" + assert_stderr --partial "no acquisition file found" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" { @@ -163,19 +164,55 @@ teardown() { config_set '.crowdsec_service.acquisition_dir=""' config_set '.common.log_media="stdout"' - run -124 --separate-stderr timeout 2s "${CROWDSEC}" + run -1 --separate-stderr timeout 2s "${CROWDSEC}" # check warning - assert_stderr_line --partial "no acquisition_path or acquisition_dir specified" + assert_stderr --partial "no acquisition_path or acquisition_dir specified" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') - rm -f "$ACQUIS_YAML" config_set '.crowdsec_service.acquisition_path=""' ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') mkdir -p "$ACQUIS_DIR" - touch "$ACQUIS_DIR"/foo.yaml + mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml run -124 --separate-stderr timeout 2s "${CROWDSEC}" + + # now, if foo.yaml is empty instead, there won't be valid datasources. + + cat /dev/null >"$ACQUIS_DIR"/foo.yaml + + run --separate-stderr -1 timeout 2s "${CROWDSEC}" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } + +@test "crowdsec (disabled datasources)" { + config_set '.common.log_media="stdout"' + + # a datasource cannot run - missing journalctl command + + ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') + mkdir -p "$ACQUIS_DIR" + cat >"$ACQUIS_DIR"/foo.yaml <<-EOT + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" + labels: + type: syslog + EOT + + run --separate-stderr -124 timeout 2s env PATH='' "${CROWDSEC}" + #shellcheck disable=SC2016 + assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' + + # if all datasources are disabled, crowdsec should exit + + ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') + rm -f "$ACQUIS_YAML" + config_set '.crowdsec_service.acquisition_path=""' + + run --separate-stderr -1 timeout 2s env PATH='' "${CROWDSEC}" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" + } diff --git a/tests/bin/assert-crowdsec-not-running b/tests/bin/assert-crowdsec-not-running index c6f381af..d678cb4f 100755 --- a/tests/bin/assert-crowdsec-not-running +++ b/tests/bin/assert-crowdsec-not-running @@ -1,7 +1,7 @@ #!/usr/bin/env bash is_crowdsec_running() { - PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover') + PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover' 2>/dev/null) } # The process can be slow, especially on CI and during test coverage. -- 2.30.2