Add windows event regex filtering to CWAgent

.github/workflows/test-artifacts.yml

@@ -1361,4 +1361,4 @@ jobs:
             else
               cd terraform/eks/addon/gpu
             fi
-            terraform destroy -auto-approve
+            terraform destroy -auto-approve

cmd/config-translator/translator_test.go

@@ -111,17 +111,18 @@ func TestLogWindowsEventConfig(t *testing.T) {
 	expectedErrorMap3 := map[string]int{}
 	expectedErrorMap3["enum"] = 1
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventFormatType.json", false, expectedErrorMap3)
-
-	//New tests for event_ids feature
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventFormatType.json", false, expectedErrorMap3)
 	expectedErrorMap4 := map[string]int{}
 	expectedErrorMap4["invalid_type"] = 1
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventIdsType.json", false, expectedErrorMap4)
-
 	expectedErrorMap5 := map[string]int{}
 	expectedErrorMap5["required"] = 1
 	expectedErrorMap5["number_any_of"] = 1
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithMissingEventIdsAndEventLevels.json", false, expectedErrorMap5)
+	expectedErrorMap6 := map[string]int{}
+	expectedErrorMap6["invalid_type"] = 1
+	expectedErrorMap6["enum"] = 1
+	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidFilterType.json", false, expectedErrorMap6)
 
 }
 

plugins/inputs/windows_event_log/windows_event_log.go

@@ -29,16 +29,17 @@ const (
 var startOnlyOnce sync.Once
 
 type EventConfig struct {
-	Name          string   `toml:"event_name"`
-	Levels        []string `toml:"event_levels"`
-	EventIDs      []int    `toml:"event_ids"`
-	RenderFormat  string   `toml:"event_format"`
-	BatchReadSize int      `toml:"batch_read_size"`
-	LogGroupName  string   `toml:"log_group_name"`
-	LogStreamName string   `toml:"log_stream_name"`
-	LogGroupClass string   `toml:"log_group_class"`
-	Destination   string   `toml:"destination"`
-	Retention     int      `toml:"retention_in_days"`
+	Name          string                     `toml:"event_name"`
+	Levels        []string                   `toml:"event_levels"`
+	EventIDs      []int                      `toml:"event_ids"`
+	Filters       []*wineventlog.EventFilter `toml:"filters"`
+	RenderFormat  string                     `toml:"event_format"`
+	BatchReadSize int                        `toml:"batch_read_size"`
+	LogGroupName  string                     `toml:"log_group_name"`
+	LogStreamName string                     `toml:"log_stream_name"`
+	LogGroupClass string                     `toml:"log_group_class"`
+	Destination   string                     `toml:"destination"`
+	Retention     int                        `toml:"retention_in_days"`
 }
 type Plugin struct {
 	FileStateFolder string          `toml:"file_state_folder"`
@@ -108,6 +109,7 @@ func (s *Plugin) Start(acc telegraf.Accumulator) error {
 			eventConfig.Name,
 			eventConfig.Levels,
 			eventConfig.EventIDs,
+			eventConfig.Filters,
 			eventConfig.LogGroupName,
 			eventConfig.LogStreamName,
 			eventConfig.RenderFormat,

plugins/inputs/windows_event_log/wineventlog/eventFilter_test.go

@@ -0,0 +1,93 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package wineventlog
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEventLogFilterInit(t *testing.T) {
+	exp := "(foo|bar|baz)"
+	filter, err := initEventLogFilter(includeFilterType, exp)
+	assert.NoError(t, err)
+	assert.NotNil(t, filter.expressionP)
+	filter, err = initEventLogFilter(excludeFilterType, exp)
+	assert.NoError(t, err)
+	assert.NotNil(t, filter.expressionP)
+}
+
+func TestLogEventFilterInitInvalidType(t *testing.T) {
+	_, err := initEventLogFilter("something wrong", "(foo|bar|baz)")
+	assert.Error(t, err)
+}
+
+func TestLogEventFilterInitInvalidRegex(t *testing.T) {
+	_, err := initEventLogFilter(excludeFilterType, "abc)")
+	assert.Error(t, err)
+}
+
+func TestLogEventFilterShouldPublishInclude(t *testing.T) {
+	exp := "(foo|bar|baz)"
+	filter, err := initEventLogFilter(includeFilterType, exp)
+	assert.NoError(t, err)
+
+	assertShouldPublish(t, filter, "foo bar baz")
+	assertShouldNotPublish(t, filter, "something else")
+}
+
+func TestEventLogFilterShouldPublishExclude(t *testing.T) {
+	exp := "(foo|bar|baz)"
+	filter, err := initEventLogFilter(excludeFilterType, exp)
+	assert.NoError(t, err)
+
+	assertShouldNotPublish(t, filter, "foo bar baz")
+	assertShouldPublish(t, filter, "something else")
+}
+
+func BenchmarkEventLogFilterShouldPublish(b *testing.B) {
+	exp := "(foo|bar|baz)"
+	filter, err := initEventLogFilter(excludeFilterType, exp)
+	assert.NoError(b, err)
+	b.ResetTimer()
+
+	msg := "foo bar baz"
+
+	for i := 0; i < b.N; i++ {
+		filter.ShouldPublish(msg)
+	}
+}
+
+func BenchmarkEventLogFilterShouldNotPublish(b *testing.B) {
+	exp := "(foo|bar|baz)"
+	filter, err := initEventLogFilter(excludeFilterType, exp)
+	assert.NoError(b, err)
+	b.ResetTimer()
+
+	msg := "something else"
+
+	for i := 0; i < b.N; i++ {
+		filter.ShouldPublish(msg)
+	}
+}
+
+func initEventLogFilter(filterType, expressionStr string) (EventFilter, error) {
+	filter := EventFilter{
+		Type:       filterType,
+		Expression: expressionStr,
+	}
+	err := filter.init()
+	return filter, err
+}
+
+func assertShouldPublish(t *testing.T, filter EventFilter, msg string) {
+	res := filter.ShouldPublish(msg)
+	assert.True(t, res)
+}
+
+func assertShouldNotPublish(t *testing.T, filter EventFilter, msg string) {
+	res := filter.ShouldPublish(msg)
+	assert.False(t, res)
+}

plugins/inputs/windows_event_log/wineventlog/eventfilter.go

@@ -0,0 +1,50 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package wineventlog
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+)
+
+const (
+	includeFilterType = "include"
+	excludeFilterType = "exclude"
+)
+
+var (
+	validFilterTypes    = []string{includeFilterType, excludeFilterType}
+	validFilterTypesSet = map[string]struct{}{
+		includeFilterType: {},
+		excludeFilterType: {},
+	}
+)
+
+type EventFilter struct {
+	Type        string `toml:"type"`
+	Expression  string `toml:"expression"`
+	expressionP *regexp.Regexp
+}
+
+func (filter *EventFilter) init() error {
+	if _, ok := validFilterTypesSet[filter.Type]; !ok {
+		return fmt.Errorf("filter type %s is incorrect, valid types are: %v", filter.Type, validFilterTypes)
+	}
+
+	var err error
+	if filter.expressionP, err = regexp.Compile(filter.Expression); err != nil {
+		return fmt.Errorf("filter regex has issue, regexp: Compile( %v ): %v", filter.Expression, err.Error())
+	}
+	return nil
+}
+
+func (filter *EventFilter) ShouldPublish(message string) bool {
+	if filter.expressionP == nil {
+		log.Printf("E! [wineventlog] Filter regex is invalid, expression: %s", filter.Expression)
+		return false
+	}
+	match := filter.expressionP.MatchString(message)
+	return (filter.Type == includeFilterType) == match
+}

plugins/inputs/windows_event_log/wineventlog/wineventlog.go

@@ -53,6 +53,7 @@ type windowsEventLog struct {
 	name          string
 	levels        []string
 	eventIDs      []int
+	filters       []*EventFilter
 	logGroupName  string
 	logStreamName string
 	logGroupClass string
@@ -71,11 +72,12 @@ type windowsEventLog struct {
 	resubscribeCh chan struct{}
 }
 
-func NewEventLog(name string, levels []string, eventIDs []int, logGroupName, logStreamName, renderFormat, destination string, stateManager state.FileRangeManager, maximumToRead int, retention int, logGroupClass string) *windowsEventLog {
+func NewEventLog(name string, levels []string, eventIDs []int, filters []*EventFilter, logGroupName, logStreamName, renderFormat, destination string, stateManager state.FileRangeManager, maximumToRead int, retention int, logGroupClass string) *windowsEventLog {
 	eventLog := &windowsEventLog{
 		name:          name,
 		levels:        levels,
 		eventIDs:      eventIDs,
+		filters:       filters,
 		logGroupName:  logGroupName,
 		logStreamName: logStreamName,
 		logGroupClass: logGroupClass,
@@ -90,20 +92,26 @@ func NewEventLog(name string, levels []string, eventIDs []int, logGroupName, log
 		done:          make(chan struct{}),
 		resubscribeCh: make(chan struct{}),
 	}
+
 	return eventLog
 }
 
 func (w *windowsEventLog) Init() error {
+
 	const (
 		minEventID = 0
 		maxEventID = 65535
 	)
-
 	for _, eventID := range w.eventIDs {
 		if eventID < minEventID || eventID > maxEventID {
 			return fmt.Errorf("invalid event ID: %d, event IDs must be between %d and %d", eventID, minEventID, maxEventID)
 		}
 	}
+	for _, filter := range w.filters {
+		if err := filter.init(); err != nil {
+			return err
+		}
+	}
 
 	go w.stateManager.Run(state.Notification{Done: w.done})
 	restored, _ := w.stateManager.Restore()
@@ -201,6 +209,19 @@ func (w *windowsEventLog) run() {
 					log.Printf("E! [wineventlog] Error happened when collecting windows events: %v", err)
 					continue
 				}
+
+				shouldPublish := true
+				for _, filter := range w.filters {
+					if !filter.ShouldPublish(value) {
+						shouldPublish = false
+						break
+					}
+				}
+
+				if !shouldPublish {
+					continue
+				}
+
 				recordNumber, _ := strconv.ParseUint(record.System.EventRecordID, 10, 64)
 				r.Shift(recordNumber)
 				evt := &LogEvent{

plugins/inputs/windows_event_log/wineventlog/wineventlog_test.go

@@ -32,6 +32,7 @@ var (
 	// 2 is ERROR
 	LEVELS          = []string{"2"}
 	EVENTIDS        = []int{777}
+	FILTERS         = []*EventFilter{}
 	GROUP_NAME      = "fake"
 	STREAM_NAME     = "fake"
 	RENDER_FMT      = FormatPlainText
@@ -393,7 +394,7 @@ func newTestEventLogWithState(t *testing.T, name string, levels []string, rl sta
 		Name:              filepath.Base(file.Name()),
 		MaxPersistedItems: 10,
 	})
-	return NewEventLog(name, levels, EVENTIDS, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
+	return NewEventLog(name, levels, EVENTIDS, FILTERS, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
 		manager, BATCH_SIZE, RETENTION, LOG_GROUP_CLASS), file.Name()
 }
 
@@ -404,7 +405,7 @@ func newTestEventLog(t *testing.T, name string, levels []string, eventids []int)
 		StateFilePrefix: logscommon.WindowsEventLogPrefix,
 		Name:            GROUP_NAME + "_" + STREAM_NAME + "_" + name,
 	})
-	return NewEventLog(name, levels, eventids, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
+	return NewEventLog(name, levels, eventids, FILTERS, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
 		manager, BATCH_SIZE, RETENTION, LOG_GROUP_CLASS)
 }
 

translator/cmdutil/translatorutil.go

@@ -99,6 +99,11 @@ func checkSchema(inputJsonMap map[string]interface{}) {
 	} else {
 		errorDetails := result.Errors()
 		for _, errorDetail := range errorDetails {
+			if errorDetail.Type() == "number_any_of" || errorDetail.Type() == "any_of" {
+				errDescription := "E! At least one of event_levels, event_ids, or filters is required"
+				translator.AddErrorMessages(config.GetFormattedPath(errorDetail.Context().String()), errDescription)
+				log.Panic("E! Invalid Json input schema.")
+			}
 			translator.AddErrorMessages(config.GetFormattedPath(errorDetail.Context().String()), errorDetail.Description())
 		}
 		log.Panic("E! Invalid Json input schema.")

translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidFilterType.json

@@ -0,0 +1,37 @@
+{
+  "logs": {
+    "logs_collected": {
+      "windows_events": {
+        "collect_list": [
+          {
+            "event_name": "System",
+            "event_ids": [2300],
+            "log_group_name": "System",
+            "log_stream_name": "System",
+            "filters": [
+              {
+                "type": "invalid",
+                "expression": "(TRACE|DEBUG)"
+              }
+            ]
+          },
+          {
+            "event_name": "Application",
+            "event_levels": [
+              "ERROR"
+            ],
+            "log_group_name": "Application",
+            "log_stream_name": "Application",
+            "filters": [
+              {
+                "type": "include",
+                "expression": 40
+              }
+            ]
+          }
+        ]
+      }
+    },
+    "log_stream_name": "LOG_STREAM_NAME"
+  }
+}

translator/config/sampleSchema/validLogWindowsEvents.json

@@ -13,12 +13,26 @@
             "log_stream_name": "System",
             "event_format": "xml"
           },
+          {
+            "event_name": "Security",
+            "event_ids": [
+              4624,
+              4625
+            ],
+            "log_group_name": "Security",
+            "log_stream_name": "Security",
+            "event_format": "text"
+          },
           {
             "event_name": "Application",
             "event_levels": [
               "INFORMATION",
               "ERROR"
             ],
+            "event_ids": [
+              1001,
+              2225
+            ],
             "log_group_name": "Application",
             "log_stream_name": "Application",
             "event_format": "text"