add TestEndToEndAllFilters

childe · Sep 9, 2024 · 5459cdd · 5459cdd
1 parent c6d2683
commit 5459cdd
Showing 1 changed file with 73 additions and 0 deletions.
diff --git a/gohangout_test.go b/gohangout_test.go
@@ -154,3 +154,76 @@ outputs:
 		convey.So(dateFilter.Times(), convey.ShouldBeGreaterThanOrEqualTo, emit.Times())
 	})
 }
+
+func TestEndToEndAllFilters(t *testing.T) {
+	mockey.PatchConvey("end-to-end process testing, test all filtes", t, func() {
+		_c := `
+inputs:
+  - Stdin:
+      codec: json
+filters:
+  - Grok:
+      failTag: grokfail
+      match:
+      - '^(?P<clientip>(?:(\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b))|((?:(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?)|((?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)))))) (?P<user>([a-zA-Z0-9._-]+)) \[(?P<ts>((?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]))/(\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b)/((\d\d){1,2}):(([^0-9]?)((?:2[0123]|[01]?[0-9])):((?:[0-5][0-9]))(?::((?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)))([^0-9]?)) ((?:[+-]?(?:[0-9]+))))\] "(?:(?P<verb>\b\w+\b) (?P<request>\S+)(?: HTTP/(?P<httpversion>(?:(([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+)))))?|(?P<rawrequest>.*?))" (?P<status>(?:(([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+)))) (?:(?P<bytes>(?:(([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+))))|-) "(?P<referrer>(.*))" (?P<ua>(.*))" "(?P<x_forwarded_for>(.*))" (?P<body>(.*))$'
+  - Filters:
+      if:
+        - '!EQ($.tags,"grokfail")'
+      filters:
+      - Date:
+          formats:
+          - '02/Jan/2006:15:04:05 -0700'
+          src: 'ts'
+          target: timestamp
+          remove_fields: [ts]
+      - Convert:
+          fields:
+            status:
+              remove_if_fail: false
+              setto_if_fail: 0
+              to: int
+  - Drop:
+      if:
+      - 'EQ($.tags,"grokfail")'
+outputs:
+  - Stdout: {}
+`
+
+		c := make(map[string]interface{})
+		yaml.Unmarshal([]byte(_c), &c) //
+
+		mockey.Mock(config.ParseConfig).Return(c, nil).Build()
+
+		rst := make([]map[string]interface{}, 0)
+		emit := mockey.Mock((*output.StdoutOutput).Emit).To(func(o *output.StdoutOutput, e map[string]interface{}) {
+			rst = append(rst, e)
+		}).Build()
+
+		eventCount := 0
+		readOneEvent := mockey.Mock((*input.StdinInput).ReadOneEvent).To(func(*input.StdinInput) map[string]interface{} {
+			switch eventCount {
+			case 0:
+				eventCount++
+				return map[string]interface{}{"message": `10.0.0.100 - [09/Sep/2024:18:40:55 +0800] "GET /assets/loading.gif HTTP/1.1" 200 838555 "http://kafka-admin.corp.com/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0" "10.10.10.100" "-"`}
+			case 1:
+				eventCount++
+				return map[string]interface{}{"message": "mock message"}
+			default:
+				return nil
+			}
+		}).Build()
+
+		options.exitWhenNil = true
+		_main()
+
+		convey.So(readOneEvent.Times(), convey.ShouldEqual, 3)
+		convey.So(emit.Times(), convey.ShouldEqual, 1)
+		convey.So(len(rst), convey.ShouldEqual, 1)
+		event := rst[0]
+		convey.So(event["x_forwarded_for"], convey.ShouldEqual, "10.10.10.100")
+		convey.So(event["body"], convey.ShouldEqual, `"-"`)
+		convey.So(event["status"], convey.ShouldEqual, 200)
+		convey.So(event["timestamp"], convey.ShouldNotBeNil)
+		convey.So(event["ts"], convey.ShouldBeNil)
+	})
+}