Skip to content

Commit e14086a

Browse files
joseacljoseacl
committed
Add FilterGrep in Banzai Logging FluentbitSpec
Signed-off-by: joseacl <joseacl@gmail.com>
1 parent 5b66b95 commit e14086a

File tree

11 files changed

+260
-0
lines changed

11 files changed

+260
-0
lines changed

‎charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_fluentbitagents.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1276,6 +1276,26 @@ spec:
12761276
vpc_id:
12771277
type: boolean
12781278
type: object
1279+
filterGrep:
1280+
properties:
1281+
Exclude:
1282+
items:
1283+
type: string
1284+
type: array
1285+
Logical_Op:
1286+
default: legacy
1287+
enum:
1288+
- legacy
1289+
- AND
1290+
- OR
1291+
type: string
1292+
Match:
1293+
type: string
1294+
Regex:
1295+
items:
1296+
type: string
1297+
type: array
1298+
type: object
12791299
filterKubernetes:
12801300
properties:
12811301
Annotations:

‎charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_loggings.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2132,6 +2132,26 @@ spec:
21322132
vpc_id:
21332133
type: boolean
21342134
type: object
2135+
filterGrep:
2136+
properties:
2137+
Exclude:
2138+
items:
2139+
type: string
2140+
type: array
2141+
Logical_Op:
2142+
default: legacy
2143+
enum:
2144+
- legacy
2145+
- AND
2146+
- OR
2147+
type: string
2148+
Match:
2149+
type: string
2150+
Regex:
2151+
items:
2152+
type: string
2153+
type: array
2154+
type: object
21352155
filterKubernetes:
21362156
properties:
21372157
Annotations:

‎charts/logging-operator/crds/logging.banzaicloud.io_fluentbitagents.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,6 +1273,26 @@ spec:
12731273
vpc_id:
12741274
type: boolean
12751275
type: object
1276+
filterGrep:
1277+
properties:
1278+
Exclude:
1279+
items:
1280+
type: string
1281+
type: array
1282+
Logical_Op:
1283+
default: legacy
1284+
enum:
1285+
- legacy
1286+
- AND
1287+
- OR
1288+
type: string
1289+
Match:
1290+
type: string
1291+
Regex:
1292+
items:
1293+
type: string
1294+
type: array
1295+
type: object
12761296
filterKubernetes:
12771297
properties:
12781298
Annotations:

‎charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2129,6 +2129,26 @@ spec:
21292129
vpc_id:
21302130
type: boolean
21312131
type: object
2132+
filterGrep:
2133+
properties:
2134+
Exclude:
2135+
items:
2136+
type: string
2137+
type: array
2138+
Logical_Op:
2139+
default: legacy
2140+
enum:
2141+
- legacy
2142+
- AND
2143+
- OR
2144+
type: string
2145+
Match:
2146+
type: string
2147+
Regex:
2148+
items:
2149+
type: string
2150+
type: array
2151+
type: object
21322152
filterKubernetes:
21332153
properties:
21342154
Annotations:

‎config/crd/bases/logging.banzaicloud.io_fluentbitagents.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,6 +1273,26 @@ spec:
12731273
vpc_id:
12741274
type: boolean
12751275
type: object
1276+
filterGrep:
1277+
properties:
1278+
Exclude:
1279+
items:
1280+
type: string
1281+
type: array
1282+
Logical_Op:
1283+
default: legacy
1284+
enum:
1285+
- legacy
1286+
- AND
1287+
- OR
1288+
type: string
1289+
Match:
1290+
type: string
1291+
Regex:
1292+
items:
1293+
type: string
1294+
type: array
1295+
type: object
12761296
filterKubernetes:
12771297
properties:
12781298
Annotations:

‎config/crd/bases/logging.banzaicloud.io_loggings.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2129,6 +2129,26 @@ spec:
21292129
vpc_id:
21302130
type: boolean
21312131
type: object
2132+
filterGrep:
2133+
properties:
2134+
Exclude:
2135+
items:
2136+
type: string
2137+
type: array
2138+
Logical_Op:
2139+
default: legacy
2140+
enum:
2141+
- legacy
2142+
- AND
2143+
- OR
2144+
type: string
2145+
Match:
2146+
type: string
2147+
Regex:
2148+
items:
2149+
type: string
2150+
type: array
2151+
type: object
21322152
filterKubernetes:
21332153
properties:
21342154
Annotations:

‎pkg/resources/fluentbit/config.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,24 @@ var fluentBitConfigTemplate = `
7272
{{- template "input" .Input }}
7373
{{- end }}
7474
75+
{{- if .FluentdFilterGrep }}
76+
[FILTER]
77+
Name grep
78+
Match {{ .FluentdFilterGrep.Match }}
79+
80+
{{- if .FluentdFilterGrep.Logical_Op }}
81+
Logical_Op {{ .FluentdFilterGrep.Logical_Op }}
82+
{{- end }}
83+
84+
{{- range $value := .FluentdFilterGrep.Regex }}
85+
Regex {{ $value }}
86+
{{- end }}
87+
88+
{{- range $value := .FluentdFilterGrep.Exclude }}
89+
Exclude {{ $value }}
90+
{{- end }}
91+
{{- end}}
92+
7593
{{- if not .DisableKubernetesFilter }}
7694
[FILTER]
7795
Name kubernetes

‎pkg/resources/fluentbit/configsecret.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ type fluentBitConfig struct {
7676
DisableKubernetesFilter bool
7777
KubernetesFilter map[string]string
7878
AwsFilter map[string]string
79+
FluentdFilterGrep *FluentdFilterGrep
7980
BufferStorage map[string]string
8081
FilterModify []v1beta1.FilterModify
8182
FluentForwardOutput *fluentForwardOutputConfig
@@ -85,6 +86,13 @@ type fluentBitConfig struct {
8586
HealthCheck *v1beta1.HealthCheck
8687
}
8788

89+
type FluentdFilterGrep struct {
90+
Match string
91+
Regex []string
92+
Exclude []string
93+
Logical_Op string
94+
}
95+
8896
type fluentForwardOutputConfig struct {
8997
Network FluentbitNetwork
9098
Options map[string]string
@@ -317,6 +325,14 @@ func (r *Reconciler) configSecret() (runtime.Object, reconciler.DesiredState, er
317325
}
318326
input.Input.Values = fluentbitInputValues
319327

328+
if r.fluentbitSpec.FilterGrep != nil {
329+
input.FluentdFilterGrep, err = toFluentdFilterGrep(r.fluentbitSpec.FilterGrep)
330+
331+
if err != nil {
332+
return nil, reconciler.StatePresent, err
333+
}
334+
}
335+
320336
input.KubernetesFilter, err = mapper.StringsMap(r.fluentbitSpec.FilterKubernetes)
321337
if err != nil {
322338
return nil, reconciler.StatePresent, errors.WrapIf(err, "failed to map kubernetes filter for fluentbit")
@@ -458,6 +474,21 @@ func (r *Reconciler) configSecret() (runtime.Object, reconciler.DesiredState, er
458474
}, reconciler.StatePresent, nil
459475
}
460476

477+
func toFluentdFilterGrep(filterGrep *v1beta1.FilterGrep) (*FluentdFilterGrep, error) {
478+
if filterGrep.Logical_Op != "legacy" && len(filterGrep.Regex) > 0 && len(filterGrep.Exclude) > 0 {
479+
return nil, errors.New("failed to parse grep filter for fluentbit, Logical_Op is set, it's not posible to set both Regex and Exclude")
480+
}
481+
482+
fluentdFilterGrep := &FluentdFilterGrep{
483+
Match: filterGrep.Match,
484+
Regex: filterGrep.Regex,
485+
Exclude: filterGrep.Exclude,
486+
Logical_Op: filterGrep.Logical_Op,
487+
}
488+
489+
return fluentdFilterGrep, nil
490+
}
491+
461492
func (r *Reconciler) applyNetworkSettings(input fluentBitConfig) {
462493
if r.fluentbitSpec.Network != nil {
463494
if input.FluentForwardOutput != nil {

‎pkg/resources/fluentbit/configsecret_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package fluentbit
2+
3+
import (
4+
"testing"
5+
6+
"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
7+
"github.com/stretchr/testify/assert"
8+
)
9+
10+
func TestInvalidFilterGrepConfig(t *testing.T) {
11+
invalidFilterGrep := &v1beta1.FilterGrep{
12+
Match: "*",
13+
Regex: []string{"regex", "reg2"},
14+
Exclude: []string{"exclude"},
15+
Logical_Op: "AND",
16+
}
17+
18+
_, err := toFluentdFilterGrep(invalidFilterGrep)
19+
20+
assert.EqualError(t, err, "failed to parse grep filter for fluentbit, Logical_Op is set, it's not posible to set both Regex and Exclude")
21+
}
22+
23+
func TestValidFilterGrepConfig(t *testing.T) {
24+
filterGrep := &v1beta1.FilterGrep{
25+
Match: "*",
26+
Regex: []string{"regex1", "regex2"},
27+
Logical_Op: "AND",
28+
}
29+
30+
expectedFluentFilterGrep := &FluentdFilterGrep{
31+
Match: "*",
32+
Regex: []string{"regex1", "regex2"},
33+
Logical_Op: "AND",
34+
}
35+
36+
parserFluentdfilterGrep, err := toFluentdFilterGrep(filterGrep)
37+
38+
assert.EqualValues(t, err, nil)
39+
assert.EqualValues(t, parserFluentdfilterGrep, expectedFluentFilterGrep)
40+
}

‎pkg/sdk/logging/api/v1beta1/fluentbit_types.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ type FluentbitSpec struct {
105105
InputTail InputTail `json:"inputTail,omitempty"`
106106
FilterAws *FilterAws `json:"filterAws,omitempty"`
107107
FilterModify []FilterModify `json:"filterModify,omitempty"`
108+
FilterGrep *FilterGrep `json:"filterGrep,omitempty"`
108109
// Deprecated, use inputTail.parser
109110
Parser string `json:"parser,omitempty"`
110111
// Parameters for Kubernetes metadata filter
@@ -393,6 +394,26 @@ type FilterAws struct {
393394
Match string `json:"Match,omitempty" plugin:"default:*"`
394395
}
395396

397+
// FilterGrep The Grep Filter plugin
398+
type FilterGrep struct {
399+
// Match filtered records (default:*)
400+
Match string `json:"Match,omitempty" plugin:"default:*"`
401+
// Keep records where the content of KEY matches the regular expression.
402+
Regex []string `json:"Regex,omitempty"`
403+
// Exclude records where the content of KEY matches the regular expression.
404+
Exclude []string `json:"Exclude,omitempty"`
405+
406+
// Specify a logical operator:
407+
// AND, OR or legacy (default).
408+
// In legacy mode the behaviour is either AND or OR depending on whether the grep is including (uses AND) or excluding (uses OR).
409+
// Available from 2.1 or higher.
410+
// Default: "legacy"
411+
//
412+
// +kubebuilder:validation:Enum=legacy;AND;OR
413+
// +kubebuilder:default=legacy
414+
Logical_Op string `json:"Logical_Op,omitempty"`
415+
}
416+
396417
// FilterModify The Modify Filter plugin allows you to change records using rules and conditions.
397418
type FilterModify struct {
398419
// FluentbitAgent Filter Modification Rule

0 commit comments

Comments
 (0)