Skip to content

Commit 3ab40da

Browse files
pepovpepov
authored
Merge pull request #2012 from joseacl/issue-1647
Add FilterGrep in Banzai Logging FluentbitSpec
2 parents cdbd51b + 9906489 commit 3ab40da

File tree

12 files changed

+302
-0
lines changed

12 files changed

+302
-0
lines changed

‎charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_fluentbitagents.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1276,6 +1276,26 @@ spec:
12761276
vpc_id:
12771277
type: boolean
12781278
type: object
1279+
filterGrep:
1280+
properties:
1281+
Exclude:
1282+
items:
1283+
type: string
1284+
type: array
1285+
LogicalOp:
1286+
default: legacy
1287+
enum:
1288+
- legacy
1289+
- AND
1290+
- OR
1291+
type: string
1292+
Match:
1293+
type: string
1294+
Regex:
1295+
items:
1296+
type: string
1297+
type: array
1298+
type: object
12791299
filterKubernetes:
12801300
properties:
12811301
Annotations:

‎charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_loggings.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2132,6 +2132,26 @@ spec:
21322132
vpc_id:
21332133
type: boolean
21342134
type: object
2135+
filterGrep:
2136+
properties:
2137+
Exclude:
2138+
items:
2139+
type: string
2140+
type: array
2141+
LogicalOp:
2142+
default: legacy
2143+
enum:
2144+
- legacy
2145+
- AND
2146+
- OR
2147+
type: string
2148+
Match:
2149+
type: string
2150+
Regex:
2151+
items:
2152+
type: string
2153+
type: array
2154+
type: object
21352155
filterKubernetes:
21362156
properties:
21372157
Annotations:

‎charts/logging-operator/crds/logging.banzaicloud.io_fluentbitagents.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,6 +1273,26 @@ spec:
12731273
vpc_id:
12741274
type: boolean
12751275
type: object
1276+
filterGrep:
1277+
properties:
1278+
Exclude:
1279+
items:
1280+
type: string
1281+
type: array
1282+
LogicalOp:
1283+
default: legacy
1284+
enum:
1285+
- legacy
1286+
- AND
1287+
- OR
1288+
type: string
1289+
Match:
1290+
type: string
1291+
Regex:
1292+
items:
1293+
type: string
1294+
type: array
1295+
type: object
12761296
filterKubernetes:
12771297
properties:
12781298
Annotations:

‎charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2129,6 +2129,26 @@ spec:
21292129
vpc_id:
21302130
type: boolean
21312131
type: object
2132+
filterGrep:
2133+
properties:
2134+
Exclude:
2135+
items:
2136+
type: string
2137+
type: array
2138+
LogicalOp:
2139+
default: legacy
2140+
enum:
2141+
- legacy
2142+
- AND
2143+
- OR
2144+
type: string
2145+
Match:
2146+
type: string
2147+
Regex:
2148+
items:
2149+
type: string
2150+
type: array
2151+
type: object
21322152
filterKubernetes:
21332153
properties:
21342154
Annotations:

‎config/crd/bases/logging.banzaicloud.io_fluentbitagents.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,6 +1273,26 @@ spec:
12731273
vpc_id:
12741274
type: boolean
12751275
type: object
1276+
filterGrep:
1277+
properties:
1278+
Exclude:
1279+
items:
1280+
type: string
1281+
type: array
1282+
LogicalOp:
1283+
default: legacy
1284+
enum:
1285+
- legacy
1286+
- AND
1287+
- OR
1288+
type: string
1289+
Match:
1290+
type: string
1291+
Regex:
1292+
items:
1293+
type: string
1294+
type: array
1295+
type: object
12761296
filterKubernetes:
12771297
properties:
12781298
Annotations:

‎config/crd/bases/logging.banzaicloud.io_loggings.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2129,6 +2129,26 @@ spec:
21292129
vpc_id:
21302130
type: boolean
21312131
type: object
2132+
filterGrep:
2133+
properties:
2134+
Exclude:
2135+
items:
2136+
type: string
2137+
type: array
2138+
LogicalOp:
2139+
default: legacy
2140+
enum:
2141+
- legacy
2142+
- AND
2143+
- OR
2144+
type: string
2145+
Match:
2146+
type: string
2147+
Regex:
2148+
items:
2149+
type: string
2150+
type: array
2151+
type: object
21322152
filterKubernetes:
21332153
properties:
21342154
Annotations:

‎docs/configuration/crds/v1beta1/fluentbit_types.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ DisableVarLibDockerContainers controls whether the /var/lib/docker/containers vo
119119
### filterAws (*FilterAws, optional) {#fluentbitspec-filteraws}
120120

121121

122+
### filterGrep (*FilterGrep, optional) {#fluentbitspec-filtergrep}
123+
124+
122125
### filterKubernetes (FilterKubernetes, optional) {#fluentbitspec-filterkubernetes}
123126

124127
Parameters for Kubernetes metadata filter
@@ -848,6 +851,32 @@ The VPC ID for current EC2 instance. (default:false)
848851
Default: false
849852

850853

854+
## FilterGrep
855+
856+
FilterGrep The Grep Filter plugin
857+
858+
### Exclude ([]string, optional) {#filtergrep-exclude}
859+
860+
Exclude records where the content of KEY matches the regular expression.
861+
862+
863+
### LogicalOp (string, optional) {#filtergrep-logicalop}
864+
865+
Specify a logical operator: AND, OR or legacy (default). In legacy mode the behavior is either AND or OR depending on whether the grep is including (uses AND) or excluding (uses OR). Available from 2.1 or higher. Default: "legacy"
866+
867+
868+
### Match (string, optional) {#filtergrep-match}
869+
870+
Match filtered records (default:*)
871+
872+
Default: *
873+
874+
### Regex ([]string, optional) {#filtergrep-regex}
875+
876+
Keep records where the content of KEY matches the regular expression.
877+
878+
879+
851880
## FilterModify
852881

853882
FilterModify The Modify Filter plugin allows you to change records using rules and conditions.

‎pkg/resources/fluentbit/config.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,24 @@ var fluentBitConfigTemplate = `
7575
{{- template "input" .Input }}
7676
{{- end }}
7777
78+
{{- if .FluentdFilterGrep }}
79+
[FILTER]
80+
Name grep
81+
Match {{ .FluentdFilterGrep.Match }}
82+
83+
{{- if .FluentdFilterGrep.LogicalOp }}
84+
Logical_Op {{ .FluentdFilterGrep.LogicalOp }}
85+
{{- end }}
86+
87+
{{- range $value := .FluentdFilterGrep.Regex }}
88+
Regex {{ $value }}
89+
{{- end }}
90+
91+
{{- range $value := .FluentdFilterGrep.Exclude }}
92+
Exclude {{ $value }}
93+
{{- end }}
94+
{{- end}}
95+
7896
{{- if not .DisableKubernetesFilter }}
7997
[FILTER]
8098
Name kubernetes

‎pkg/resources/fluentbit/configsecret.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ type fluentBitConfig struct {
7676
DisableKubernetesFilter bool
7777
KubernetesFilter map[string]string
7878
AwsFilter map[string]string
79+
FluentdFilterGrep *FluentdFilterGrep
7980
BufferStorage map[string]string
8081
FilterModify []v1beta1.FilterModify
8182
FluentForwardOutput *fluentForwardOutputConfig
@@ -86,6 +87,13 @@ type fluentBitConfig struct {
8687
HealthCheck *v1beta1.HealthCheck
8788
}
8889

90+
type FluentdFilterGrep struct {
91+
Match string
92+
Regex []string
93+
Exclude []string
94+
LogicalOp string
95+
}
96+
8997
type fluentForwardOutputConfig struct {
9098
Network FluentbitNetwork
9199
Options map[string]string
@@ -319,6 +327,13 @@ func (r *Reconciler) configSecret() (runtime.Object, reconciler.DesiredState, er
319327
}
320328
input.Input.Values = fluentbitInputValues
321329

330+
if r.fluentbitSpec.FilterGrep != nil {
331+
input.FluentdFilterGrep, err = toFluentdFilterGrep(r.fluentbitSpec.FilterGrep)
332+
if err != nil {
333+
return nil, reconciler.StatePresent, err
334+
}
335+
}
336+
322337
input.KubernetesFilter, err = mapper.StringsMap(r.fluentbitSpec.FilterKubernetes)
323338
if err != nil {
324339
return nil, reconciler.StatePresent, errors.WrapIf(err, "failed to map kubernetes filter for fluentbit")
@@ -460,6 +475,21 @@ func (r *Reconciler) configSecret() (runtime.Object, reconciler.DesiredState, er
460475
}, reconciler.StatePresent, nil
461476
}
462477

478+
func toFluentdFilterGrep(filterGrep *v1beta1.FilterGrep) (*FluentdFilterGrep, error) {
479+
if filterGrep.LogicalOp != "legacy" && len(filterGrep.Regex) > 0 && len(filterGrep.Exclude) > 0 {
480+
return nil, errors.New("failed to parse grep filter for fluentbit, LogicalOp is set, it's not possible to set both Regex and Exclude")
481+
}
482+
483+
fluentdFilterGrep := &FluentdFilterGrep{
484+
Match: filterGrep.Match,
485+
Regex: filterGrep.Regex,
486+
Exclude: filterGrep.Exclude,
487+
LogicalOp: filterGrep.LogicalOp,
488+
}
489+
490+
return fluentdFilterGrep, nil
491+
}
492+
463493
func (r *Reconciler) applyNetworkSettings(input fluentBitConfig) {
464494
if r.fluentbitSpec.Network != nil {
465495
if input.FluentForwardOutput != nil {

‎pkg/resources/fluentbit/configsecret_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright © 2025 Kube logging authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package fluentbit
16+
17+
import (
18+
"testing"
19+
20+
"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
21+
"github.com/stretchr/testify/assert"
22+
)
23+
24+
func TestInvalidFilterGrepConfig(t *testing.T) {
25+
invalidFilterGrep := &v1beta1.FilterGrep{
26+
Match: "*",
27+
Regex: []string{"regex", "reg2"},
28+
Exclude: []string{"exclude"},
29+
LogicalOp: "AND",
30+
}
31+
32+
_, err := toFluentdFilterGrep(invalidFilterGrep)
33+
34+
assert.EqualError(t, err, "failed to parse grep filter for fluentbit, LogicalOp is set, it's not possible to set both Regex and Exclude")
35+
}
36+
37+
func TestValidFilterGrepConfig(t *testing.T) {
38+
filterGrep := &v1beta1.FilterGrep{
39+
Match: "*",
40+
Regex: []string{"regex1", "regex2"},
41+
LogicalOp: "AND",
42+
}
43+
44+
expectedFluentFilterGrep := &FluentdFilterGrep{
45+
Match: "*",
46+
Regex: []string{"regex1", "regex2"},
47+
LogicalOp: "AND",
48+
}
49+
50+
parserFluentdFilterGrep, err := toFluentdFilterGrep(filterGrep)
51+
52+
assert.NoError(t, err)
53+
assert.EqualValues(t, parserFluentdFilterGrep, expectedFluentFilterGrep)
54+
}

0 commit comments

Comments
 (0)