Skip to content

Commit 6c6016a

Browse files
wenchajunwenchajun
authored
Merge pull request #1462 from penekk/add-fluentd-elasticsearch-parameters
feat: Add support for reload_after, sniffer_class_name es output parameters
2 parents dd91ab5 + 68171fe commit 6c6016a

File tree

9 files changed

+174
-0
lines changed

9 files changed

+174
-0
lines changed

‎apis/fluentd/v1alpha1/plugins/output/es.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,17 @@ type ElasticsearchCommon struct {
4343
FailOnPuttingTemplateRetryExceeded *bool `json:"failOnPuttingTemplateRetryExceeded,omitempty"`
4444
// Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false)
4545
ReconnectOnError *bool `json:"reconnectOnError,omitempty"`
46+
// Optional, When ReloadConnections true, this is the integer number of operations after which the plugin will reload the connections. The default value is 10000.
47+
ReloadAfter *uint32 `json:"reloadAfter,omitempty"`
4648
// Optional, Automatically reload connection after 10000 documents (default: true)
4749
ReloadConnections *bool `json:"reloadConnections,omitempty"`
4850
// Optional, Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the request, this can be useful to quickly remove a dead node from the list of addresses (default: false)
4951
ReloadOnFailure *bool `json:"reloadOnFailure,omitempty"`
5052
// Optional, HTTP Timeout (default: 5)
5153
// +kubebuilder:validation:Pattern:="^\\d+(s|m|h|d)$"
5254
RequestTimeout *string `json:"requestTimeout,omitempty"`
55+
// Optional, Provide a different sniffer class name
56+
SnifferClassName *string `json:"snifferClassName,omitempty"`
5357
// Optional, Suppress '[types removal]' warnings on elasticsearch 7.x
5458
SuppressTypeName *bool `json:"suppressTypeName,omitempty"`
5559
// Optional, Enable Index Lifecycle Management (ILM)

‎apis/fluentd/v1alpha1/plugins/output/types.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,10 @@ func (o *Output) elasticsearchPluginCommon(common *ElasticsearchCommon, parent *
467467
parent.InsertPairs("reconnect_on_error", fmt.Sprint(*common.ReconnectOnError))
468468
}
469469

470+
if common.ReloadAfter != nil {
471+
parent.InsertPairs("reload_after", fmt.Sprint(*common.ReloadAfter))
472+
}
473+
470474
if common.ReloadConnections != nil {
471475
parent.InsertPairs("reload_connections", fmt.Sprint(*common.ReloadConnections))
472476
}
@@ -479,6 +483,10 @@ func (o *Output) elasticsearchPluginCommon(common *ElasticsearchCommon, parent *
479483
parent.InsertPairs("request_timeout", fmt.Sprint(*common.RequestTimeout))
480484
}
481485

486+
if common.SnifferClassName != nil {
487+
parent.InsertPairs("sniffer_class_name", fmt.Sprint(*common.SnifferClassName))
488+
}
489+
482490
if common.SuppressTypeName != nil {
483491
parent.InsertPairs("suppress_type_name", fmt.Sprint(*common.SuppressTypeName))
484492
}

‎charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,12 @@ spec:
803803
reset connection on any error (reconnect on next send)
804804
(default: false)'
805805
type: boolean
806+
reloadAfter:
807+
description: Optional, When ReloadConnections true, this
808+
is the integer number of operations after which the plugin
809+
will reload the connections. The default value is 10000.
810+
format: int32
811+
type: integer
806812
reloadConnections:
807813
description: 'Optional, Automatically reload connection
808814
after 10000 documents (default: true)'
@@ -822,6 +828,10 @@ spec:
822828
description: 'Specify https if your Elasticsearch endpoint
823829
supports SSL (default: http).'
824830
type: string
831+
snifferClassName:
832+
description: Optional, Provide a different sniffer class
833+
name
834+
type: string
825835
sslVerify:
826836
description: Optional, Force certificate validation
827837
type: boolean
@@ -1118,6 +1128,12 @@ spec:
11181128
reset connection on any error (reconnect on next send)
11191129
(default: false)'
11201130
type: boolean
1131+
reloadAfter:
1132+
description: Optional, When ReloadConnections true, this
1133+
is the integer number of operations after which the plugin
1134+
will reload the connections. The default value is 10000.
1135+
format: int32
1136+
type: integer
11211137
reloadConnections:
11221138
description: 'Optional, Automatically reload connection
11231139
after 10000 documents (default: true)'
@@ -1137,6 +1153,10 @@ spec:
11371153
description: 'Specify https if your Elasticsearch endpoint
11381154
supports SSL (default: http).'
11391155
type: string
1156+
snifferClassName:
1157+
description: Optional, Provide a different sniffer class
1158+
name
1159+
type: string
11401160
sslVerify:
11411161
description: Optional, Force certificate validation
11421162
type: boolean

‎charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,12 @@ spec:
803803
reset connection on any error (reconnect on next send)
804804
(default: false)'
805805
type: boolean
806+
reloadAfter:
807+
description: Optional, When ReloadConnections true, this
808+
is the integer number of operations after which the plugin
809+
will reload the connections. The default value is 10000.
810+
format: int32
811+
type: integer
806812
reloadConnections:
807813
description: 'Optional, Automatically reload connection
808814
after 10000 documents (default: true)'
@@ -822,6 +828,10 @@ spec:
822828
description: 'Specify https if your Elasticsearch endpoint
823829
supports SSL (default: http).'
824830
type: string
831+
snifferClassName:
832+
description: Optional, Provide a different sniffer class
833+
name
834+
type: string
825835
sslVerify:
826836
description: Optional, Force certificate validation
827837
type: boolean
@@ -1118,6 +1128,12 @@ spec:
11181128
reset connection on any error (reconnect on next send)
11191129
(default: false)'
11201130
type: boolean
1131+
reloadAfter:
1132+
description: Optional, When ReloadConnections true, this
1133+
is the integer number of operations after which the plugin
1134+
will reload the connections. The default value is 10000.
1135+
format: int32
1136+
type: integer
11211137
reloadConnections:
11221138
description: 'Optional, Automatically reload connection
11231139
after 10000 documents (default: true)'
@@ -1137,6 +1153,10 @@ spec:
11371153
description: 'Specify https if your Elasticsearch endpoint
11381154
supports SSL (default: http).'
11391155
type: string
1156+
snifferClassName:
1157+
description: Optional, Provide a different sniffer class
1158+
name
1159+
type: string
11401160
sslVerify:
11411161
description: Optional, Force certificate validation
11421162
type: boolean

‎config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,12 @@ spec:
803803
reset connection on any error (reconnect on next send)
804804
(default: false)'
805805
type: boolean
806+
reloadAfter:
807+
description: Optional, When ReloadConnections true, this
808+
is the integer number of operations after which the plugin
809+
will reload the connections. The default value is 10000.
810+
format: int32
811+
type: integer
806812
reloadConnections:
807813
description: 'Optional, Automatically reload connection
808814
after 10000 documents (default: true)'
@@ -822,6 +828,10 @@ spec:
822828
description: 'Specify https if your Elasticsearch endpoint
823829
supports SSL (default: http).'
824830
type: string
831+
snifferClassName:
832+
description: Optional, Provide a different sniffer class
833+
name
834+
type: string
825835
sslVerify:
826836
description: Optional, Force certificate validation
827837
type: boolean
@@ -1118,6 +1128,12 @@ spec:
11181128
reset connection on any error (reconnect on next send)
11191129
(default: false)'
11201130
type: boolean
1131+
reloadAfter:
1132+
description: Optional, When ReloadConnections true, this
1133+
is the integer number of operations after which the plugin
1134+
will reload the connections. The default value is 10000.
1135+
format: int32
1136+
type: integer
11211137
reloadConnections:
11221138
description: 'Optional, Automatically reload connection
11231139
after 10000 documents (default: true)'
@@ -1137,6 +1153,10 @@ spec:
11371153
description: 'Specify https if your Elasticsearch endpoint
11381154
supports SSL (default: http).'
11391155
type: string
1156+
snifferClassName:
1157+
description: Optional, Provide a different sniffer class
1158+
name
1159+
type: string
11401160
sslVerify:
11411161
description: Optional, Force certificate validation
11421162
type: boolean

‎config/crd/bases/fluentd.fluent.io_outputs.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,12 @@ spec:
803803
reset connection on any error (reconnect on next send)
804804
(default: false)'
805805
type: boolean
806+
reloadAfter:
807+
description: Optional, When ReloadConnections true, this
808+
is the integer number of operations after which the plugin
809+
will reload the connections. The default value is 10000.
810+
format: int32
811+
type: integer
806812
reloadConnections:
807813
description: 'Optional, Automatically reload connection
808814
after 10000 documents (default: true)'
@@ -822,6 +828,10 @@ spec:
822828
description: 'Specify https if your Elasticsearch endpoint
823829
supports SSL (default: http).'
824830
type: string
831+
snifferClassName:
832+
description: Optional, Provide a different sniffer class
833+
name
834+
type: string
825835
sslVerify:
826836
description: Optional, Force certificate validation
827837
type: boolean
@@ -1118,6 +1128,12 @@ spec:
11181128
reset connection on any error (reconnect on next send)
11191129
(default: false)'
11201130
type: boolean
1131+
reloadAfter:
1132+
description: Optional, When ReloadConnections true, this
1133+
is the integer number of operations after which the plugin
1134+
will reload the connections. The default value is 10000.
1135+
format: int32
1136+
type: integer
11211137
reloadConnections:
11221138
description: 'Optional, Automatically reload connection
11231139
after 10000 documents (default: true)'
@@ -1137,6 +1153,10 @@ spec:
11371153
description: 'Specify https if your Elasticsearch endpoint
11381154
supports SSL (default: http).'
11391155
type: string
1156+
snifferClassName:
1157+
description: Optional, Provide a different sniffer class
1158+
name
1159+
type: string
11401160
sslVerify:
11411161
description: Optional, Force certificate validation
11421162
type: boolean

‎docs/plugins/fluentd/output/es.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@ Elasticsearch defines the parameters for out_es output plugin
2323
| maxRetryPuttingTemplate | Optional, You can specify times of retry putting template (default: 10) | *uint32 |
2424
| failOnPuttingTemplateRetryExceeded | Optional, Indicates whether to fail when max_retry_putting_template is exceeded. If you have multiple output plugin, you could use this property to do not fail on fluentd statup (default: false) | *bool |
2525
| reconnectOnError | Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false) | *bool |
26+
| reloadAfter | Optional, When ReloadConnections true, this is the integer number of operations after which the plugin will reload the connections. The default value is 10000. | *uint32 |
2627
| reloadConnections | Optional, Automatically reload connection after 10000 documents (default: true) | *bool |
2728
| reloadOnFailure | Optional, Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the request, this can be useful to quickly remove a dead node from the list of addresses (default: false) | *bool |
2829
| requestTimeout | Optional, HTTP Timeout (default: 5) | *string |
30+
| snifferClassName | Optional, Provide a different sniffer class name | *string |
2931
| suppressTypeName | Optional, Suppress '[types removal]' warnings on elasticsearch 7.x | *bool |
3032
| enableIlm | Optional, Enable Index Lifecycle Management (ILM) | *bool |
3133
| ilmPolicyId | Optional, Specify ILM policy id | *string |

‎manifests/setup/fluent-operator-crd.yaml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9287,6 +9287,12 @@ spec:
92879287
reset connection on any error (reconnect on next send)
92889288
(default: false)'
92899289
type: boolean
9290+
reloadAfter:
9291+
description: Optional, When ReloadConnections true, this
9292+
is the integer number of operations after which the plugin
9293+
will reload the connections. The default value is 10000.
9294+
format: int32
9295+
type: integer
92909296
reloadConnections:
92919297
description: 'Optional, Automatically reload connection
92929298
after 10000 documents (default: true)'
@@ -9306,6 +9312,10 @@ spec:
93069312
description: 'Specify https if your Elasticsearch endpoint
93079313
supports SSL (default: http).'
93089314
type: string
9315+
snifferClassName:
9316+
description: Optional, Provide a different sniffer class
9317+
name
9318+
type: string
93099319
sslVerify:
93109320
description: Optional, Force certificate validation
93119321
type: boolean
@@ -9602,6 +9612,12 @@ spec:
96029612
reset connection on any error (reconnect on next send)
96039613
(default: false)'
96049614
type: boolean
9615+
reloadAfter:
9616+
description: Optional, When ReloadConnections true, this
9617+
is the integer number of operations after which the plugin
9618+
will reload the connections. The default value is 10000.
9619+
format: int32
9620+
type: integer
96059621
reloadConnections:
96069622
description: 'Optional, Automatically reload connection
96079623
after 10000 documents (default: true)'
@@ -9621,6 +9637,10 @@ spec:
96219637
description: 'Specify https if your Elasticsearch endpoint
96229638
supports SSL (default: http).'
96239639
type: string
9640+
snifferClassName:
9641+
description: Optional, Provide a different sniffer class
9642+
name
9643+
type: string
96249644
sslVerify:
96259645
description: Optional, Force certificate validation
96269646
type: boolean
@@ -38097,6 +38117,12 @@ spec:
3809738117
reset connection on any error (reconnect on next send)
3809838118
(default: false)'
3809938119
type: boolean
38120+
reloadAfter:
38121+
description: Optional, When ReloadConnections true, this
38122+
is the integer number of operations after which the plugin
38123+
will reload the connections. The default value is 10000.
38124+
format: int32
38125+
type: integer
3810038126
reloadConnections:
3810138127
description: 'Optional, Automatically reload connection
3810238128
after 10000 documents (default: true)'
@@ -38116,6 +38142,10 @@ spec:
3811638142
description: 'Specify https if your Elasticsearch endpoint
3811738143
supports SSL (default: http).'
3811838144
type: string
38145+
snifferClassName:
38146+
description: Optional, Provide a different sniffer class
38147+
name
38148+
type: string
3811938149
sslVerify:
3812038150
description: Optional, Force certificate validation
3812138151
type: boolean
@@ -38412,6 +38442,12 @@ spec:
3841238442
reset connection on any error (reconnect on next send)
3841338443
(default: false)'
3841438444
type: boolean
38445+
reloadAfter:
38446+
description: Optional, When ReloadConnections true, this
38447+
is the integer number of operations after which the plugin
38448+
will reload the connections. The default value is 10000.
38449+
format: int32
38450+
type: integer
3841538451
reloadConnections:
3841638452
description: 'Optional, Automatically reload connection
3841738453
after 10000 documents (default: true)'
@@ -38431,6 +38467,10 @@ spec:
3843138467
description: 'Specify https if your Elasticsearch endpoint
3843238468
supports SSL (default: http).'
3843338469
type: string
38470+
snifferClassName:
38471+
description: Optional, Provide a different sniffer class
38472+
name
38473+
type: string
3843438474
sslVerify:
3843538475
description: Optional, Force certificate validation
3843638476
type: boolean

0 commit comments

Comments
 (0)