auto gen plugins documentation directory

cmd/doc-gen/main.go

@@ -130,6 +130,11 @@ func plugins(docsLocations []DocumentsLocation) {
 			}
 
 			dst := fmt.Sprintf("./docs/plugins/%s/%s.md", dl.name, src_name)
+
+			if err := genDocDirs(dst); err != nil {
+				fmt.Printf("Error while generating documentation directories: %s\n", err.Error())
+			}
+
 			f, err := os.Create(dst)
 			if err != nil {
 				fmt.Printf("Error while generating documentation: %s\n", err.Error())
@@ -183,12 +188,20 @@ func crds(docsLocations []DocumentsLocation) {
 				buffer.WriteString("[Back to TOC](#table-of-contents)\n")
 			}
 		}
-
+		
 		f, _ := os.Create(fmt.Sprintf("./docs/%s.md", dl.name))
 		f.WriteString(fmt.Sprintf(firstParagraph, dl.name) + buffer.String())
 	}
 }
 
+func genDocDirs(docPath string) error {
+	dirPath := filepath.Dir(docPath)
+	if _, err := os.Stat(dirPath); os.IsNotExist(err) {
+		return os.MkdirAll(dirPath, os.ModePerm)
+	}
+	return nil
+}
+
 func toSectionLink(name string) string {
 	name = strings.ToLower(name)
 	name = strings.Replace(name, " ", "-", -1)

docs/fluentbit.md

@@ -8,6 +8,8 @@ This Document documents the types introduced by the fluentbit Operator.
 * [ClusterFluentBitConfigList](#clusterfluentbitconfiglist)
 * [ClusterInput](#clusterinput)
 * [ClusterInputList](#clusterinputlist)
+* [ClusterMultilineParser](#clustermultilineparser)
+* [ClusterMultilineParserList](#clustermultilineparserlist)
 * [ClusterOutput](#clusteroutput)
 * [ClusterOutputList](#clusteroutputlist)
 * [ClusterParser](#clusterparser)
@@ -29,6 +31,8 @@ This Document documents the types introduced by the fluentbit Operator.
 * [FluentBitService](#fluentbitservice)
 * [FluentBitSpec](#fluentbitspec)
 * [InputSpec](#inputspec)
+* [MultilineParser](#multilineparser)
+* [MultilineParserList](#multilineparserlist)
 * [NamespacedFluentBitCfgSpec](#namespacedfluentbitcfgspec)
 * [Output](#output)
 * [OutputList](#outputlist)
@@ -104,6 +108,28 @@ ClusterInputList contains a list of ClusterInput
 | metadata |  | [metav1.ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#listmeta-v1-meta) |
 | items |  | []ClusterInput |
 
+[Back to TOC](#table-of-contents)
+# ClusterMultilineParser
+
+ClusterMultilineParser is the Schema for the cluster-level multiline parser API
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| metadata |  | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#objectmeta-v1-meta) |
+| spec |  | MultilineParserSpec |
+
+[Back to TOC](#table-of-contents)
+# ClusterMultilineParserList
+
+ClusterMultilineParserList contains a list of ClusterMultilineParser
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| metadata |  | [metav1.ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#listmeta-v1-meta) |
+| items |  | []ClusterMultilineParser |
+
 [Back to TOC](#table-of-contents)
 # ClusterOutput
 
@@ -331,6 +357,7 @@ FluentBitConfigSpec defines the desired state of ClusterFluentBitConfig
 | filterSelector | Select filter plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
 | outputSelector | Select output plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
 | parserSelector | Select parser plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
+| multilineParserSelector | Select multiline parser plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
 | namespace | If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. If it is not defined, it is in the namespace of the fluentd-operator | *string |
 
 [Back to TOC](#table-of-contents)
@@ -429,6 +456,28 @@ InputSpec defines the desired state of ClusterInput
 | syslog | Syslog defines the Syslog input plugin configuration | *[input.Syslog](plugins/input/syslog.md) |
 | tcp | TCP defines the TCP input plugin configuration | *[input.TCP](plugins/input/tcp.md) |
 
+[Back to TOC](#table-of-contents)
+# MultilineParser
+
+MultilineParser is the Schema of namespace-level multiline parser API
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| metadata |  | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#objectmeta-v1-meta) |
+| spec |  | MultilineParserSpec |
+
+[Back to TOC](#table-of-contents)
+# MultilineParserList
+
+MultilineParserList contains a list of MultilineParser
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| metadata |  | [metav1.ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#listmeta-v1-meta) |
+| items |  | []MultilineParser |
+
 [Back to TOC](#table-of-contents)
 # NamespacedFluentBitCfgSpec
 
@@ -440,7 +489,10 @@ NamespacedFluentBitCfgSpec defines the desired state of FluentBit
 | filterSelector | Select filter plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
 | outputSelector | Select output plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
 | parserSelector | Select parser plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
+| service | Service defines the global behaviour of the Fluent Bit engine. | *Service |
 | clusterParserSelector | Select cluster level parser config | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
+| multilineParserSelector | Select multiline parser plugins | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
+| clusterMultilineParserSelector | Select cluster level multiline parser config | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#labelselector-v1-meta) |
 
 [Back to TOC](#table-of-contents)
 # Output
@@ -572,7 +624,11 @@ ParserSpec defines the desired state of ClusterParser
 | logFile | File to log diagnostic output | string |
 | logLevel | Diagnostic level (error/warning/info/debug/trace) | string |
 | parsersFile | Optional 'parsers' config file (can be multiple) | string |
+| parsersFiles | backward compatible | []string |
 | storage | Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount separately for this storage. The hostPath type should be used for that Volume in Fluentbit daemon set. | *Storage |
+| emitterName | Per-namespace re-emitter configuration | string |
+| emitterMemBufLimit |  | string |
+| emitterStorageType |  | string |
 
 [Back to TOC](#table-of-contents)
 # Storage

docs/plugins/fluentbit/filter/lua.md

@@ -7,6 +7,7 @@ The Lua Filter allows you to modify the incoming records using custom Lua Script
 | ----- | ----------- | ------ |
 | script | Path to the Lua script that will be used. | v1.ConfigMapKeySelector |
 | call | Lua function name that will be triggered to do filtering. It's assumed that the function is declared inside the Script defined above. | string |
+| code | Inline LUA code instead of loading from a path via script. | string |
 | typeIntKey | If these keys are matched, the fields are converted to integer. If more than one key, delimit by space. Note that starting from Fluent Bit v1.6 integer data types are preserved and not converted to double as in previous versions. | []string |
 | protectedMode | If enabled, Lua script will be executed in protected mode. It prevents to crash when invalid Lua script is executed. Default is true. | *bool |
 | timeAsTable | By default when the Lua script is invoked, the record timestamp is passed as a Floating number which might lead to loss precision when the data is converted back. If you desire timestamp precision enabling this option will pass the timestamp as a Lua table with keys sec for seconds since epoch and nsec for nanoseconds. | bool |

docs/plugins/fluentbit/filter/multiline.md

@@ -7,3 +7,9 @@
 | ----- | ----------- | ------ |
 | parser | Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline parsers to detect different formats by separating them with a comma. | string |
 | keyContent | Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. | string |
+| mode |  | string |
+| buffer |  | bool |
+| flushMs |  | int |
+| emitterName | Name for the emitter input instance which re-emits the completed records at the beginning of the pipeline. | string |
+| emitterType | The storage type for the emitter input instance. This option supports the values memory (default) and filesystem. | string |
+| emitterMemBufLimit | Set a limit on the amount of memory in MB the emitter can consume if the outputs provide backpressure. The default for this limit is 10M. The pipeline will pause once the buffer exceeds the value of this setting. For example, if the value is set to 10MB then the pipeline will pause if the buffer exceeds 10M. The pipeline will remain paused until the output drains the buffer below the 10M limit. | int |

docs/plugins/fluentbit/filter/rewritetag.md

@@ -7,3 +7,5 @@ RewriteTag define a `rewrite_tag` filter, allows to re-emit a record under a new
 | ----- | ----------- | ------ |
 | rules | Defines the matching criteria and the format of the Tag for the matching record. The Rule format have four components: KEY REGEX NEW_TAG KEEP. | []string |
 | emitterName | When the filter emits a record under the new Tag, there is an internal emitter plugin that takes care of the job. Since this emitter expose metrics as any other component of the pipeline, you can use this property to configure an optional name for it. | string |
+| emitterMemBufLimit |  | string |
+| emitterStorageType |  | string |

docs/plugins/fluentbit/multilineparser/multilineparser.md

@@ -0,0 +1,22 @@
+# MultilineParser
+
+**For full documentation, refer to https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/multiline-parsing**
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| type | Set the multiline mode, for now, we support the type regex. | string |
+| parser | Name of a pre-defined parser that must be applied to the incoming content before applying the regex rule. If no parser is defined, it's assumed that's a raw text and not a structured message. | string |
+| keyContent | For an incoming structured message, specify the key that contains the data that should be processed by the regular expression and possibly concatenated. | string |
+| flushTimeout | Timeout in milliseconds to flush a non-terminated multiline buffer. Default is set to 5 seconds. | int |
+| rules | Configure a rule to match a multiline pattern. The rule has a specific format described below. Multiple rules can be defined. | []Rule |
+# Rule
+
+
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| start |  | string |
+| regex |  | string |
+| next |  | string |