refactor: create generic arrow codec

Author: benjamin-awd

Cargo.lock

@@ -339,7 +339,6 @@ arc-swap = { version = "1.7", default-features = false, optional = true }
 async-compression = { version = "0.4.27", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true }
 apache-avro = { version = "0.16.0", default-features = false, optional = true }
 arrow = { version = "56.2.0", default-features = false, features = ["ipc"], optional = true }
-arrow-schema = { version = "56.2.0", default-features = false, optional = true }
 axum = { version = "0.6.20", default-features = false }
 base64 = { workspace = true, optional = true }
 bloomy = { version = "1.2.0", default-features = false, optional = true }
@@ -580,8 +579,9 @@ enrichment-tables-mmdb = ["dep:maxminddb"]
 enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"]
 
 # Codecs
-codecs-syslog = ["vector-lib/syslog"]
+codecs-arrow = ["vector-lib/arrow"]
 codecs-opentelemetry = ["vector-lib/opentelemetry"]
+codecs-syslog = ["vector-lib/syslog"]
 
 # Secrets
 secrets = ["secrets-aws-secrets-manager"]
@@ -845,7 +845,7 @@ sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage",
 sinks-azure_monitor_logs = []
 sinks-blackhole = []
 sinks-chronicle = []
-sinks-clickhouse = ["dep:arrow", "dep:arrow-schema", "dep:rust_decimal"]
+sinks-clickhouse = []
 sinks-console = []
 sinks-databend = ["dep:databend-client"]
 sinks-datadog_events = []

Cargo.toml

@@ -0,0 +1,3 @@
+A generic Arrow codec has been added to support Apache Arrow IPC serialization across Vector. This enables sinks like `clickhouse` sink to use the ArrowStream format endpoint with significantly better performance and smaller payload sizes compared to JSON-based formats.
+
+authors: benjamin-awd

changelog.d/24074_arrow_batch_codec.feature.md

@@ -1,3 +1,3 @@
-The `clickhouse` sink now supports the `arrow_stream` format option, enabling high-performance binary data transfer using Apache Arrow IPC via Clickhouse's ArrowStream format endpoint. This provides significantly better performance and smaller payload sizes compared to JSON-based formats.
+A generic Arrow codec has been added to support Apache Arrow IPC serialization across Vector. This enables sinks like `clickhouse` sink to use the ArrowStream format endpoint with significantly better performance and smaller payload sizes compared to JSON-based formats.
 
 authors: benjamin-awd

changelog.d/24074_clickhouse_arrow_format.enhancement.md

@@ -11,8 +11,10 @@ path = "tests/bin/generate-avro-fixtures.rs"
 
 [dependencies]
 apache-avro = { version = "0.16.0", default-features = false }
+arrow = { version = "56.2.0", default-features = false, features = ["ipc"] }
 bytes.workspace = true
 chrono.workspace = true
+rust_decimal = { version = "1.37", default-features = false, features = ["std"] }
 csv-core = { version = "0.1.12", default-features = false }
 derivative.workspace = true
 dyn-clone = { version = "1", default-features = false }
@@ -53,5 +55,6 @@ uuid.workspace = true
 vrl.workspace = true
 
 [features]
-syslog = ["dep:syslog_loose"]
+arrow = []
 opentelemetry = ["dep:opentelemetry-proto"]
+syslog = ["dep:syslog_loose"]