bump(librdkafka): from v1.9.2 to v2.3.0 (SeldonIO#5321)

* bump(librdkafka): from v1.9.2 to v2.3.0

- update needed for automatic re-authentication of kafka connections via
SASL (used when connecting to Confluent Cloud via OAUTHBEARER)

* fix(librdkafka): Copy certs bundle to path expected by librdkafka v2.3.0

librdkafka uses libcurl for fetching OIDC tokens. However, depending on OS, the
location where the certs ca-bundle for validating the tls connection between
librdkafka and the auth provider is searched differs. We now copy the ca cert
bundle to multiple paths to make sure it is found.

While this is a bit hacky, there are limited alternatives we can take because:

- until now (with librdkafka 1.9.2) those certs were searched under
/etc/ssl/certs/ca-certificates.crt but after the update they are searched
under /etc/pki/tls/certs/ca-bundle.crt

- libcurl doesn't get passed/doesn't respect the caCert location
or .pem file set via librdkafka/kafka configs
(see confluentinc/librdkafka#375).

- there is no way clear way to influence the libcurl search path via env vars
(setting the CURL_CA_BUNDLE env var has no effect), with search paths fully
decided at compile-time.

The likely chain of reasons this has popped-up when updating librdkafka to
v2.3.0:
  - we're building {model, pipeline}gateway docker images based on debian
  bullseye image
  - we're copying the build results into redhat ubi9 container
  - libcurl was statically built in previous librdkafka, so was
  using the debian search paths on ubi9
  - libcurl ends up being dynamically linked in the current librdkafka
  (to ubi9 version) or has changed its default ca bundle search path

Author: lc525

hodometer/go.sum

@@ -379,7 +379,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -646,6 +646,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

operator/go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	emperror.dev/errors v0.8.1
 	github.com/banzaicloud/k8s-objectmatcher v1.8.0
-	github.com/confluentinc/confluent-kafka-go v1.9.2
+	github.com/confluentinc/confluent-kafka-go/v2 v2.3.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v1.4.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -27,6 +27,12 @@ require (
 	sigs.k8s.io/controller-runtime v0.12.3
 )
 
+require (
+	github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.0 // indirect
+)
+
 require (
 	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect

operator/go.sum

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/confluentinc/confluent-kafka-go/kafka"
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 

operator/pkg/cli/kafka.go

@@ -13,8 +13,9 @@ FROM registry.access.redhat.com/ubi9/ubi-minimal as certs
 # Kafka dependencies necessitate leaving CGo enabled and using a base image with C dependencies
 FROM registry.access.redhat.com/ubi9/ubi-micro:9.3-13
 
-# Kafka OIDC token retriva certs: https://github.com/confluentinc/librdkafka/issues/3751
+# Kafka OIDC token retrieve certs (librdkafka using curl): https://github.com/confluentinc/librdkafka/issues/3751
 COPY --from=certs /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=certs /etc/ssl/certs/ca-bundle.crt /etc/pki/tls/certs/ca-bundle.crt
 
 # Broker Certificates
 COPY --from=certs /etc/ssl/certs/ca-bundle.crt /tmp/certs/kafka/broker/ca.crt

scheduler/Dockerfile.modelgateway

@@ -13,8 +13,9 @@ FROM registry.access.redhat.com/ubi9/ubi-minimal as certs
 # Kafka dependencies necessitate leaving CGo enabled and using a base image with C dependencies
 FROM registry.access.redhat.com/ubi9/ubi-micro:9.3-13
 
-# Kafka OIDC token retriva certs: https://github.com/confluentinc/librdkafka/issues/3751
+# Kafka OIDC token retrieve certs (librdkafka using curl): https://github.com/confluentinc/librdkafka/issues/3751
 COPY --from=certs /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=certs /etc/ssl/certs/ca-bundle.crt /etc/pki/tls/certs/ca-bundle.crt
 
 # Broker Certificates
 COPY --from=certs /etc/ssl/certs/ca-bundle.crt /tmp/certs/kafka/broker/ca.crt

scheduler/Dockerfile.pipelinegateway

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/OneOfOne/xxhash v1.2.8
 	github.com/cenkalti/backoff/v4 v4.2.1
-	github.com/confluentinc/confluent-kafka-go v1.9.2
+	github.com/confluentinc/confluent-kafka-go/v2 v2.3.0
 	github.com/dgraph-io/badger/v3 v3.2103.5
 	github.com/envoyproxy/go-control-plane v0.12.0
 	github.com/fsnotify/fsnotify v1.7.0
@@ -26,7 +26,7 @@ require (
 	github.com/seldonio/seldon-core/apis/go/v2 v2.0.0-00010101000000-000000000000
 	github.com/seldonio/seldon-core/components/tls/v2 v2.0.0-00010101000000-000000000000
 	github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b
-	github.com/signalfx/splunk-otel-go/instrumentation/github.com/confluentinc/confluent-kafka-go/kafka/splunkkafka v1.13.0
+	github.com/signalfx/splunk-otel-go/instrumentation/github.com/confluentinc/confluent-kafka-go/v2/kafka/splunkkafka v1.13.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.4
 	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.48.0
@@ -71,6 +71,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect