Operator fails during deploying ClickHouseKeeperInstallation custom resource #1863
Description
During deploying ClickHouseKeeperInstallation custom resource the Operator fails with the next errors in the trace:
025-11-19T17:54:07Z INFO KubeAPIWarningLogger would violate PodSecurity "baseline:v1.25": hostPath volumes (volume "keeper-storage")
I1119 17:54:07.434770 1 panic.go:783] worker-reconciler-chk.go:342:reconcileHostStatefulSet():end:Host:0-0[0/0]:clickhouse/clickhouse:reconcile StatefulSet end
I1119 17:54:07.435401 1 worker-service.go:39] reconcileService():unknown:Service found: clickhouse/keeper-clickhouse. Will try to update
I1119 17:54:07.447355 1 worker-service.go:173] updateService():unknown:Update Service success: clickhouse/keeper-clickhouse
I1119 17:54:07.447402 1 worker-service.go:60] reconcileService():unknown:Service reconcile successful: clickhouse/keeper-clickhouse
I1119 17:54:07.447435 1 panic.go:783] worker-reconciler-chk.go:465:reconcileShardsAndHosts():end:reconcileShardsAndHosts end
I1119 17:54:07.447456 1 panic.go:783] worker-reconciler-chk.go:50:reconcileCR():end:unknown
2025-11-19T17:54:07Z INFO Observed a panic in reconciler: runtime error: invalid memory address or nil pointer dereference {"controller": "clickhousekeeperinstallation", "controllerGroup": "clickhouse-keeper.altinity.com", "controllerKind": "ClickHouseKeeperInstallation", "ClickHouseKeeperInstallation": {"name":"clickhouse","namespace":"clickhouse"}, "namespace": "clickhouse", "name": "clickhouse", "reconcileID": "184f87fe-2e4b-4227-a9b5-6cdea85b3fae"}
panic: runtime error: invalid memory address or nil pointer dereference [recovered, repanicked]
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x181599e]
goroutine 359 [running]:
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile.func1()
/clickhouse-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:115 +0x1d4
panic({0x1c3d7c0?, 0x3236390?})
/usr/local/go/src/runtime/panic.go:783 +0x132
github.com/altinity/clickhouse-operator/pkg/controller/common/poller/domain.(*HostObjectsPoller).WaitHostStatefulSetReady.func1({0x2239c70?, 0x32906a0?}, 0xc000e4ea08)
/clickhouse-operator/pkg/controller/common/poller/domain/poller-host-objects.go:98 +0x3e
github.com/altinity/clickhouse-operator/pkg/controller/common/poller/domain.(*HostObjectPoller[...]).Poll.func2({0x1effb40?, 0xc000e4ea08?})
/clickhouse-operator/pkg/controller/common/poller/domain/poller-host-object.go:62 +0x3b
github.com/altinity/clickhouse-operator/pkg/controller/common/poller.(*Functions).CallIsDone(...)
/clickhouse-operator/pkg/controller/common/poller/poller-functions.go:46
github.com/altinity/clickhouse-operator/pkg/controller/common/poller.(*poller).Poll(0xc000bc0c60)
/clickhouse-operator/pkg/controller/common/poller/poller.go:78 +0x423
github.com/altinity/clickhouse-operator/pkg/controller/common/poller/domain.(*HostObjectPoller[...]).Poll(0x2220ce0, {0x2239c70, 0x32906a0}, 0xc0007449c0, 0xc000c26468, {0x0, 0x0, 0x0})
/clickhouse-operator/pkg/controller/common/poller/domain/poller-host-object.go:72 +0x405
github.com/altinity/clickhouse-operator/pkg/controller/common/poller/domain.(*HostObjectsPoller).WaitHostStatefulSetReady(0xc000431b20, {0x2239c70, 0x32906a0}, 0xc0007449c0)
/clickhouse-operator/pkg/controller/common/poller/domain/poller-host-objects.go:91 +0x1b2
github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset.(*Reconciler).doDeleteStatefulSet(0xc00002b560, {0x2239c70, 0x32906a0}, 0xc0007449c0)
/clickhouse-operator/pkg/controller/common/statefulset/statefulset-reconciler.go:484 +0x8c9
github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset.(*Reconciler).recreateStatefulSet(0xc00002b560, {0x2239c70, 0x32906a0}, 0xc0007449c0, 0x0, 0x0)
/clickhouse-operator/pkg/controller/common/statefulset/statefulset-reconciler.go:197 +0x73f
github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset.(*Reconciler).updateStatefulSet(0xc00002b560, {0x2239c70, 0x32906a0}, 0xc0007449c0, 0x0, 0x0)
/clickhouse-operator/pkg/controller/common/statefulset/statefulset-reconciler.go:288 +0x1806
github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset.(*Reconciler).ReconcileStatefulSet(0xc00002b560, {0x2239c70, 0x32906a0}, 0xc0007449c0, 0x0, 0x0)
/clickhouse-operator/pkg/controller/common/statefulset/statefulset-reconciler.go:173 +0xed7
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcileHostStatefulSet(0xc000ab01e0, {0x2239c70, 0x32906a0}, 0xc0007449c0, 0x0)
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:353 +0xb5b
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcileHostMain(0xc000ab01e0, {0x2239c70, 0x32906a0}, 0xc0007449c0)
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:672 +0xcac
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcileHost(0xc000ab01e0, {0x2239c70, 0x32906a0}, 0xc0007449c0)
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:578 +0x833
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcileShardWithHosts.func1(0x41d3c5?)
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:534 +0x25
github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1.(*ChkShard).WalkHostsAbortOnError(0xc000068900, 0xc00060f900)
/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1/type_shard.go:124 +0x72
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcileShardWithHosts(0xc000ab01e0, {0x2239c70, 0x32906a0}, {0x224dd58, 0xc000068900})
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:533 +0xa2
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcileShardsAndHosts(0xc000ab01e0, {0x2239c70, 0x32906a0}, {0xc0005b0638, 0x1, 0x1})
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:480 +0x5fa
github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1.(*ClickHouseKeeperInstallation).WalkTillError(0xc0008e5680, {0x2239c70, 0x32906a0}, 0x0?, 0xc000bc33d0, 0xc000bc33c0, 0xc000bc33b0)
/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1/type_chk.go:671 +0x15f
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcile(0xc000ab01e0, {0x2239c70, 0x32906a0}, 0xc0008e5680)
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:179 +0xbe5
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*worker).reconcileCR(0xc000ab01e0, {0x2239c70, 0x32906a0}, 0x0?, 0xc0008e4000)
/clickhouse-operator/pkg/controller/chk/worker-reconciler-chk.go:68 +0x845
github.com/altinity/clickhouse-operator/pkg/controller/chk.(*Controller).Reconcile(0xc000250c40, {0x2239b90, 0xc000b10cc0}, {{{0xc000cc03d4?, 0x1de4fc0?}, {0xc0000117c0?, 0xc00005c5d0?}}})
/clickhouse-operator/pkg/controller/chk/controller.go:80 +0x3c5
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile(0x2239b90?, {0x2239b90?, 0xc000b10cc0?}, {{{0xc000cc03d4?, 0x1b18e60?}, {0xc0000117c0?, 0x0?}}})
/clickhouse-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:118 +0xa5
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc0005ca500, {0x2239bc8, 0xc0001d08c0}, {0x1d4f020, 0xc000431cc0})
/clickhouse-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:314 +0x31c
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc0005ca500, {0x2239bc8, 0xc0001d08c0})
/clickhouse-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:265 +0x197
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
/clickhouse-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:226 +0x73
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2 in goroutine 263
/clickhouse-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222 +0x509
I1119 17:54:10.638609 1 exporter.go:81] Collect started
Here is the example of ClickHouseKeeperInstallation manifest which we try to use
apiVersion: "clickhouse-keeper.altinity.com/v1"
kind: "ClickHouseKeeperInstallation"
metadata:
name: "clickhouse"
annotations:
prometheus.io/port: "7000"
prometheus.io/scrape: "true"
spec:
configuration:
clusters:
- name: "0"
layout:
replicasCount: 3
settings:
logger/level: "information"
logger/console: "true"
listen_host: "0.0.0.0"
keeper_server/four_letter_word_white_list: "*"
keeper_server/coordination_settings/raft_logs_level: "information"
keeper_server/path: "/var/lib/clickhouse-keeper"
prometheus/endpoint: "/metrics"
prometheus/port: "7000"
prometheus/metrics: "true"
prometheus/events: "true"
prometheus/asynchronous_metrics: "true"
prometheus/status_info: "false"
defaults:
templates:
podTemplate: keeper-default
templates:
podTemplates:
- name: keeper-default
metadata:
labels:
app: clickhouse-keeper
annotations:
prometheus.io/port: "7000"
prometheus.io/scrape: "true"
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- clickhouse-keeper
topologyKey: "kubernetes.io/hostname"
nodeSelector:
node-role.kubernetes.io/controlplane: "true"
tolerations:
- key: "node-role.kubernetes.io/controlplane"
operator: "Exists"
effect: "NoSchedule"
- key: "node-role.kubernetes.io/etcd"
operator: "Exists"
effect: "NoExecute"
containers:
- name: clickhouse-keeper
imagePullPolicy: IfNotPresent
image: "docker.io/clickhouse/clickhouse-keeper:23.8.16"
resources:
requests:
memory: "256Mi"
cpu: "1"
limits:
memory: "4Gi"
cpu: "2"
volumeMounts:
- name: keeper-storage
mountPath: /var/lib/clickhouse-keeper
securityContext:
fsGroup: 101
volumes:
- name: keeper-storage
hostPath:
path: /mnt/data/clickhouse-keeper
type: DirectoryOrCreate