Skip to content

feat: operator watch namespaces #6434

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
May 2, 2025
Merged

feat: operator watch namespaces #6434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
14faac5
Modified helm-chart to watch for specific namespaces
RobertSamoilescu Apr 30, 2025
71af523
Included watch-namespaces flag and validation of the flags
RobertSamoilescu Apr 30, 2025
0dd5835
Removed extra indent
RobertSamoilescu Apr 30, 2025
f0c1d04
Fixed autogenerated charts
RobertSamoilescu Apr 30, 2025
655b47c
Added new line at the end of patch_controller.yaml
RobertSamoilescu Apr 30, 2025
6ae633c
Refactored operator code so watchNamespaces if used in conjuction wit…
RobertSamoilescu Apr 30, 2025
522de63
Ensure the current namespace is included in the cache config and impo…
RobertSamoilescu May 1, 2025
0fff362
Implemented the option to skip the creation of ClusterRole
RobertSamoilescu May 1, 2025
da4908c
Replaced Release.Name with Release.Namespace
RobertSamoilescu May 1, 2025
3f0cb66
Fixed rolebinding config
RobertSamoilescu May 1, 2025
fb4bd0c
Remove namespace from the name of rolebind since it is not required
RobertSamoilescu May 1, 2025
a52d68c
Improved flag description
RobertSamoilescu May 2, 2025
3340c2a
Renamed skipClusterRoleCreation to skipOperatorClusterRoleCreation
RobertSamoilescu May 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions k8s/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ create-helm-charts: kustomize
sed 's/labelz:/labels:\n\ \ \ /' ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml > ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
sed -i 's/annotationz:/annotations:\n\ \ \ /' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
sed -i 's/HACK_REMOVE_ME//' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole.*\- get\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding.*ServiceAccount\n name: seldon-v2-controller-manager\n namespace:[^-]\{27\}\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole.*\- get\n\-\-\-\)#{{- if and (not .Values.controller.skipClusterRoleCreation) (or .Values.controller.clusterwide .Values.controller.watchNamespaces) -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding.*ServiceAccount\n name: seldon-v2-controller-manager\n namespace:[^-]\{27\}\n\-\-\-\)#{{- if or .Values.controller.clusterwide .Values.controller.watchNamespaces -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
mv ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml

.PHONY: create-yaml
Expand Down
9 changes: 6 additions & 3 deletions k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ rules:
- list
- watch
---
{{- if .Values.controller.clusterwide -}}
{{- if and (not .Values.controller.skipClusterRoleCreation) (or .Values.controller.clusterwide .Values.controller.watchNamespaces) -}}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the cluster role specific to the operator. Perhaps we should reflect that in the value name? Something like skipOperatorClusterRoleCreation

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
Expand Down Expand Up @@ -351,11 +351,11 @@ subjects:
name: seldon-v2-controller-manager
namespace: '{{ .Release.Namespace }}'
---
{{- if .Values.controller.clusterwide -}}
{{- if or .Values.controller.clusterwide .Values.controller.watchNamespaces -}}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: seldon-v2-manager-rolebinding
name: seldon-v2-manager-rolebinding-{{ .Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
Expand Down Expand Up @@ -408,6 +408,7 @@ spec:
- args:
- --health-probe-bind-address=:8081
- --leader-elect
- --watch-namespaces=$(WATCH_NAMESPACES)
- --namespace=$(POD_NAMESPACE)
- --clusterwide=$(CLUSTERWIDE)
- --log-level=$(LOG_LEVEL)
Expand All @@ -434,6 +435,8 @@ spec:
- name: LOG_LEVEL
value: '{{ hasKey .Values.controller "logLevel" | ternary .Values.controller.logLevel
.Values.logging.logLevel }}'
- name: WATCH_NAMESPACES
value: '{{ join "," .Values.controller.watchNamespaces }}'
- name: POD_NAMESPACE
valueFrom:
fieldRef:
Expand Down
1 change: 1 addition & 0 deletions k8s/helm-charts/seldon-core-v2-setup/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ dataflow:

controller:
clusterwide: false
skipOperatorClusterRoleCreation: false
image:
pullPolicy: IfNotPresent
registry: docker.io
Expand Down
1 change: 1 addition & 0 deletions k8s/helm-charts/seldon-core-v2-setup/values.yaml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ dataflow:

controller:
clusterwide: false
skipOperatorClusterRoleCreation: false
image:
pullPolicy: IfNotPresent
registry: docker.io
Expand Down
2 changes: 2 additions & 0 deletions k8s/kustomize/helm-components-sc/patch_controller.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,5 @@ spec:
value: '{{ .Values.security.controlplane.ssl.client.serverCaPath }}'
- name: LOG_LEVEL
value: '{{ hasKey .Values.controller "logLevel" | ternary .Values.controller.logLevel .Values.logging.logLevel }}'
- name: WATCH_NAMESPACES
value: '{{ join "," .Values.controller.watchNamespaces }}'
3 changes: 3 additions & 0 deletions k8s/yaml/components.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ spec:
- args:
- --health-probe-bind-address=:8081
- --leader-elect
- --watch-namespaces=$(WATCH_NAMESPACES)
- --namespace=$(POD_NAMESPACE)
- --clusterwide=$(CLUSTERWIDE)
- --log-level=$(LOG_LEVEL)
Expand All @@ -289,6 +290,8 @@ spec:
value: '/tmp/certs/cps/ca.crt'
- name: LOG_LEVEL
value: 'info'
- name: WATCH_NAMESPACES
value: ''
- name: POD_NAMESPACE
valueFrom:
fieldRef:
Expand Down
1 change: 1 addition & 0 deletions operator/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
sed -i 's/ClusterRole/Role/' config/rbac/namespace_role.yaml
cp config/rbac/role_binding.yaml config/rbac/namespace_role_binding.yaml
sed -i 's/ClusterRole/Role/' config/rbac/namespace_role_binding.yaml
sed -i 's/-{{[[:space:]]*\.Release\.Namespace[[:space:]]*}}//' config/rbac/namespace_role_binding.yaml

.PHONY: generate
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
Expand Down
3 changes: 3 additions & 0 deletions operator/config/manager/manager.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ spec:
args:
- --health-probe-bind-address=:8081
- --leader-elect
- --watch-namespaces=$(WATCH_NAMESPACES)
- --namespace=$(POD_NAMESPACE)
- --clusterwide=$(CLUSTERWIDE)
- --log-level=$(LOG_LEVEL)
Expand All @@ -39,6 +40,8 @@ spec:
fieldPath: metadata.namespace
- name: LOG_LEVEL
value: "warn"
- name: WATCH_NAMESPACES
value: ""
securityContext:
allowPrivilegeEscalation: false
livenessProbe:
Expand Down
2 changes: 1 addition & 1 deletion operator/config/rbac/role_binding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: v2-manager-rolebinding
name: v2-manager-rolebinding-{{ .Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
Expand Down
45 changes: 38 additions & 7 deletions operator/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,56 @@ func getLogLevel(l string) zap2.AtomicLevel {
return level
}

func getWatchNamespaceConfig(namespace, watchNamespaces string, clusterwide bool) map[string]cache.Config {
configs := make(map[string]cache.Config)

if !clusterwide {
setupLog.Info("Watching namespace", "namespace", namespace)
configs[namespace] = cache.Config{}
return configs
}

if watchNamespaces == "" {
setupLog.Info("Clusterwide mode enabled, watching all namespaces")
return configs
}

setupLog.Info("Clusterwide mode enabled, watching namespaces", "namespaces", watchNamespaces)
nsSet := make(map[string]struct{})

for _, ns := range strings.Split(watchNamespaces, ",") {
ns = strings.TrimSpace(ns)
if ns != "" {
nsSet[ns] = struct{}{}
}
}

nsSet[namespace] = struct{}{}
for ns := range nsSet {
configs[ns] = cache.Config{}
}

return configs
}

func main() {
var (
metricsAddr string
enableLeaderElection bool
probeAddr string
namespace string
watchNamespaces string
clusterwide bool
logLevel string
)

flag.StringVar(&metricsAddr, "metrics-bind-address", ":4000", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":4001", "The address the probe endpoint binds to.")
flag.StringVar(&namespace, "namespace", "", "The namespace to restrict the operator.")
flag.StringVar(&watchNamespaces, "watch-namespaces", "",
"Comma separated list of namespaces to watch. "+
"Only used when --clusterwide is set to true. "+
"Defaults to all namespaces if not set and --clusterwide is true.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
Expand All @@ -85,18 +122,12 @@ func main() {
flag.Parse()

opts.Level = getLogLevel(logLevel)

logger := zap.New(zap.UseFlagOptions(&opts))
ctrl.SetLogger(logger)

watchNamespaceConfig := map[string]cache.Config{namespace: {}}
if clusterwide {
watchNamespaceConfig = map[string]cache.Config{} // unset namespace so manager watches all namespaces
}
setupLog.Info("Starting manager", "clusterwide", clusterwide)

setupLog.Info("Setting log level", "level", logLevel)

watchNamespaceConfig := getWatchNamespaceConfig(namespace, watchNamespaces, clusterwide)
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{BindAddress: metricsAddr},
Expand Down
Loading