Skip to content

Commit 5aaa50c

Browse files
redscholarredscholar
committed
fix: Kube_vip may fail when used with Kubernetes versions above v1.29.
Signed-off-by: joyceliu <[email protected]>
1 parent 71265ad commit 5aaa50c

File tree

11 files changed

+117
-102
lines changed

11 files changed

+117
-102
lines changed

‎api/capkk/infrastructure/v1beta1/kkcluster_types.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ const (
8080
// - ARP Mode: Requires the management cluster and worker cluster nodes to be in the same network segment.
8181
// - BGP Mode: Requires a network environment that supports BGP, with proper configuration in both
8282
// the management and worker clusters.
83-
ControlPlaneEndpointTypeVIP ControlPlaneEndpointType = "vip"
83+
ControlPlaneEndpointTypeVIP ControlPlaneEndpointType = "kube_vip"
8484
)
8585

8686
type InventoryHostConnector struct {

‎builtin/capkk/roles/install/cloud-config/tasks/main.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
---
22
- name: Install kube-vip
33
when:
4-
- eq .kubernetes.control_plane_endpoint.type "vip"
4+
- eq .kubernetes.control_plane_endpoint.type "kube_vip"
55
- or (.kubernetes.roles | has "master") (.kubernetes.roles | has "control-plane")
66
template:
77
src: kube-vip.yaml

‎builtin/core/playbooks/delete_cluster.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
ignore_errors: true
1919
command: |
2020
sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }}
21-
sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
21+
sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
2222
when: .deleteDNS
2323
loop: "{{ .localDNS | toJson }}"
2424

‎builtin/core/playbooks/delete_nodes.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
ignore_errors: true
6262
command: |
6363
sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' {{ .item }}
64-
sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
64+
sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
6565
when:
6666
- .deleteDNS
6767
- .delete_nodes | default list | has .inventory_hostname

‎builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -33,26 +33,37 @@
3333
{{- end }}
3434
dest: /etc/kubernetes/kubeadm-config.yaml
3535

36-
# HAProxy is deployed as a static Pod, which starts only after Kubelet is running.
37-
# Therefore, the control plane must be reachable before HAProxy starts (e.g., by configuring localDNS file).
38-
- name: Write tmp dns to localDNS file
36+
- name: Init kubernetes cluster
37+
block:
38+
- name: pre init
39+
when:
40+
- .kube_version | semverCompare ">=v1.29.0"
41+
- eq .kubernetes.control_plane_endpoint.type "kube_vip"
42+
command: |
43+
sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \
44+
/etc/kubernetes/manifests/kube-vip.yaml
45+
- name: init
46+
command: |
47+
/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull {{ if not .kubernetes.kube_proxy.enabled }}--skip-phases=addon/kube-proxy{{ end }}
48+
- name: post init
49+
when:
50+
- .kube_version | semverCompare ">=v1.29.0"
51+
- eq .kubernetes.control_plane_endpoint.type "kube_vip"
52+
command: |
53+
sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \
54+
/etc/kubernetes/manifests/kube-vip.yaml
55+
56+
# reset localDNS 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}.
57+
# if not the control_plane_endpoint will valid after kube_vip pod running. the task which will execute kubectl apply in current node may be failed.
58+
- name: reset control_plane localDNS
3959
command: |
60+
sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
4061
cat >> {{ .item }} << EOF
41-
# haproxy dns for kubekey begin
42-
{{- if ne .internal_ipv4 "" }}
43-
{{ .internal_ipv4 }} {{ .kubernetes.control_plane_endpoint.host }}
44-
{{- else if ne .internal_ipv6 "" }}
45-
{{ .internal_ipv6 }} {{ .kubernetes.control_plane_endpoint.host }}
46-
{{- end }}
47-
# haproxy dns for kubekey end
48-
EOF
49-
when:
50-
- .kubernetes.control_plane_endpoint.type | eq "haproxy"
62+
# kubekey control_plane BEGIN.
63+
127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
64+
# kubekey control_plane END
65+
EOF
5166
loop: "{{ .localDNS | toJson }}"
52-
53-
- name: Init kubernetes cluster
54-
command: |
55-
/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull {{ if not .kubernetes.kube_proxy.enabled }}--skip-phases=addon/kube-proxy{{ end }}
5667

5768
- name: Copy kubeconfig to default dir
5869
command: |

‎builtin/core/roles/kubernetes/init-kubernetes/tasks/main.yaml

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,16 @@
2222
{{ index $notInitNodes (randInt 0 ((sub ($notInitNodes | len) 1) | int)) }}
2323
{{- end -}}
2424
25-
- name: Set custom hosts to localDNS file
26-
when: eq .kubernetes.control_plane_endpoint.type "local"
25+
- name: Set control_plane hosts to localDNS file
26+
when:
27+
- .kubernetes.control_plane_endpoint.type | eq "local"
28+
- .inventory_hostname | eq .init_kubernetes_node | not
2729
command: |
28-
sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
30+
sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
2931
cat >> {{ .item }} <<EOF
30-
# kubekey custom BEGIN
31-
127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
32-
# kubekey custom END
32+
# kubekey control_plane BEGIN.
33+
{{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }}
34+
# kubekey control_plane END
3335
EOF
3436
loop: "{{ .localDNS | toJson }}"
3537

‎builtin/core/roles/kubernetes/join-kubernetes/tasks/join_kubernetes.yaml

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,4 @@
11
---
2-
- name: Set init_kubernetes_node hosts to localDNS file
3-
when: eq .kubernetes.control_plane_endpoint.type "local"
4-
command: |
5-
sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
6-
cat >> {{ .item }} <<EOF
7-
# kubekey custom BEGIN
8-
{{- if and (index .hostvars .init_kubernetes_node "internal_ipv4") (ne (index .hostvars .init_kubernetes_node "internal_ipv4") "") }}
9-
{{ index .hostvars .init_kubernetes_node "internal_ipv4" }} {{ .kubernetes.control_plane_endpoint.host }}
10-
{{- end }}
11-
{{- if and (index .hostvars .init_kubernetes_node "internal_ipv6") (ne (index .hostvars .init_kubernetes_node "internal_ipv6") "") }}
12-
{{ index .hostvars .init_kubernetes_node "internal_ipv6" }} {{ .kubernetes.control_plane_endpoint.host }}
13-
{{- end }}
14-
# kubekey custom END
15-
EOF
16-
loop: "{{ .localDNS | toJson }}"
17-
182
- name: Generate kubeadm join config
193
template:
204
src: |
@@ -52,15 +36,18 @@
5236
command: |
5337
kubectl annotate {{ .hostname }} {{- range $k,$v := .annotations }}{{ printf "%s=%s" $k $v}} {{- end }}
5438
55-
- name: Set change custom hosts to localDNS file
56-
when:
57-
- eq .kubernetes.control_plane_endpoint.type "local"
39+
# reset localDNS 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}.
40+
# if not the control_plane_endpoint will valid after kube_vip pod running. the task which will execute kubectl apply in current node may be failed.
41+
# the haproxy has be 127.0.0.1
42+
- name: reset control_plane localDNS
43+
when:
5844
- .groups.kube_control_plane | default list | has .inventory_hostname
45+
- .kubernetes.control_plane_endpoint.type | ne "haproxy"
5946
command: |
60-
sed -i ':a;$!{N;ba};s@# kubekey custom BEGIN.*# kubekey custom END@@' {{ .item }}
61-
cat >> {{ .item }} <<EOF
62-
# kubekey custom BEGIN
47+
sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
48+
cat >> {{ .item }} << EOF
49+
# kubekey control_plane BEGIN.
6350
127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
64-
# kubekey custom END
65-
EOF
51+
# kubekey control_plane END
52+
EOF
6653
loop: "{{ .localDNS | toJson }}"

‎builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml

Lines changed: 4 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,4 @@
1-
- name: Join kubernetes
2-
when: ne .inventory_hostname .init_kubernetes_node
3-
block:
4-
# HAProxy is deployed as a static Pod, which starts only after Kubelet is running.
5-
# Therefore, the control plane must be reachable before HAProxy starts (e.g., by configuring localDNS file).
6-
- name: Write tmp dns to localDNS file
7-
command: |
8-
cat >> {{ .item }} << EOF
9-
# haproxy dns for kubekey begin
10-
{{- $internalIPv4 := index .hostvars (.groups.kube_control_plane | default list | first) "internal_ipv4" | default "" }}
11-
{{- $internalIPv6 := index .hostvars (.groups.kube_control_plane | default list | first) "internal_ipv6" | default "" }}
12-
{{- if ne $internalIPv4 "" }}
13-
{{ $internalIPv4 }} {{ .kubernetes.control_plane_endpoint.host }}
14-
{{- else if ne $internalIPv6 "" }}
15-
{{ $internalIPv6 }} {{ .kubernetes.control_plane_endpoint.host }}
16-
{{- end }}
17-
# haproxy dns for kubekey end
18-
EOF
19-
when: .kubernetes.control_plane_endpoint.type | eq "haproxy"
20-
loop: "{{ .localDNS | toJson }}"
21-
- include_tasks: join_kubernetes.yaml
22-
when: .kubernetes_install_service.stdout | eq "inactive"
23-
- name: Replace haproxy dns to localDNS file
24-
when: .kubernetes.control_plane_endpoint.type | eq "haproxy"
25-
block:
26-
- name: Replace control_plane by local hosts
27-
when: .groups.kube_control_plane | default list | has .inventory_hostname
28-
command: |
29-
new_dns="# haproxy dns for kubekey begin
30-
{{- if ne .internal_ipv4 "" }}
31-
{{ .internal_ipv4 }} {{ .kubernetes.control_plane_endpoint.host }}
32-
{{- else if ne .internal_ipv6 "" }}
33-
{{ .internal_ipv6 }} {{ .kubernetes.control_plane_endpoint.host }}
34-
{{- end }}
35-
# haproxy dns for kubekey end"
36-
sed -i '/# haproxy dns for kubekey begin/,/# haproxy dns for kubekey end/c\'"$new_dns" {{ .item }}
37-
loop: "{{ .localDNS | toJson }}"
38-
- name: Replace worker by haproxy hosts
39-
when: .groups.worker | default list | has .inventory_hostname
40-
command: |
41-
new_dns="# haproxy dns for kubekey begin
42-
{{ .kubernetes.control_plane_endpoint.haproxy.address }} {{ .kubernetes.control_plane_endpoint.host }}
43-
# haproxy dns for kubekey end"
44-
sed -i '/# haproxy dns for kubekey begin/,/# haproxy dns for kubekey end/c\'"$new_dns" {{ .item }}
45-
loop: "{{ .localDNS | toJson }}"
1+
- include_tasks: join_kubernetes.yaml
2+
when:
3+
- ne .inventory_hostname .init_kubernetes_node
4+
- .kubernetes_install_service.stdout | eq "inactive"

‎builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/kube_vip.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
- name: Get network interface for kube_vip
33
command: |
44
{{- if .kubernetes.control_plane_endpoint.kube_vip.address | ipFamily | eq "IPv4" }}
5-
ip route | grep ' {{ .internal_ipv4 }} ' | grep 'proto kernel scope link src' | sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\"| uniq
5+
ip route | grep '{{ .internal_ipv4 }}' | grep 'proto kernel scope link src' | awk '{print $3}'
66
{{- else if .kubernetes.control_plane_endpoint.host | ipFamily | eq "IPv6" }}
7-
ip route | grep ' {{ .internal_ipv6 }} ' | grep 'proto kernel scope link src' | sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\"| uniq
7+
ip route | grep '{{ .internal_ipv6 }}' | grep 'proto kernel scope link src' | awk '{print $3}'
88
{{- else }}
99
echo "kubernetes.control_plane_endpoint.kube_vip.address" should be ipv4 or ipv6
1010
exit 1
@@ -14,12 +14,12 @@
1414
- name: Check if network is exist
1515
assert:
1616
that:
17-
- .interface.stderr == ""
18-
- .interface.stdout != ""
17+
- .interface.stderr | eq ""
18+
- .interface.stdout | ne ""
1919
fail_msg: "cannot find network interface to match kube_vip"
2020

2121
- name: Generate kube_vip manifest
2222
template:
2323
src: |
2424
kubevip/kubevip.{{ .kubernetes.control_plane_endpoint.kube_vip.mode }}
25-
dest: /etc/kubernetes/manifests/kubevip.yaml
25+
dest: /etc/kubernetes/manifests/kube-vip.yaml

‎builtin/core/roles/kubernetes/pre-kubernetes/tasks/high-availability/main.yaml

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,59 @@
1+
# set localDNS for each .kubernetes.control_plane_endpoint.type
2+
# local:
3+
# before init cluster
4+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
5+
# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
6+
# after init cluster
7+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
8+
# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
9+
# before join cluster
10+
# - control_plane: {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }}
11+
# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
12+
# after join cluster
13+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
14+
# - worker: {{ .init_kubernetes_node }} {{ .kubernetes.control_plane_endpoint.host }}
15+
#
16+
# kube_vip:
17+
# before init cluster
18+
# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
19+
# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
20+
# after init cluster
21+
# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
22+
# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
23+
# before join cluster
24+
# - control_plane: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
25+
# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
26+
# after join cluster
27+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
28+
# - worker: {{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
29+
#
30+
# haproxy:
31+
# before init cluster
32+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
33+
# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
34+
# after init cluster
35+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
36+
# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
37+
# before join cluster
38+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
39+
# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
40+
# after join cluster
41+
# - control_plane: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
42+
# - worker: 127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
43+
- name: Set Control Plane to localDNS file
44+
command: |
45+
sed -i ':a;$!{N;ba};s@# kubekey control_plane BEGIN.*# kubekey control_plane END@@' {{ .item }}
46+
cat >> {{ .item }} << EOF
47+
# kubekey control_plane BEGIN.
48+
{{- if .kubernetes.control_plane_endpoint.type | eq "kube_vip" }}
49+
{{ .kubernetes.control_plane_endpoint.kube_vip.address }} {{ .kubernetes.control_plane_endpoint.host }}
50+
{{- else }}
51+
127.0.0.1 {{ .kubernetes.control_plane_endpoint.host }}
52+
{{- end }}
53+
# kubekey control_plane END
54+
EOF
55+
loop: "{{ .localDNS | toJson }}"
56+
157
# install with static pod: https://kube-vip.io/docs/installation/static/
258
- include_tasks: high-availability/kube_vip.yaml
359
when:

0 commit comments

Comments
 (0)