Skip to content

Commit 551844d

Browse files
pixiakepixiake
committed
Refine IPv6 Supported
Signed-off-by: pixiake <[email protected]>
1 parent 5f8c6cf commit 551844d

File tree

10 files changed

+102
-35
lines changed

10 files changed

+102
-35
lines changed

‎cmd/kk/apis/kubekey/v1alpha2/default.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ const (
6363
DefaultMaxPods = 110
6464
DefaultPodPidsLimit = 10000
6565
DefaultNodeCidrMaskSize = 24
66+
DefaultNodeCidrMaskSizeIPv6 = 120
6667
DefaultIPIPMode = "Always"
6768
DefaultVXLANMode = "Never"
6869
DefaultVethMTU = 0
@@ -139,6 +140,9 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, map[string][]*Kub
139140
if cfg.Kubernetes.NodeCidrMaskSize == 0 {
140141
clusterCfg.Kubernetes.NodeCidrMaskSize = DefaultNodeCidrMaskSize
141142
}
143+
if cfg.Kubernetes.NodeCidrMaskSizeIPv6 == 0 {
144+
clusterCfg.Kubernetes.NodeCidrMaskSizeIPv6 = DefaultNodeCidrMaskSizeIPv6
145+
}
142146
if cfg.Kubernetes.ProxyMode == "" {
143147
clusterCfg.Kubernetes.ProxyMode = DefaultProxyMode
144148
}

‎cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ type Kubernetes struct {
3232
MaxPods int `yaml:"maxPods" json:"maxPods,omitempty"`
3333
PodPidsLimit int `yaml:"podPidsLimit" json:"podPidsLimit,omitempty"`
3434
NodeCidrMaskSize int `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"`
35+
NodeCidrMaskSizeIPv6 int `yaml:"nodeCidrMaskSizeIPv6" json:"nodeCidrMaskSizeIPv6,omitempty"`
3536
ApiserverCertExtraSans []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"`
3637
ProxyMode string `yaml:"proxyMode" json:"proxyMode,omitempty"`
3738
AutoRenewCerts *bool `yaml:"autoRenewCerts" json:"autoRenewCerts,omitempty"`

‎cmd/kk/apis/kubekey/v1alpha2/network_types.go

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,25 @@ type NetworkConfig struct {
2828
}
2929

3030
type CalicoCfg struct {
31-
IPIPMode string `yaml:"ipipMode" json:"ipipMode,omitempty"`
32-
VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"`
33-
VethMTU int `yaml:"vethMTU" json:"vethMTU,omitempty"`
34-
Ipv4NatOutgoing *bool `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"`
35-
DefaultIPPOOL *bool `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"`
36-
EnableTypha *bool `yaml:"enableTypha" json:"enableTypha,omitempty"`
37-
Replicas int `yaml:"replicas" json:"replicas,omitempty"`
38-
NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
31+
IPIPMode string `yaml:"ipipMode" json:"ipipMode,omitempty"`
32+
VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"`
33+
VethMTU int `yaml:"vethMTU" json:"vethMTU,omitempty"`
34+
Ipv4NatOutgoing *bool `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"`
35+
Ipv6NatOutgoing *bool `yaml:"ipv6NatOutgoing" json:"ipv6NatOutgoing,omitempty"`
36+
DefaultIPPOOL *bool `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"`
37+
Typha Typha `yaml:"typha" json:"typha,omitempty"`
38+
Controller Controller `yaml:"controller" json:"controller,omitempty"`
39+
}
40+
41+
type Typha struct {
42+
Replicas int `yaml:"replicas" json:"replicas,omitempty"`
43+
NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
44+
Enabled *bool `yaml:"enabled" json:"enabled,omitempty"`
45+
}
46+
47+
type Controller struct {
48+
Replicas int `yaml:"replicas" json:"replicas,omitempty"`
49+
NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
3950
}
4051

4152
type FlannelCfg struct {
@@ -178,6 +189,14 @@ func (c *CalicoCfg) EnableIPV4POOL_NAT_OUTGOING() bool {
178189
return *c.Ipv4NatOutgoing
179190
}
180191

192+
// EnableIPV6POOL_NAT_OUTGOING is used to determine whether to enable CALICO_IPV6POOL_NAT_OUTGOING.
193+
func (c *CalicoCfg) EnableIPV6POOL_NAT_OUTGOING() bool {
194+
if c.Ipv6NatOutgoing == nil {
195+
return false
196+
}
197+
return *c.Ipv6NatOutgoing
198+
}
199+
181200
// EnableDefaultIPPOOL is used to determine whether to create default ippool
182201
func (c *CalicoCfg) EnableDefaultIPPOOL() bool {
183202
if c.DefaultIPPOOL == nil {
@@ -186,12 +205,12 @@ func (c *CalicoCfg) EnableDefaultIPPOOL() bool {
186205
return *c.DefaultIPPOOL
187206
}
188207

189-
// Typha is used to determine whether to enable calico Typha
190-
func (c *CalicoCfg) Typha() bool {
191-
if c.EnableTypha == nil {
208+
// EnableTypha is used to determine whether to enable calico Typha
209+
func (c *CalicoCfg) EnableTypha() bool {
210+
if c.Typha.Enabled == nil {
192211
return false
193212
}
194-
return *c.EnableTypha
213+
return *c.Typha.Enabled
195214
}
196215

197216
// EnableInit is used to determine whether to create default network

‎cmd/kk/pkg/bootstrap/os/module.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ func (c *ConfigureOSModule) Init() {
6464
Template: templates.InitOsScriptTmpl,
6565
Dst: filepath.Join(common.KubeScriptDir, "initOS.sh"),
6666
Data: util.Data{
67-
"Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf),
67+
"Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf),
68+
"IPv6Support": templates.EnabledIPv6(c.KubeConf),
6869
},
6970
},
7071
Parallel: true,

‎cmd/kk/pkg/bootstrap/os/templates/init_script.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,16 @@ echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf
9999
echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf
100100
echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf
101101
102+
{{- if .IPv6Support }}
102103
#add for ipv6
103104
echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf
104105
echo 'net.ipv6.conf.default.disable_ipv6 = 0' >> /etc/sysctl.conf
105106
echo 'net.ipv6.conf.lo.disable_ipv6 = 0' >> /etc/sysctl.conf
106107
echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf
108+
echo 'net.ipv6.conf.default.accept_dad=0' >> /etc/sysctl.conf
109+
echo 'net.ipv6.route.max_size=65536' >> /etc/sysctl.conf
110+
echo 'net.ipv6.neigh.default.retrans_time_ms=1000' >> /etc/sysctl.conf
111+
{{- end}}
107112
108113
#See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y
109114
sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf
@@ -145,6 +150,18 @@ sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.d
145150
sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf
146151
sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf
147152
153+
{{- if .IPv6Support }}
154+
#add for ipv6
155+
sed -r -i "s@#{0,}?net.ipv6.conf.all.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.all.disable_ipv6 = 0@g" /etc/sysctl.conf
156+
sed -r -i "s@#{0,}?net.ipv6.conf.default.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.default.disable_ipv6 = 0@g" /etc/sysctl.conf
157+
sed -r -i "s@#{0,}?net.ipv6.conf.lo.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.lo.disable_ipv6 = 0@g" /etc/sysctl.conf
158+
sed -r -i "s@#{0,}?net.ipv6.conf.all.forwarding ?= ?([0-9]{1,})@net.ipv6.conf.all.forwarding = 1@g" /etc/sysctl.conf
159+
sed -r -i "s@#{0,}?net.ipv6.conf.default.accept_dad ?= ?([0-9]{1,})@net.ipv6.conf.default.accept_dad = 0@g" /etc/sysctl.conf
160+
sed -r -i "s@#{0,}?net.ipv6.route.max_size ?= ?([0-9]{1,})@net.ipv6.route.max_size = 65536@g" /etc/sysctl.conf
161+
sed -r -i "s@#{0,}?net.ipv6.neigh.default.retrans_time_ms ?= ?([0-9]{1,})@net.ipv6.neigh.default.retrans_time_ms = 1000@g" /etc/sysctl.conf
162+
{{- end}}
163+
164+
148165
tmpfile="$$.tmp"
149166
awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf
150167
mv $tmpfile /etc/sysctl.conf
@@ -287,3 +304,10 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [
287304
hostsList = append(hostsList, lbHost)
288305
return hostsList
289306
}
307+
308+
func EnabledIPv6(kubeConf *common.KubeConf) bool {
309+
if len(strings.Split(kubeConf.Cluster.Network.KubePodsCIDR, ",")) == 2 {
310+
return true
311+
}
312+
return false
313+
}

‎cmd/kk/pkg/kubernetes/tasks.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,7 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
307307
"BootstrapToken": bootstrapToken,
308308
"CertificateKey": certificateKey,
309309
"IPv6Support": host.GetInternalIPv6Address() != "",
310+
"NodeCidrMaskSizeIPv6": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6,
310311
},
311312
}
312313

‎cmd/kk/pkg/kubernetes/templates/kubeadm_config.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ controllerManager:
9898
extraArgs:
9999
{{- if .IPv6Support }}
100100
node-cidr-mask-size-ipv4: "{{ .NodeCidrMaskSize }}"
101-
node-cidr-mask-size-ipv6: "64"
101+
node-cidr-mask-size-ipv6: "{{ .NodeCidrMaskSizeIPv6 }}"
102102
{{- else }}
103103
node-cidr-mask-size: "{{ .NodeCidrMaskSize }}"
104104
{{- end }}

‎cmd/kk/pkg/plugins/network/tasks.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -460,17 +460,21 @@ func (g *GenerateCalicoManifests) Execute(runtime connector.Runtime) error {
460460
"CalicoFlexvolImage": images.GetImage(runtime, g.KubeConf, "calico-flexvol").ImageName(),
461461
"CalicoControllersImage": images.GetImage(runtime, g.KubeConf, "calico-kube-controllers").ImageName(),
462462
"CalicoTyphaImage": images.GetImage(runtime, g.KubeConf, "calico-typha").ImageName(),
463-
"TyphaEnabled": len(runtime.GetHostsByRole(common.K8s)) > 50 || g.KubeConf.Cluster.Network.Calico.Typha(),
463+
"TyphaEnabled": len(runtime.GetHostsByRole(common.K8s)) > 50 || g.KubeConf.Cluster.Network.Calico.EnableTypha(),
464464
"VethMTU": g.KubeConf.Cluster.Network.Calico.VethMTU,
465465
"NodeCidrMaskSize": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize,
466466
"IPIPMode": g.KubeConf.Cluster.Network.Calico.IPIPMode,
467467
"VXLANMode": g.KubeConf.Cluster.Network.Calico.VXLANMode,
468468
"ConatinerManagerIsIsula": g.KubeConf.Cluster.Kubernetes.ContainerManager == "isula",
469469
"IPV4POOLNATOUTGOING": g.KubeConf.Cluster.Network.Calico.EnableIPV4POOL_NAT_OUTGOING(),
470+
"IPV6POOLNATOUTGOING": g.KubeConf.Cluster.Network.Calico.EnableIPV6POOL_NAT_OUTGOING(),
470471
"DefaultIPPOOL": g.KubeConf.Cluster.Network.Calico.EnableDefaultIPPOOL(),
471472
"IPv6Support": IPv6Support,
472-
"Replicas": g.KubeConf.Cluster.Network.Calico.Replicas,
473-
"NodeSelector": g.KubeConf.Cluster.Network.Calico.NodeSelector,
473+
"NodeCidrMaskSizeIPv6": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6,
474+
"TyphaReplicas": g.KubeConf.Cluster.Network.Calico.Typha.Replicas,
475+
"TyphaNodeSelector": g.KubeConf.Cluster.Network.Calico.Typha.NodeSelector,
476+
"ControllerReplicas": g.KubeConf.Cluster.Network.Calico.Controller.Replicas,
477+
"ControllerNodeSelector": g.KubeConf.Cluster.Network.Calico.Controller.NodeSelector,
474478
},
475479
}
476480
templateAction.Init(nil, nil)

‎cmd/kk/pkg/plugins/network/templates/calico.tmpl

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,17 @@ data:
8888
"datastore_type": "kubernetes",
8989
"nodename": "__KUBERNETES_NODE_NAME__",
9090
"mtu": __CNI_MTU__,
91+
{{- if .IPv6Support }}
92+
"ipam": {
93+
"type": "calico-ipam",
94+
"assign_ipv4": "true",
95+
"assign_ipv6": "true"
96+
},
97+
{{- else }}
9198
"ipam": {
9299
"type": "calico-ipam"
93100
},
101+
{{- end }}
94102
"policy": {
95103
"type": "k8s"
96104
},
@@ -4981,17 +4989,16 @@ spec:
49814989
value: "false"
49824990
{{- end }}
49834991
{{- if .IPv6Support }}
4992+
# Enable IPIP
4993+
- name: CALICO_IPV6POOL_IPIP
4994+
value: "{{ .IPIPMode }}"
49844995
# Enable or Disable VXLAN on the default IPv6 IP pool.
49854996
- name: CALICO_IPV6POOL_VXLAN
4986-
value: "Always"
4997+
value: "{{ .VXLANMode }}"
4998+
{{- if .IPV6POOLNATOUTGOING }}
49874999
- name: CALICO_IPV6POOL_NAT_OUTGOING
49885000
value: "true"
4989-
{{- else }}
4990-
# Enable or Disable VXLAN on the default IPv6 IP pool.
4991-
- name: CALICO_IPV6POOL_VXLAN
4992-
value: "Never"
4993-
- name: CALICO_IPV6POOL_NAT_OUTGOING
4994-
value: "false"
5001+
{{- end }}
49955002
{{- end }}
49965003
# Set MTU for tunnel device used if ipip is enabled
49975004
- name: FELIX_IPINIPMTU
@@ -5023,7 +5030,7 @@ spec:
50235030
- name: CALICO_IPV6POOL_CIDR
50245031
value: "{{ .KubePodsV6CIDR }}"
50255032
- name: CALICO_IPV6POOL_BLOCK_SIZE
5026-
value: "120"
5033+
value: "{{ .NodeCidrMaskSizeIPv6 }}"
50275034
{{- end }}
50285035
{{- else }}
50295036
- name: NO_DEFAULT_POOLS
@@ -5072,6 +5079,9 @@ spec:
50725079
- /bin/calico-node
50735080
- -felix-live
50745081
- -bird-live
5082+
{{- if .IPv6Support }}
5083+
- -bird6-live
5084+
{{- end }}
50755085
periodSeconds: 10
50765086
initialDelaySeconds: 10
50775087
failureThreshold: 6
@@ -5082,6 +5092,9 @@ spec:
50825092
- /bin/calico-node
50835093
- -felix-ready
50845094
- -bird-ready
5095+
{{- if .IPv6Support }}
5096+
- -bird6-ready
5097+
{{- end }}
50855098
periodSeconds: 10
50865099
timeoutSeconds: 10
50875100
volumeMounts:
@@ -5171,7 +5184,7 @@ metadata:
51715184
k8s-app: calico-kube-controllers
51725185
spec:
51735186
# The controllers can only have a single active instance.
5174-
replicas: {{ if .Replicas }}{{ .Replicas }}{{ else }}1{{ end }}
5187+
replicas: {{ if .ControllerReplicas }}{{ .ControllerReplicas }}{{ else }}1{{ end }}
51755188
selector:
51765189
matchLabels:
51775190
k8s-app: calico-kube-controllers
@@ -5186,8 +5199,8 @@ spec:
51865199
spec:
51875200
nodeSelector:
51885201
kubernetes.io/os: linux
5189-
{{ if .NodeSelector }}
5190-
{{- toYaml .NodeSelector | indent 8 }}
5202+
{{ if .ControllerNodeSelector }}
5203+
{{- toYaml .ControllerNodeSelector | indent 8 }}
51915204
{{- end }}
51925205
tolerations:
51935206
# Mark the pod as a critical add-on for rescheduling.
@@ -5197,7 +5210,7 @@ spec:
51975210
effect: NoSchedule
51985211
- key: node-role.kubernetes.io/control-plane
51995212
effect: NoSchedule
5200-
{{ if .NodeSelector }}
5213+
{{ if .ControllerNodeSelector }}
52015214
affinity:
52025215
podAntiAffinity:
52035216
preferredDuringSchedulingIgnoredDuringExecution:
@@ -5258,7 +5271,7 @@ spec:
52585271
# We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential
52595272
# (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In
52605273
# production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.
5261-
replicas: {{ if .Replicas }}{{ .Replicas }}{{ else }}1{{ end }}
5274+
replicas: {{ if .TyphaReplicas }}{{ .TyphaReplicas }}{{ else }}1{{ end }}
52625275
revisionHistoryLimit: 2
52635276
selector:
52645277
matchLabels:
@@ -5285,14 +5298,14 @@ spec:
52855298
spec:
52865299
nodeSelector:
52875300
kubernetes.io/os: linux
5288-
{{ if .NodeSelector }}
5289-
{{- toYaml .NodeSelector | indent 8 }}
5301+
{{ if .TyphaNodeSelector }}
5302+
{{- toYaml .TyphaNodeSelector | indent 8 }}
52905303
{{- end }}
52915304
hostNetwork: true
52925305
# Typha supports graceful shut down, disconnecting clients slowly during the grace period.
52935306
# The TYPHA_SHUTDOWNTIMEOUTSECS env var should be kept in sync with this value.
52945307
terminationGracePeriodSeconds: 300
5295-
{{ if .NodeSelector }}
5308+
{{ if .TyphaNodeSelector }}
52965309
affinity:
52975310
podAntiAffinity:
52985311
preferredDuringSchedulingIgnoredDuringExecution:

‎docs/config-example.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,8 +148,8 @@ spec:
148148
ipipMode: Always # IPIP Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, vxlanMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Always]
149149
vxlanMode: Never # VXLAN Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, ipipMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Never]
150150
vethMTU: 0 # The maximum transmission unit (MTU) setting determines the largest packet size that can be transmitted through your network. By default, MTU is auto-detected. [Default: 0]
151-
kubePodsCIDR: 10.233.64.0/18,fc00::/48
152-
kubeServiceCIDR: 10.233.0.0/18,fd00::/108
151+
kubePodsCIDR: 10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112
152+
kubeServiceCIDR: 10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116
153153
storage:
154154
openebs:
155155
basePath: /var/openebs/local # base path of the local PV provisioner

0 commit comments

Comments
 (0)