From a872ccca6c4f8e6cd4029dfed57a09deaafc6c09 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Mon, 15 Jun 2020 10:38:34 -0600 Subject: [PATCH] adding support for HA clusters, using HAProxy (#317) * adding support for HA clusters, using HAProxy Signed-off-by: David Lyle * fixing typo * fixing load balancer port default value --- clr-k8s-examples/README.md | 37 ++++++++++++++++++++++++++-- clr-k8s-examples/create_stack.sh | 10 +++++++- clr-k8s-examples/haproxy.cfg.example | 34 +++++++++++++++++++++++++ clr-k8s-examples/setup_system.sh | 2 +- 4 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 clr-k8s-examples/haproxy.cfg.example diff --git a/clr-k8s-examples/README.md b/clr-k8s-examples/README.md index 8753af5..095c544 100644 --- a/clr-k8s-examples/README.md +++ b/clr-k8s-examples/README.md @@ -63,6 +63,28 @@ devicemapper storage. This should not be used for production. > NOTE: This step is done automatically if using vagrant. +### For HA, setup the load balancer node + +Ideally, the load balancer node will be a separate node. However, one of the +master nodes can also serve as the load balancer for the cluster. [HAProxy](https://www.haproxy.org/) +is used in these instructions. + +```bash +sudo swupd bundle-add haproxy +sudo systemctl enable haproxy +``` + +Edit the master IP addresses and load balancer address and ports in [`haproxy.cfg.example`](haproxy.cfg.example) +to match the IPs for the new cluster. If using a master node for the load balancer +make sure that the `frontend bind` port is different than the Kubernetes API port, 6443. +If using a separate machine for load balancing, the port can be 6443 if desired. + +```bash +sudo mkdir -p /etc/haproxy +sudo cp haproxy.cfg.example /etc/haproxy/haproxy.cfg +sudo systemctl start haproxy +``` + ## Bring up the master Run [`create_stack.sh`](create_stack.sh) on the master node. This sets up the @@ -75,8 +97,9 @@ more information. > NOTE: Before running [`create_stack.sh`](create_stack.sh) script, make sure to export the necessary environment variables if needed to be changed. By default it will use -`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the -Vagrant. +`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the +Vagrant. If creating an HA cluster, make sure to specify `LOAD_BALANCER_IP` and +`LOAD_BALANCER_PORT`. ```bash # default shows help @@ -86,6 +109,13 @@ Vagrant. In order to enable running greater than 110 pods per node, set the environment variable `HIGH_POD_COUNT` to any non-empty value. +If creating an HA cluster, join the other master nodes to the cluster. + +```bash +kubeadm join : --token --discovery-token-ca-cert-hash \ + --control-plane --certificate-key --cri-socket=/run/crio/crio.sock +``` + ## Join Workers to the cluster ```bash @@ -94,6 +124,9 @@ kubeadm join : --token --discovery-token-ca-cert Note: Remember to append `--cri-socket=/run/crio/crio.sock` to the join command generated by the master. +If creating an HA cluster, join the other worker nodes to the cluster. The same way, +but replacing the `:` with `:`. + On workers just use the join command that the master spits out. There nothing else you need to run on the worker. All the other Kubernetes customizations are pushed in from master via the values setup in the `kubeadm.yaml` file. diff --git a/clr-k8s-examples/create_stack.sh b/clr-k8s-examples/create_stack.sh index 94a0662..095c84c 100755 --- a/clr-k8s-examples/create_stack.sh +++ b/clr-k8s-examples/create_stack.sh @@ -14,6 +14,8 @@ SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" : ${MASTER_IP:=} : ${CERT_SANS:=} HIGH_POD_COUNT=${HIGH_POD_COUNT:-""} +LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""} +LOAD_BALANCER_PORT="${LOAD_BALANCER_PORT:-6444}" # versions CANAL_VER="${CLRK8S_CANAL_VER:-v3.10}" @@ -90,7 +92,13 @@ function cluster_init() { echo "/var/lib/etcd exists! skipping init." return fi - sudo -E kubeadm init --config=./kubeadm.yaml + + if [[ -n "${LOAD_BALANCER_IP}" ]]; then + sed -i "s/ClusterConfiguration/ClusterConfiguration\ncontrolPlaneEndpoint: ${LOAD_BALANCER_IP}:${LOAD_BALANCER_PORT}/g" ./kubeadm.yaml + fi + # upload-certs will automatically upload certificates that should be shared + # across control-plane nodes in HA clusters. It is harmless in non-HA cases. + sudo -E kubeadm init --upload-certs --config=./kubeadm.yaml rm -rf "${HOME}/.kube" mkdir -p "${HOME}/.kube" diff --git a/clr-k8s-examples/haproxy.cfg.example b/clr-k8s-examples/haproxy.cfg.example new file mode 100644 index 0000000..e5b1f17 --- /dev/null +++ b/clr-k8s-examples/haproxy.cfg.example @@ -0,0 +1,34 @@ +global + log /dev/log local0 + chroot /var/lib/haproxy + stats socket /run/haproxy-master.sock mode 660 level admin + stats timeout 30s + user haproxy + group haproxy + daemon + # Default SSL material locations + ca-base /etc/ssl/certs + ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS + ssl-default-bind-options no-sslv3 +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 5000 + timeout client 50000 + timeout server 50000 + timeout tunnel 4h +frontend kubernetes + bind 10.0.0.100:6444 + option tcplog + mode tcp + default_backend kubernetes-master-nodes + +backend kubernetes-master-nodes + mode tcp + balance source + option tcp-check + server master-1 10.0.0.100:6443 check fall 3 rise 2 + server master-2 10.0.0.101:6443 check fall 3 rise 2 + server master-3 10.0.0.102:6443 check fall 3 rise 2 diff --git a/clr-k8s-examples/setup_system.sh b/clr-k8s-examples/setup_system.sh index 2238e27..746cc7b 100755 --- a/clr-k8s-examples/setup_system.sh +++ b/clr-k8s-examples/setup_system.sh @@ -210,7 +210,7 @@ if [[ -n "${HIGH_POD_COUNT}" ]]; then fi echo "Reloading daemons..." daemon_reload -echo "Enabling Kublet runner..." +echo "Enabling kubelet runner..." enable_kubelet_runner echo "Ensuring system is ready..." ensure_system_ready