From b476b5d6e5c7868d5a62fc3b867aba19b38675e8 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Fri, 6 Sep 2019 16:20:20 -0600 Subject: [PATCH] Moving from metrics config to setup_system.sh --- clr-k8s-examples/README.md | 7 +++++ clr-k8s-examples/setup_system.sh | 54 ++++++++++++++++++++++++++++++++ metrics/README.md | 1 - metrics/config/README.md | 5 --- metrics/config/node-config.sh | 47 --------------------------- 5 files changed, 61 insertions(+), 53 deletions(-) delete mode 100644 metrics/config/README.md delete mode 100755 metrics/config/node-config.sh diff --git a/clr-k8s-examples/README.md b/clr-k8s-examples/README.md index d3514c3..4ef2586 100644 --- a/clr-k8s-examples/README.md +++ b/clr-k8s-examples/README.md @@ -36,6 +36,13 @@ This script ensures the following > NOTE: This step is done automatically if using vagrant. +### Configuration for high numbers of pods per node + +In order to enable running greater than 110 pods per node, set the environment +variable `HIGH_POD_COUNT` to any non-empty value. + +> NOTE: Use this configuration when utilizing the [metrics](../metrics) tooling in this repo. + ### Enabling experimental firecracker support > EXPERIMENTAL: Optionally run [`setup_kata_firecracker.sh`](setup_kata_firecracker.sh) to be diff --git a/clr-k8s-examples/setup_system.sh b/clr-k8s-examples/setup_system.sh index 30cf1ee..14831a1 100755 --- a/clr-k8s-examples/setup_system.sh +++ b/clr-k8s-examples/setup_system.sh @@ -5,6 +5,7 @@ set -o nounset # global vars CLRK8S_OS=${CLRK8S_OS:-""} +HIGH_POD_COUNT=${HIGH_POD_COUNT:-""} # set no proxy ADD_NO_PROXY="10.244.0.0/16,10.96.0.0/12" @@ -79,6 +80,55 @@ function setup_hosts() { fi } +# write increased limits to specified file +function write_limits_conf() { + cat < $1" +[Service] +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=1048576 +TimeoutStartSec=0 +MemoryLimit=infinity +EOT +} + +# update configuration to enable high pod counts +function config_high_pod_count() { + # install bundle dependencies + sudo -E swupd bundle-add --quiet jq bc + + # increase max inotify watchers + cat < /etc/sysctl.conf" +fs.inotify.max_queued_events=1048576 +fs.inotify.max_user_watches=1048576 +fs.inotify.max_user_instances=1048576 +EOT + sudo sysctl -p + + # write configuration files + sudo mkdir -p /etc/systemd/system/kubelet.service.d + write_limits_conf "/etc/systemd/system/kubelet.service.d/limits.conf" + if [ "$RUNNER" == "containerd" ]; then + sudo mkdir -p /etc/systemd/system/containerd.service.d + write_limits_conf "/etc/systemd/system/containerd.service.d/limits.conf" + fi + if [ "$RUNNER" == "crio" ]; then + sudo mkdir -p /etc/systemd/system/crio.service.d + write_limits_conf "/etc/systemd/system/crio.service.d/limits.conf" + fi + + # increase limits in kubelet + sudo sed -i 's/^maxPods\:.*/maxPods\: 5000/' /var/lib/kubelet/config.yaml + sudo sed -i 's/^maxOpenFiles\:.*/maxOpenFiles\: 1048576/' /var/lib/kubelet/config.yaml + + # increase the address range per node + cat <> kubeadm.yaml +controllerManager: + extraArgs: + node-cidr-mask-size: "20" +EOT +} + # daemon reload function daemon_reload() { sudo systemctl daemon-reload @@ -155,6 +205,10 @@ echo "Setting up modules to load..." setup_modules_load echo "Setting up /etc/hosts..." setup_hosts +if [[ -n "${HIGH_POD_COUNT}" ]]; then + echo "Configure high pod count scaling..." + config_high_pod_count +fi echo "Reloading daemons..." daemon_reload echo "Enabling Kublet runner..." diff --git a/metrics/README.md b/metrics/README.md index c1b0353..02c8461 100644 --- a/metrics/README.md +++ b/metrics/README.md @@ -13,7 +13,6 @@ is below: | Tool | Description | | ---- | ----------- | -| config | Configuration scripts | | lib | General library helper functions for forming and launching workloads, and storing results in a uniform manner to aid later analysis | | scaling | Tests to measure scaling, such as linear or parallel launching of pods | | report | Rmarkdown based report generator, used to produce a PDF comparison report of 1 or more sets of results | diff --git a/metrics/config/README.md b/metrics/config/README.md deleted file mode 100644 index 6f79096..0000000 --- a/metrics/config/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Node configuration -In order to push beyond the default maximum for pods per node in Kubernetes, -some additional configuration is necessary on each node in the cluster. For -large pod number (> 110) scaling tests, execute `node-config.sh` on each node -in your cluster before executing other scripts. diff --git a/metrics/config/node-config.sh b/metrics/config/node-config.sh deleted file mode 100755 index f322c67..0000000 --- a/metrics/config/node-config.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# this is needed for the scaling scripts -sudo swupd bundle-add jq - -# increase max inotify watchers -cat < /etc/sysctl.conf" -fs.inotify.max_queued_events=1048576 -fs.inotify.max_user_watches=1048576 -fs.inotify.max_user_instances=1048576 -EOT -sudo sysctl -p - -sudo mkdir -p /etc/systemd/system/kubelet.service.d/ -cat < /etc/systemd/system/kubelet.service.d/limits.conf" -[Service] -LimitNOFILE=1048576 -LimitNPROC=1048576 -LimitCORE=1048576 -TimeoutStartSec=0 -MemoryLimit=infinity -EOT - -sudo systemctl daemon-reload -sudo systemctl restart kubelet - -sudo mkdir -p /etc/systemd/system/containerd.service.d/ -cat < /etc/systemd/system/containerd.service.d/limits.conf" -[Service] -LimitNOFILE=1048576 -LimitNPROC=1048576 -LimitCORE=1048576 -TimeoutStartSec=0 -MemoryLimit=infinity -EOT - -sudo systemctl daemon-reload -sudo systemctl restart containerd - -# increase limits in kubelet -sudo sed -i 's/^maxPods\:.*/maxPods\: 5000/' /var/lib/kubelet/config.yaml -sudo sed -i 's/^maxOpenFiles\:.*/maxOpenFiles\: 1048576/' /var/lib/kubelet/config.yaml - -sudo systemctl daemon-reload -sudo systemctl restart kubelet