Moving from metrics config to setup_system.sh

clr-k8s-examples/README.md

@@ -36,6 +36,13 @@ This script ensures the following
 
 > NOTE: This step is done automatically if using vagrant.
 
+### Configuration for high numbers of pods per node
+
+In order to enable running greater than 110 pods per node, set the environment
+variable `HIGH_POD_COUNT` to any non-empty value.
+
+> NOTE: Use this configuration when utilizing the [metrics](../metrics) tooling in this repo.
+
 ### Enabling experimental firecracker support
 
 > EXPERIMENTAL: Optionally run [`setup_kata_firecracker.sh`](setup_kata_firecracker.sh) to be

clr-k8s-examples/setup_system.sh

@@ -5,6 +5,7 @@ set -o nounset
 
 # global vars
 CLRK8S_OS=${CLRK8S_OS:-""}
+HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
 
 # set no proxy
 ADD_NO_PROXY="10.244.0.0/16,10.96.0.0/12"
@@ -79,6 +80,55 @@ function setup_hosts() {
 	fi
 }
 
+# write increased limits to specified file
+function write_limits_conf() {
+	cat <<EOT | sudo bash -c "cat > $1"
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=1048576
+TimeoutStartSec=0
+MemoryLimit=infinity
+EOT
+}
+
+# update configuration to enable high pod counts
+function config_high_pod_count() {
+	# install bundle dependencies
+	sudo -E swupd bundle-add --quiet jq bc
+
+	# increase max inotify watchers
+	cat <<EOT | sudo bash -c "cat > /etc/sysctl.conf"
+fs.inotify.max_queued_events=1048576
+fs.inotify.max_user_watches=1048576
+fs.inotify.max_user_instances=1048576
+EOT
+	sudo sysctl -p
+
+	# write configuration files
+	sudo mkdir -p /etc/systemd/system/kubelet.service.d
+	write_limits_conf "/etc/systemd/system/kubelet.service.d/limits.conf"
+	if [ "$RUNNER" == "containerd" ]; then
+		sudo mkdir -p /etc/systemd/system/containerd.service.d
+		write_limits_conf "/etc/systemd/system/containerd.service.d/limits.conf"
+	fi
+	if [ "$RUNNER" == "crio" ]; then
+		sudo mkdir -p /etc/systemd/system/crio.service.d
+		write_limits_conf "/etc/systemd/system/crio.service.d/limits.conf"
+	fi
+
+	# increase limits in kubelet
+	sudo sed -i 's/^maxPods\:.*/maxPods\: 5000/' /var/lib/kubelet/config.yaml
+	sudo sed -i 's/^maxOpenFiles\:.*/maxOpenFiles\: 1048576/' /var/lib/kubelet/config.yaml
+
+	# increase the address range per node
+	cat <<EOT >> kubeadm.yaml
+controllerManager:
+  extraArgs:
+    node-cidr-mask-size: "20"
+EOT
+}
+
 # daemon reload
 function daemon_reload() {
 	sudo systemctl daemon-reload
@@ -155,6 +205,10 @@ echo "Setting up modules to load..."
 setup_modules_load
 echo "Setting up /etc/hosts..."
 setup_hosts
+if [[ -n "${HIGH_POD_COUNT}" ]]; then
+	echo "Configure high pod count scaling..."
+	config_high_pod_count
+fi
 echo "Reloading daemons..."
 daemon_reload
 echo "Enabling Kublet runner..."

metrics/README.md

@@ -13,7 +13,6 @@ is below:
 
 | Tool | Description |
 | ---- | ----------- |
-| config | Configuration scripts |
 | lib | General library helper functions for forming and launching workloads, and storing results in a uniform manner to aid later analysis |
 | scaling | Tests to measure scaling, such as linear or parallel launching of pods |
 | report | Rmarkdown based report generator, used to produce a PDF comparison report of 1 or more sets of results |

metrics/config/README.md

@@ -1,5 +0,0 @@
-## Node configuration
-In order to push beyond the default maximum for pods per node in Kubernetes,
-some additional configuration is necessary on each node in the cluster. For
-large pod number (> 110) scaling tests, execute `node-config.sh` on each node
-in your cluster before executing other scripts.

metrics/config/node-config.sh

@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# this is needed for the scaling scripts
-sudo swupd bundle-add jq
-
-# increase max inotify watchers
-cat <<EOT | sudo bash -c "cat > /etc/sysctl.conf"
-fs.inotify.max_queued_events=1048576
-fs.inotify.max_user_watches=1048576
-fs.inotify.max_user_instances=1048576
-EOT
-sudo sysctl -p
-
-sudo mkdir -p /etc/systemd/system/kubelet.service.d/
-cat <<EOT | sudo bash -c "cat > /etc/systemd/system/kubelet.service.d/limits.conf"
-[Service]
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=1048576
-TimeoutStartSec=0
-MemoryLimit=infinity
-EOT
-
-sudo systemctl daemon-reload
-sudo systemctl restart kubelet
-
-sudo mkdir -p /etc/systemd/system/containerd.service.d/
-cat <<EOT | sudo bash -c "cat > /etc/systemd/system/containerd.service.d/limits.conf"
-[Service]
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=1048576
-TimeoutStartSec=0
-MemoryLimit=infinity
-EOT
-
-sudo systemctl daemon-reload
-sudo systemctl restart containerd
-
-# increase limits in kubelet
-sudo sed -i 's/^maxPods\:.*/maxPods\: 5000/' /var/lib/kubelet/config.yaml
-sudo sed -i 's/^maxOpenFiles\:.*/maxOpenFiles\: 1048576/' /var/lib/kubelet/config.yaml
-
-sudo systemctl daemon-reload
-sudo systemctl restart kubelet