From 5e53ce9afbc658b58a98155c629b1c51e1679409 Mon Sep 17 00:00:00 2001
From: Graham Whaley <graham.whaley@intel.com>
Date: Tue, 20 Aug 2019 11:48:32 +0100
Subject: [PATCH] metrics: scaling: calculate mem consumed and pod/Gb

Keep track of memory consumed on the nodes, log that, and
generate an overall cluster value for 'pods-per-Gb'.

Generate this data here, whilst we have the data to hand, as
it can get much more difficult to calculate this information
the further you get down the monitoring stream (such as the
farther you get into an ELK stack).

Signed-off-by: Graham Whaley <graham.whaley@intel.com>
---
 metrics/scaling/k8s_scale.sh | 38 ++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/metrics/scaling/k8s_scale.sh b/metrics/scaling/k8s_scale.sh
index 8f37759..03f425d 100755
--- a/metrics/scaling/k8s_scale.sh
+++ b/metrics/scaling/k8s_scale.sh
@@ -36,6 +36,7 @@ TEST_ARGS="runtime=${RUNTIME}"
 TEST_NAME="k8s scaling"
 
 declare -a new_pods
+declare -a node_basemem
 
 # $1 is the launch time in seconds this pod/container took to start up.
 # $2 is the number of pod/containers under test
@@ -44,6 +45,8 @@ grab_stats() {
 	local n_pods=$2
 	local cpu_idle=()
 	local mem_free=()
+	local total_mem_used=0
+
 	info "And grab some stats"
 
 	local date_json="$(cat << EOF
@@ -101,6 +104,17 @@ EOF
 		cpu_idle=${cpu_idle:-0}
 		mem_free=${mem_free:-0}
 
+		# If this is the 0 node instance, store away the base memory value
+		if [ $n_pods -eq 0 ]; then
+			node_basemem[$node]=$mem_free
+		fi
+
+		local mem_used=$((node_basemem[$node]-mem_free))
+		# Only account for memory usage on schedulable nodes
+		if [ $noschedule == false ]; then
+			total_mem_used=$((total_mem_used+mem_used))
+		fi
+
 		local util_json="$(cat << EOF
 		{
 			"node": "${node}",
@@ -112,6 +126,10 @@ EOF
 			"mem_free": {
 				"Result": ${mem_free},
 				"Units" : "kb"
+			},
+			"mem_used": {
+				"Result": ${mem_used},
+				"Units" : "kb"
 			}
 		}
 EOF
@@ -156,6 +174,26 @@ EOF
 	fi
 	metrics_json_end_nested_array "launched_pods"
 
+	# And store off the total memory consumed across all nodes, and the pod/Gb value
+	if [ $n_pods -eq 0 ]; then
+		local pods_per_gb=0
+	else
+		local pods_per_gb=$(bc -l <<< "scale=2; ($total_mem_used/1024) / $n_pods")
+	fi
+	local mem_json="$(cat << EOF
+			"memory": {
+				"consumed": {
+					"Result": ${total_mem_used},
+					"Units": "Kb"
+				},
+				"pods_per_gb": {
+					"Result": ${pods_per_gb}
+				}
+			}
+EOF
+	)"
+	metrics_json_add_array_fragment "$mem_json"
+
 	metrics_json_close_array_element
 }