From d72621ae2ec55d8b10383ba88559a43fb541bbc9 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Fri, 2 Aug 2019 17:25:16 -0600 Subject: [PATCH] show which node the pod launched on, store pod name --- metrics/report/report_dockerfile/scaling.R | 37 ++++++++++++- metrics/scaling/k8s_scale.sh | 64 ++++++++++++++++++++++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/metrics/report/report_dockerfile/scaling.R b/metrics/report/report_dockerfile/scaling.R index e8b9b7d..5e8874e 100755 --- a/metrics/report/report_dockerfile/scaling.R +++ b/metrics/report/report_dockerfile/scaling.R @@ -19,6 +19,7 @@ testnames=c( data=c() fndata=c() +pndata=c() stats=c() rstats=c() rstats_names=c() @@ -118,9 +119,39 @@ for (currentdir in resultdirs) { skip_points=1 } + # format the pod data from 2 nested columns in a series + # of index specific columns to just 2 columns + pdata=data.frame(fdata$BootResults$launched_pods) + pudata=c() + for (i in seq(length(cdata[, "boot_time"]))) { + if (i == 1) { + # there are no valid values for this index, skipping + next + } + else { + # shift to 0 based indexing + index=i-1 + sindex=(index*2)+1 + eindex=sindex+1 + row=cbind(pdata[,sindex:eindex]) + c1=cbind(podname=row[,1]) + c2=cbind(node=row[,2]) + c3=cbind(count=rep(i, length(pdata$pod_name))) + c4=cbind(boot_time=rep(cdata[, "boot_time"][i],length(pdata$pod_name))) + c5=cbind(dataset=rep(testname, length(pdata$pod_name))) + prow=cbind(c1,c2,c3,c4,c5) + pudata=rbind(pudata,prow) + } + } + # pndata is considered a vector for some reason so converting it to a data.frame + pudata=as.data.frame(pudata) + pudata$count=as.numeric(as.character(pudata$count)) + pudata$boot_time=as.numeric(as.character(pudata$boot_time)) + cdata=cbind(cdata, count=seq_len(length(cdata[, "boot_time"]))) cdata=cbind(cdata, testname=rep(testname, length(cdata[, "boot_time"]) )) cdata=cbind(cdata, dataset=rep(datasetname, length(cdata[, "boot_time"]) )) + #cdata=cbind(cdata, pndata) # Gather our statistics # '-1' containers, as the first entry should be a data capture of before @@ -156,6 +187,7 @@ for (currentdir in resultdirs) { # Store away as a single set data=rbind(data, cdata) fndata=rbind(fndata, fudata) + pndata=rbind(pndata, pudata) stats=rbind(stats, sdata) ms = c( @@ -193,7 +225,7 @@ mem_stats_plot = suppressWarnings(ggtexttable(data.frame(rstats), rows=NULL )) -# plot how samples varied over 'time' +# plot how samples varied over 'time' mem_line_plot <- ggplot(data=fndata, aes(as.numeric(as.character(pod)), as.numeric(as.character(mem_free)), colour=testname, group=interaction(testname, node))) + @@ -231,7 +263,6 @@ if ( skip_points == 0 ) { # Show how boot time changed boot_line_plot <- ggplot() + geom_line( data=data, aes(count, boot_time, colour=testname, group=dataset), alpha=0.2) + - geom_smooth( data=data, aes(count, boot_time, colour=testname, group=dataset), se=FALSE, method="loess", size=0.3) + xlab("pods") + ylab("Boot time (s)") + ggtitle("Pod boot time") + @@ -239,7 +270,7 @@ boot_line_plot <- ggplot() + theme(axis.text.x=element_text(angle=90)) if ( skip_points == 0 ) { - boot_line_plot = boot_line_plot + geom_point( data=data, aes(count, boot_time, colour=testname, group=dataset), alpha=0.3) + boot_line_plot = boot_line_plot + geom_point( data=pndata, aes(count, boot_time, shape=node, group=dataset), alpha=0.3) } mem_text <- paste("Footprint density statistics") diff --git a/metrics/scaling/k8s_scale.sh b/metrics/scaling/k8s_scale.sh index a729b79..7c8d087 100755 --- a/metrics/scaling/k8s_scale.sh +++ b/metrics/scaling/k8s_scale.sh @@ -35,6 +35,8 @@ grace=${grace:-30} TEST_ARGS="runtime=${RUNTIME}" TEST_NAME="k8s scaling" +declare -a new_pods + # $1 is the launch time in seconds this pod/container took to start up. # $2 is the number of pod/containers under test grab_stats() { @@ -62,6 +64,7 @@ EOF )" metrics_json_add_array_fragment "$launch_json" + # start the node utilization array metrics_json_start_nested_array # grab pods in the stats daemonset @@ -111,6 +114,39 @@ EOF metrics_json_end_nested_array "node_util" + # start the new pods array + metrics_json_start_nested_array + + # for the first call to grab stats, there are no new pods + # so we need to fill in with NA (R specific value) in matching + # diminsion to the rest of the calls to grab_stats, so $STEP items + if [[ ${#new_pods[@]} == 0 ]]; then + for i in $STEP; do + local new_pod_json="$(cat << EOF + { + "pod_name": "NA", + "node": "NA" + } +EOF + )" + metrics_json_add_nested_array_element "$new_pod_json" + done + else + local maxelem=$(( ${#new_pods[@]} - 1 )) + for index in $(seq 0 $maxelem); do + local node=$(kubectl get pod ${new_pods[$index]} -o json | jq -r '"\(.spec.nodeName)"') + local new_pod_json="$(cat << EOF + { + "pod_name": "${new_pods[$index]}", + "node": "${node}" + } +EOF + )" + metrics_json_add_nested_array_element "$new_pod_json" + done + fi + metrics_json_end_nested_array "launched_pods" + metrics_json_close_array_element } @@ -203,6 +239,9 @@ run() { -e "s|@GRACE@|${grace}|g" \ < ${input_template} > ${generated_file} + # get list of workload pods before launching another one + local pods_before=$(kubectl get pods --selector ${LABEL}=${LABELVALUE} -o json | jq -r '.items[] | "\(.metadata.name)"') + info "Applying changes" local start_time=$(date +%s%N) if [ "$use_api" != "no" ]; then @@ -221,11 +260,36 @@ run() { local end_time=$(date +%s%N) local total_milliseconds=$(( (end_time - start_time) / 1000000 )) info "Took $total_milliseconds ms ($end_time - $start_time)" + + # grab list of workload pods after + local pods_after=$(kubectl get pods --selector ${LABEL}=${LABELVALUE} -o json | jq -r '.items[] | "\(.metadata.name)"') + find_unique_pods "${pods_after}" "${pods_before}" + sleep ${settle_time} grab_stats $total_milliseconds $reqs done } +# finds elements in $1 that are not in $2 +find_unique_pods() { + local list_a=$1 + local list_b=$2 + + new_pods=() + for a in $list_a; do + local in_b=false + for b in $list_b; do + if [[ $a == $b ]]; then + in_b=true + break + fi + done + if [[ $in_b == false ]]; then + new_pods[${#new_pods[@]}]=$a + fi + done +} + cleanup() { info "Cleaning up"