diff --git a/metrics/lib/common.bash b/metrics/lib/common.bash index 9ca0cd1..cfa8ef2 100755 --- a/metrics/lib/common.bash +++ b/metrics/lib/common.bash @@ -8,23 +8,25 @@ THIS_FILE=$(readlink -f ${BASH_SOURCE[0]}) LIB_DIR=${THIS_FILE%/*} RESULT_DIR="${LIB_DIR}/../results" -source ${LIB_DIR}/kata-common.bash source ${LIB_DIR}/json.bash source ${LIB_DIR}/k8s-api.bash source /etc/os-release || source /usr/lib/os-release -# Set variables to reasonable defaults if unset or empty -DOCKER_EXE="${DOCKER_EXE:-docker}" +die() { + local msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} -KSM_BASE="/sys/kernel/mm/ksm" -KSM_ENABLE_FILE="${KSM_BASE}/run" -KSM_PAGES_FILE="${KSM_BASE}/pages_to_scan" -KSM_SLEEP_FILE="${KSM_BASE}/sleep_millisecs" +warn() { + local msg="$*" + echo "WARNING: $msg" +} -# The settings we use for an 'aggresive' KSM setup -# Scan 1000 pages every 50ms - 20,000 pages/s -KSM_AGGRESIVE_PAGES=1000 -KSM_AGGRESIVE_SLEEP=50 +info() { + local msg="$*" + echo "INFO: $msg" +} # This function checks existence of commands. # They can be received standalone or as an array, e.g. @@ -42,93 +44,6 @@ check_cmds() done } -# This function performs a docker pull on the image names -# passed in (notionally as 'about to be used'), to ensure -# - that we have the most upto date images -# - that any pull/refresh time (for a first pull) does not -# happen during the test itself. -# -# The image list can be received standalone or as an array, e.g. -# -# images=(“img1” “img2”) -# check_imgs "${images[@]}" -check_images() -{ - local img req_images=( "$@" ) - for img in "${req_images[@]}"; do - echo "docker pull'ing: $img" - if ! docker pull "$img"; then - die "Failed to docker pull image $img" - fi - echo "docker pull'd: $img" - done -} - -# This function performs a docker build on the image names -# passed in, to ensure that we have the latest changes from -# the dockerfiles -build_dockerfile_image() -{ - local image="$1" - local dockerfile_path="$2" - local dockerfile_dir=${2%/*} - - echo "docker building $image" - if ! docker build --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then - die "Failed to docker build image $image" - fi -} - -# This function verifies that the dockerfile version is -# equal to the test version in order to build the image or -# just run the test -check_dockerfiles_images() -{ - local image="$1" - local dockerfile_path="$2" - - if [ -z "$image" ] || [ -z "$dockerfile_path" ]; then - die "Missing image or dockerfile path variable" - fi - - # Verify that dockerfile version is equal to test version - check_image=$(docker images "$image" -q) - if [ -n "$check_image" ]; then - # Check image label - check_image_version=$(docker image inspect $image | grep -w DOCKERFILE_VERSION | head -1 | cut -d '"' -f4) - if [ -n "$check_image_version" ]; then - echo "$image is not updated" - build_dockerfile_image "$image" "$dockerfile_path" - else - # Check dockerfile label - dockerfile_version=$(grep DOCKERFILE_VERSION $dockerfile_path | cut -d '"' -f2) - if [ "$dockerfile_version" != "$check_image_version" ]; then - echo "$dockerfile_version is not equal to $check_image_version" - build_dockerfile_image "$image" "$dockerfile_path" - fi - fi - else - build_dockerfile_image "$image" "$dockerfile_path" - fi -} - -# A one time (per uber test cycle) init that tries to get the -# system to a 'known state' as much as possible -metrics_onetime_init() -{ - # The onetime init must be called once, and only once - if [ ! -z "$onetime_init_done" ]; then - die "onetime_init() called more than once" - fi - - # Restart services - sudo systemctl restart docker - - # We want this to be seen in sub shells as well... - # otherwise init_env() cannot check us - export onetime_init_done=1 -} - # Print a banner to the logs noting clearly which test # we are about to run test_banner() @@ -142,151 +57,13 @@ init_env() { test_banner "${TEST_NAME}" - cmd=("docker") + cmd=("kubectl") # check dependencies check_cmds "${cmd[@]}" - # Remove all stopped containers - clean_env - - # This clean up is more aggressive, this is in order to - # decrease the factors that could affect the metrics results. - kill_processes_before_start -} - -# This function checks if there are containers or -# shim/proxy/hypervisor processes up, if found, they are -# killed to start test with clean environment. -kill_processes_before_start() { - DOCKER_PROCS=$(${DOCKER_EXE} ps -q) - [[ -n "${DOCKER_PROCS}" ]] && clean_env - check_processes -} - -# Generate a random name - generally used when creating containers, but can -# be used for any other appropriate purpose -random_name() { - mktemp -u kata-XXXXXX -} - -# Dump diagnostics about our current system state. -# Very useful for diagnosing if we have failed a sanity check -show_system_state() { - echo "Showing system state:" - echo " --Docker ps--" - ${DOCKER_EXE} ps -a - echo " --${RUNTIME} list--" - local RPATH=$(command -v ${RUNTIME}) - sudo ${RPATH} list - - local processes="kata-proxy kata-shim kata-runtime qemu" - - for p in ${processes}; do - echo " --pgrep ${p}--" - pgrep -a ${p} - done -} - -# Save the current KSM settings so we can restore them later -save_ksm_settings(){ - echo "saving KSM settings" - ksm_stored_run=$(cat ${KSM_ENABLE_FILE}) - ksm_stored_pages=$(cat ${KSM_ENABLE_FILE}) - ksm_stored_sleep=$(cat ${KSM_ENABLE_FILE}) -} - -set_ksm_aggressive(){ - echo "setting KSM to aggressive mode" - # Flip the run off/on to ensure a restart/rescan - sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" - sudo bash -c "echo ${KSM_AGGRESIVE_PAGES} > ${KSM_PAGES_FILE}" - sudo bash -c "echo ${KSM_AGGRESIVE_SLEEP} > ${KSM_SLEEP_FILE}" - sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}" -} - -restore_ksm_settings(){ - echo "restoring KSM settings" - # First turn off the run to ensure if we are then re-enabling - # that any changes take effect - sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" - sudo bash -c "echo ${ksm_stored_pages} > ${KSM_PAGES_FILE}" - sudo bash -c "echo ${ksm_stored_sleep} > ${KSM_SLEEP_FILE}" - sudo bash -c "echo ${ksm_stored_run} > ${KSM_ENABLE_FILE}" -} - -disable_ksm(){ - echo "disabling KSM" - sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" -} - -# See if KSM is enabled. -# If so, amend the test name to reflect that -check_for_ksm(){ - if [ ! -f ${KSM_ENABLE_FILE} ]; then - return - fi - - ksm_on=$(< ${KSM_ENABLE_FILE}) - - if [ $ksm_on == "1" ]; then - TEST_NAME="${TEST_NAME} ksm" - fi -} - -# Wait for KSM to settle down, or timeout waiting -# The basic algorithm is to look at the pages_shared value -# at the end of every 'full scan', and if the value -# has changed very little, then we are done (because we presume -# a full scan has managed to do few new merges) -# -# arg1 - timeout in seconds -wait_ksm_settle(){ - [[ "$RUNTIME" == "runc" ]] || [[ "$RUNTIME" == "kata-fc" ]] && return - local t pcnt - local oldscan=-1 newscan - local oldpages=-1 newpages - - oldscan=$(cat /sys/kernel/mm/ksm/full_scans) - - # Go around the loop until either we see a small % change - # between two full_scans, or we timeout - for ((t=0; t<$1; t++)); do - - newscan=$(cat /sys/kernel/mm/ksm/full_scans) - newpages=$(cat /sys/kernel/mm/ksm/pages_shared) - [[ "$newpages" -eq 0 ]] && echo "No need to wait for KSM to settle" && return - - if (( newscan != oldscan )); then - echo -e "\nnew full_scan ($oldscan to $newscan)" - - # Do we have a previous scan to compare with - echo "check pages $oldpages to $newpages" - - if (( oldpages != -1 )); then - # avoid divide by zero problems - if (( $oldpages > 0 )); then - pcnt=$(( 100 - ((newpages * 100) / oldpages) )) - # abs() - pcnt=$(( $pcnt * -1 )) - - echo "$oldpages to $newpages is ${pcnt}%" - - if (( $pcnt <= 5 )); then - echo "KSM stabilised at ${t}s" - return - fi - else - echo "$oldpages KSM pages... waiting" - fi - fi - oldscan=$newscan - oldpages=$newpages - else - echo -n "." - fi - sleep 1 - done - echo "Timed out after ${1}s waiting for KSM to settle" + # We could try to clean the k8s cluster here... but that + # might remove some pre-installed soak tests etc. that have + # been deliberately injected into the cluster under test. } diff --git a/metrics/lib/kata-common.bash b/metrics/lib/kata-common.bash deleted file mode 100755 index 96bfcb8..0000000 --- a/metrics/lib/kata-common.bash +++ /dev/null @@ -1,200 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2018-2019 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 - -# This file contains common functions that -# are being used by our metrics and integration tests - -# Place where virtcontainers keeps its active pod info -VC_POD_DIR="${VC_POD_DIR:-/var/lib/vc/sbs}" - -# Sandbox runtime directory -RUN_SBS_DIR="${RUN_SBS_DIR:-/run/vc/sbs}" - -KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" - -die() { - local msg="$*" - echo "ERROR: $msg" >&2 - exit 1 -} - -warn() { - local msg="$*" - echo "WARNING: $msg" -} - -info() { - local msg="$*" - echo "INFO: $msg" -} - -# Try to find the real runtime path for the docker runtime passed in $1 -get_docker_kata_path(){ - local jpaths=$(docker info --format "{{json .Runtimes}}" || true) - local rpath=$(jq .\"$1\".path <<< "$jpaths") - # Now we have to de-quote it.. - rpath="${rpath%\"}" - rpath="${rpath#\"}" - echo "$rpath" -} - -# Gets versions and paths of all the components -# list in kata-env -extract_kata_env(){ - local toml - local rpath=$(get_docker_kata_path "$RUNTIME") - if [ -n "$rpath" ]; then - rpath=$(command -v "$rpath" || true) - fi - - # If we can execute the path handed back to us - if [ -x "$rpath" ]; then - # and if the kata-env command does not error out. Bash hack so we can get $? even - # when the sub-command fails, but does not invoke the errexit in this parent shell. - local is_valid=$( $rpath kata-env >/dev/null 2>&1 && echo $? || echo $? ) - - if [ "$is_valid" == "0" ]; then - # then we can parse out the data we want - local toml="$($rpath kata-env)" - - # The runtime path itself, for kata-runtime, will be contained in the `kata-env` - # section. For other runtimes we do not know where the runtime Docker is using lives. - RUNTIME_CONFIG_PATH=$(awk '/^ \[Runtime.Config\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - RUNTIME_VERSION=$(awk '/^ \[Runtime.Version\]$/ {foundit=1} /^ Semver =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - RUNTIME_COMMIT=$(awk '/^ \[Runtime.Version\]$/ {foundit=1} /^ Commit =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - RUNTIME_PATH=$(awk '/^\[Runtime\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - - SHIM_PATH=$(awk '/^\[Shim\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - SHIM_VERSION=$(awk '/^\[Shim\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g') - - PROXY_PATH=$(awk '/^\[Proxy\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - PROXY_VERSION=$(awk '/^\[Proxy\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {print $5; foundit=0} } ' <<< "$toml" | sed 's/"//g') - - HYPERVISOR_PATH=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - HYPERVISOR_VERSION=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g') - - INITRD_PATH=$(awk '/^\[Initrd\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - - NETMON_PATH=$(awk '/^\[Netmon\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') - return 0 - fi - fi - - # We have not found a command with a 'kata-env' option we can run. Set up some - # default values. - # We could be more diligent here and search for each individual component, - # but if the runtime cannot tell us the exact details it is configured for then - # we would be guessing anyway - so, set some defaults that may be true and give - # strong hints that we 'made them up'. - info "Runtime environment not found - setting defaults" - RUNTIME_CONFIG_PATH="/usr/share/defaults/kata-containers/configuration.toml" - RUNTIME_VERSION="0.0.0" - RUNTIME_COMMIT="unknown" - # If docker is broken, disabled or not installed then we may not get a runtime - # path from it... - if [ -z "$RUNTIME_PATH" ]; then - RUNTIME_PATH="/usr/bin/kata-runtime" - else - RUNTIME_PATH="$rpath" - fi - SHIM_PATH="/usr/libexec/kata-containers/kata-shim" - SHIM_VERSION="0.0.0" - PROXY_PATH="/usr/libexec/kata-containers/kata-proxy" - PROXY_VERSION="0.0.0" - if [ "$KATA_HYPERVISOR" == firecracker ]; then - HYPERVISOR_PATH="/usr/bin/firecracker" - else - # We would use $(${cidir}/kata-arch.sh -d) here but we don't know - # that the callee has set up ${cidir} for us. - HYPERVISOR_PATH="/usr/bin/qemu-system-$(uname -m)" - fi - HYPERVISOR_VERSION="0.0.0" - INITRD_PATH="" - NETMON_PATH="/usr/libexec/kata-containers/kata-netmon" -} - -# Checks that processes are not running -check_processes() { - extract_kata_env - - # Only check the kata-env if we have managed to find the kata executable... - if [ -x "$RUNTIME_PATH" ]; then - local vsock_configured=$($RUNTIME_PATH kata-env | awk '/UseVSock/ {print $3}') - local vsock_supported=$($RUNTIME_PATH kata-env | awk '/SupportVSock/ {print $3}') - else - local vsock_configured="false" - local vsock_supported="false" - fi - if [ "$vsock_configured" == true ] && [ "$vsock_supported" == true ]; then - general_processes=( ${HYPERVISOR_PATH} ${SHIM_PATH} ) - else - general_processes=( ${PROXY_PATH} ${HYPERVISOR_PATH} ${SHIM_PATH} ) - fi - for i in "${general_processes[@]}"; do - if pgrep -f "$i"; then - die "Found unexpected ${i} present" - fi - done -} - -# Checks that pods were not left in a directory -check_pods_in_dir() { - local DIR=$1 - if [ -d ${DIR} ]; then - # Verify that pods were not left - pods_number=$(ls ${DIR} | wc -l) - if [ ${pods_number} -ne 0 ]; then - ls ${DIR} - die "${pods_number} pods left and found at ${DIR}" - fi - else - echo "Not ${DIR} directory found" - fi -} - -# Checks that pods were not left -check_pods() { - check_pods_in_dir ${VC_POD_DIR} -} - -# Check that runtimes are not running, they should be transient -check_runtimes() { - runtime_number=$(ps --no-header -C ${RUNTIME} | wc -l) - if [ ${runtime_number} -ne 0 ]; then - die "Unexpected runtime ${RUNTIME} running" - fi -} - -# Clean environment, this function will try to remove all -# stopped/running containers. -clean_env() -{ - # If the timeout has not been set, default it to 30s - # Docker has a built in 10s default timeout, so make ours - # longer than that. - KATA_DOCKER_TIMEOUT=${KATA_DOCKER_TIMEOUT:-30} - containers_running=$(timeout ${KATA_DOCKER_TIMEOUT} docker ps -q) - - if [ ! -z "$containers_running" ]; then - # First stop all containers that are running - # Use kill, as the containers are generally benign, and most - # of the time our 'stop' request ends up doing a `kill` anyway - sudo timeout ${KATA_DOCKER_TIMEOUT} docker kill $containers_running - - # Remove all containers - sudo timeout ${KATA_DOCKER_TIMEOUT} docker rm -f $(docker ps -qa) - fi -} - -get_pod_config_dir() { - if kubectl get runtimeclass 2> /dev/null | grep -q "kata"; then - pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads" - info "k8s configured to use runtimeclass" - else - pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads" - info "k8s configured to use trusted and untrusted annotations" - fi -} diff --git a/metrics/report/makereport.sh b/metrics/report/makereport.sh index 7090079..7338c71 100755 --- a/metrics/report/makereport.sh +++ b/metrics/report/makereport.sh @@ -22,6 +22,54 @@ HOSTOUTPUTDIR="${SCRIPT_PATH}/output" GUESTINPUTDIR="/inputdir/" GUESTOUTPUTDIR="/outputdir/" +# This function performs a docker build on the image names +# passed in, to ensure that we have the latest changes from +# the dockerfiles +build_dockerfile_image() +{ + local image="$1" + local dockerfile_path="$2" + local dockerfile_dir=${2%/*} + + echo "docker building $image" + if ! docker build --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then + die "Failed to docker build image $image" + fi +} + +# This function verifies that the dockerfile version is +# equal to the test version in order to build the image or +# just run the test +check_dockerfiles_images() +{ + local image="$1" + local dockerfile_path="$2" + + if [ -z "$image" ] || [ -z "$dockerfile_path" ]; then + die "Missing image or dockerfile path variable" + fi + + # Verify that dockerfile version is equal to test version + check_image=$(docker images "$image" -q) + if [ -n "$check_image" ]; then + # Check image label + check_image_version=$(docker image inspect $image | grep -w DOCKERFILE_VERSION | head -1 | cut -d '"' -f4) + if [ -n "$check_image_version" ]; then + echo "$image is not updated" + build_dockerfile_image "$image" "$dockerfile_path" + else + # Check dockerfile label + dockerfile_version=$(grep DOCKERFILE_VERSION $dockerfile_path | cut -d '"' -f2) + if [ "$dockerfile_version" != "$check_image_version" ]; then + echo "$dockerfile_version is not equal to $check_image_version" + build_dockerfile_image "$image" "$dockerfile_path" + fi + fi + else + build_dockerfile_image "$image" "$dockerfile_path" + fi +} + setup() { echo "Checking subdirectories" check_subdir="$(ls -dx ${HOSTINPUTDIR}/*/ 2> /dev/null | wc -l)"