metrics: common: remove un-used docker-esque code

We were carrying a bunch of legacy docker based functions
that we do not use. Drop them. We can always add them back
in later if we find we do need some.

Signed-off-by: Graham Whaley <graham.whaley@intel.com>
This commit is contained in:
Graham Whaley
2019-07-30 13:13:35 -06:00
committed by David Lyle
parent fa94c2ce90
commit 59a1f36e37
3 changed files with 65 additions and 440 deletions
+17 -240
View File
@@ -8,23 +8,25 @@ THIS_FILE=$(readlink -f ${BASH_SOURCE[0]})
LIB_DIR=${THIS_FILE%/*}
RESULT_DIR="${LIB_DIR}/../results"
source ${LIB_DIR}/kata-common.bash
source ${LIB_DIR}/json.bash
source ${LIB_DIR}/k8s-api.bash
source /etc/os-release || source /usr/lib/os-release
# Set variables to reasonable defaults if unset or empty
DOCKER_EXE="${DOCKER_EXE:-docker}"
die() {
local msg="$*"
echo "ERROR: $msg" >&2
exit 1
}
KSM_BASE="/sys/kernel/mm/ksm"
KSM_ENABLE_FILE="${KSM_BASE}/run"
KSM_PAGES_FILE="${KSM_BASE}/pages_to_scan"
KSM_SLEEP_FILE="${KSM_BASE}/sleep_millisecs"
warn() {
local msg="$*"
echo "WARNING: $msg"
}
# The settings we use for an 'aggresive' KSM setup
# Scan 1000 pages every 50ms - 20,000 pages/s
KSM_AGGRESIVE_PAGES=1000
KSM_AGGRESIVE_SLEEP=50
info() {
local msg="$*"
echo "INFO: $msg"
}
# This function checks existence of commands.
# They can be received standalone or as an array, e.g.
@@ -42,93 +44,6 @@ check_cmds()
done
}
# This function performs a docker pull on the image names
# passed in (notionally as 'about to be used'), to ensure
# - that we have the most upto date images
# - that any pull/refresh time (for a first pull) does not
# happen during the test itself.
#
# The image list can be received standalone or as an array, e.g.
#
# images=(“img1” “img2”)
# check_imgs "${images[@]}"
check_images()
{
local img req_images=( "$@" )
for img in "${req_images[@]}"; do
echo "docker pull'ing: $img"
if ! docker pull "$img"; then
die "Failed to docker pull image $img"
fi
echo "docker pull'd: $img"
done
}
# This function performs a docker build on the image names
# passed in, to ensure that we have the latest changes from
# the dockerfiles
build_dockerfile_image()
{
local image="$1"
local dockerfile_path="$2"
local dockerfile_dir=${2%/*}
echo "docker building $image"
if ! docker build --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
die "Failed to docker build image $image"
fi
}
# This function verifies that the dockerfile version is
# equal to the test version in order to build the image or
# just run the test
check_dockerfiles_images()
{
local image="$1"
local dockerfile_path="$2"
if [ -z "$image" ] || [ -z "$dockerfile_path" ]; then
die "Missing image or dockerfile path variable"
fi
# Verify that dockerfile version is equal to test version
check_image=$(docker images "$image" -q)
if [ -n "$check_image" ]; then
# Check image label
check_image_version=$(docker image inspect $image | grep -w DOCKERFILE_VERSION | head -1 | cut -d '"' -f4)
if [ -n "$check_image_version" ]; then
echo "$image is not updated"
build_dockerfile_image "$image" "$dockerfile_path"
else
# Check dockerfile label
dockerfile_version=$(grep DOCKERFILE_VERSION $dockerfile_path | cut -d '"' -f2)
if [ "$dockerfile_version" != "$check_image_version" ]; then
echo "$dockerfile_version is not equal to $check_image_version"
build_dockerfile_image "$image" "$dockerfile_path"
fi
fi
else
build_dockerfile_image "$image" "$dockerfile_path"
fi
}
# A one time (per uber test cycle) init that tries to get the
# system to a 'known state' as much as possible
metrics_onetime_init()
{
# The onetime init must be called once, and only once
if [ ! -z "$onetime_init_done" ]; then
die "onetime_init() called more than once"
fi
# Restart services
sudo systemctl restart docker
# We want this to be seen in sub shells as well...
# otherwise init_env() cannot check us
export onetime_init_done=1
}
# Print a banner to the logs noting clearly which test
# we are about to run
test_banner()
@@ -142,151 +57,13 @@ init_env()
{
test_banner "${TEST_NAME}"
cmd=("docker")
cmd=("kubectl")
# check dependencies
check_cmds "${cmd[@]}"
# Remove all stopped containers
clean_env
# This clean up is more aggressive, this is in order to
# decrease the factors that could affect the metrics results.
kill_processes_before_start
}
# This function checks if there are containers or
# shim/proxy/hypervisor processes up, if found, they are
# killed to start test with clean environment.
kill_processes_before_start() {
DOCKER_PROCS=$(${DOCKER_EXE} ps -q)
[[ -n "${DOCKER_PROCS}" ]] && clean_env
check_processes
}
# Generate a random name - generally used when creating containers, but can
# be used for any other appropriate purpose
random_name() {
mktemp -u kata-XXXXXX
}
# Dump diagnostics about our current system state.
# Very useful for diagnosing if we have failed a sanity check
show_system_state() {
echo "Showing system state:"
echo " --Docker ps--"
${DOCKER_EXE} ps -a
echo " --${RUNTIME} list--"
local RPATH=$(command -v ${RUNTIME})
sudo ${RPATH} list
local processes="kata-proxy kata-shim kata-runtime qemu"
for p in ${processes}; do
echo " --pgrep ${p}--"
pgrep -a ${p}
done
}
# Save the current KSM settings so we can restore them later
save_ksm_settings(){
echo "saving KSM settings"
ksm_stored_run=$(cat ${KSM_ENABLE_FILE})
ksm_stored_pages=$(cat ${KSM_ENABLE_FILE})
ksm_stored_sleep=$(cat ${KSM_ENABLE_FILE})
}
set_ksm_aggressive(){
echo "setting KSM to aggressive mode"
# Flip the run off/on to ensure a restart/rescan
sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
sudo bash -c "echo ${KSM_AGGRESIVE_PAGES} > ${KSM_PAGES_FILE}"
sudo bash -c "echo ${KSM_AGGRESIVE_SLEEP} > ${KSM_SLEEP_FILE}"
sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}"
}
restore_ksm_settings(){
echo "restoring KSM settings"
# First turn off the run to ensure if we are then re-enabling
# that any changes take effect
sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
sudo bash -c "echo ${ksm_stored_pages} > ${KSM_PAGES_FILE}"
sudo bash -c "echo ${ksm_stored_sleep} > ${KSM_SLEEP_FILE}"
sudo bash -c "echo ${ksm_stored_run} > ${KSM_ENABLE_FILE}"
}
disable_ksm(){
echo "disabling KSM"
sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
}
# See if KSM is enabled.
# If so, amend the test name to reflect that
check_for_ksm(){
if [ ! -f ${KSM_ENABLE_FILE} ]; then
return
fi
ksm_on=$(< ${KSM_ENABLE_FILE})
if [ $ksm_on == "1" ]; then
TEST_NAME="${TEST_NAME} ksm"
fi
}
# Wait for KSM to settle down, or timeout waiting
# The basic algorithm is to look at the pages_shared value
# at the end of every 'full scan', and if the value
# has changed very little, then we are done (because we presume
# a full scan has managed to do few new merges)
#
# arg1 - timeout in seconds
wait_ksm_settle(){
[[ "$RUNTIME" == "runc" ]] || [[ "$RUNTIME" == "kata-fc" ]] && return
local t pcnt
local oldscan=-1 newscan
local oldpages=-1 newpages
oldscan=$(cat /sys/kernel/mm/ksm/full_scans)
# Go around the loop until either we see a small % change
# between two full_scans, or we timeout
for ((t=0; t<$1; t++)); do
newscan=$(cat /sys/kernel/mm/ksm/full_scans)
newpages=$(cat /sys/kernel/mm/ksm/pages_shared)
[[ "$newpages" -eq 0 ]] && echo "No need to wait for KSM to settle" && return
if (( newscan != oldscan )); then
echo -e "\nnew full_scan ($oldscan to $newscan)"
# Do we have a previous scan to compare with
echo "check pages $oldpages to $newpages"
if (( oldpages != -1 )); then
# avoid divide by zero problems
if (( $oldpages > 0 )); then
pcnt=$(( 100 - ((newpages * 100) / oldpages) ))
# abs()
pcnt=$(( $pcnt * -1 ))
echo "$oldpages to $newpages is ${pcnt}%"
if (( $pcnt <= 5 )); then
echo "KSM stabilised at ${t}s"
return
fi
else
echo "$oldpages KSM pages... waiting"
fi
fi
oldscan=$newscan
oldpages=$newpages
else
echo -n "."
fi
sleep 1
done
echo "Timed out after ${1}s waiting for KSM to settle"
# We could try to clean the k8s cluster here... but that
# might remove some pre-installed soak tests etc. that have
# been deliberately injected into the cluster under test.
}
-200
View File
@@ -1,200 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2018-2019 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
# This file contains common functions that
# are being used by our metrics and integration tests
# Place where virtcontainers keeps its active pod info
VC_POD_DIR="${VC_POD_DIR:-/var/lib/vc/sbs}"
# Sandbox runtime directory
RUN_SBS_DIR="${RUN_SBS_DIR:-/run/vc/sbs}"
KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
die() {
local msg="$*"
echo "ERROR: $msg" >&2
exit 1
}
warn() {
local msg="$*"
echo "WARNING: $msg"
}
info() {
local msg="$*"
echo "INFO: $msg"
}
# Try to find the real runtime path for the docker runtime passed in $1
get_docker_kata_path(){
local jpaths=$(docker info --format "{{json .Runtimes}}" || true)
local rpath=$(jq .\"$1\".path <<< "$jpaths")
# Now we have to de-quote it..
rpath="${rpath%\"}"
rpath="${rpath#\"}"
echo "$rpath"
}
# Gets versions and paths of all the components
# list in kata-env
extract_kata_env(){
local toml
local rpath=$(get_docker_kata_path "$RUNTIME")
if [ -n "$rpath" ]; then
rpath=$(command -v "$rpath" || true)
fi
# If we can execute the path handed back to us
if [ -x "$rpath" ]; then
# and if the kata-env command does not error out. Bash hack so we can get $? even
# when the sub-command fails, but does not invoke the errexit in this parent shell.
local is_valid=$( $rpath kata-env >/dev/null 2>&1 && echo $? || echo $? )
if [ "$is_valid" == "0" ]; then
# then we can parse out the data we want
local toml="$($rpath kata-env)"
# The runtime path itself, for kata-runtime, will be contained in the `kata-env`
# section. For other runtimes we do not know where the runtime Docker is using lives.
RUNTIME_CONFIG_PATH=$(awk '/^ \[Runtime.Config\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
RUNTIME_VERSION=$(awk '/^ \[Runtime.Version\]$/ {foundit=1} /^ Semver =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
RUNTIME_COMMIT=$(awk '/^ \[Runtime.Version\]$/ {foundit=1} /^ Commit =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
RUNTIME_PATH=$(awk '/^\[Runtime\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
SHIM_PATH=$(awk '/^\[Shim\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
SHIM_VERSION=$(awk '/^\[Shim\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g')
PROXY_PATH=$(awk '/^\[Proxy\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
PROXY_VERSION=$(awk '/^\[Proxy\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {print $5; foundit=0} } ' <<< "$toml" | sed 's/"//g')
HYPERVISOR_PATH=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
HYPERVISOR_VERSION=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g')
INITRD_PATH=$(awk '/^\[Initrd\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
NETMON_PATH=$(awk '/^\[Netmon\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
return 0
fi
fi
# We have not found a command with a 'kata-env' option we can run. Set up some
# default values.
# We could be more diligent here and search for each individual component,
# but if the runtime cannot tell us the exact details it is configured for then
# we would be guessing anyway - so, set some defaults that may be true and give
# strong hints that we 'made them up'.
info "Runtime environment not found - setting defaults"
RUNTIME_CONFIG_PATH="/usr/share/defaults/kata-containers/configuration.toml"
RUNTIME_VERSION="0.0.0"
RUNTIME_COMMIT="unknown"
# If docker is broken, disabled or not installed then we may not get a runtime
# path from it...
if [ -z "$RUNTIME_PATH" ]; then
RUNTIME_PATH="/usr/bin/kata-runtime"
else
RUNTIME_PATH="$rpath"
fi
SHIM_PATH="/usr/libexec/kata-containers/kata-shim"
SHIM_VERSION="0.0.0"
PROXY_PATH="/usr/libexec/kata-containers/kata-proxy"
PROXY_VERSION="0.0.0"
if [ "$KATA_HYPERVISOR" == firecracker ]; then
HYPERVISOR_PATH="/usr/bin/firecracker"
else
# We would use $(${cidir}/kata-arch.sh -d) here but we don't know
# that the callee has set up ${cidir} for us.
HYPERVISOR_PATH="/usr/bin/qemu-system-$(uname -m)"
fi
HYPERVISOR_VERSION="0.0.0"
INITRD_PATH=""
NETMON_PATH="/usr/libexec/kata-containers/kata-netmon"
}
# Checks that processes are not running
check_processes() {
extract_kata_env
# Only check the kata-env if we have managed to find the kata executable...
if [ -x "$RUNTIME_PATH" ]; then
local vsock_configured=$($RUNTIME_PATH kata-env | awk '/UseVSock/ {print $3}')
local vsock_supported=$($RUNTIME_PATH kata-env | awk '/SupportVSock/ {print $3}')
else
local vsock_configured="false"
local vsock_supported="false"
fi
if [ "$vsock_configured" == true ] && [ "$vsock_supported" == true ]; then
general_processes=( ${HYPERVISOR_PATH} ${SHIM_PATH} )
else
general_processes=( ${PROXY_PATH} ${HYPERVISOR_PATH} ${SHIM_PATH} )
fi
for i in "${general_processes[@]}"; do
if pgrep -f "$i"; then
die "Found unexpected ${i} present"
fi
done
}
# Checks that pods were not left in a directory
check_pods_in_dir() {
local DIR=$1
if [ -d ${DIR} ]; then
# Verify that pods were not left
pods_number=$(ls ${DIR} | wc -l)
if [ ${pods_number} -ne 0 ]; then
ls ${DIR}
die "${pods_number} pods left and found at ${DIR}"
fi
else
echo "Not ${DIR} directory found"
fi
}
# Checks that pods were not left
check_pods() {
check_pods_in_dir ${VC_POD_DIR}
}
# Check that runtimes are not running, they should be transient
check_runtimes() {
runtime_number=$(ps --no-header -C ${RUNTIME} | wc -l)
if [ ${runtime_number} -ne 0 ]; then
die "Unexpected runtime ${RUNTIME} running"
fi
}
# Clean environment, this function will try to remove all
# stopped/running containers.
clean_env()
{
# If the timeout has not been set, default it to 30s
# Docker has a built in 10s default timeout, so make ours
# longer than that.
KATA_DOCKER_TIMEOUT=${KATA_DOCKER_TIMEOUT:-30}
containers_running=$(timeout ${KATA_DOCKER_TIMEOUT} docker ps -q)
if [ ! -z "$containers_running" ]; then
# First stop all containers that are running
# Use kill, as the containers are generally benign, and most
# of the time our 'stop' request ends up doing a `kill` anyway
sudo timeout ${KATA_DOCKER_TIMEOUT} docker kill $containers_running
# Remove all containers
sudo timeout ${KATA_DOCKER_TIMEOUT} docker rm -f $(docker ps -qa)
fi
}
get_pod_config_dir() {
if kubectl get runtimeclass 2> /dev/null | grep -q "kata"; then
pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
info "k8s configured to use runtimeclass"
else
pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
info "k8s configured to use trusted and untrusted annotations"
fi
}
+48
View File
@@ -22,6 +22,54 @@ HOSTOUTPUTDIR="${SCRIPT_PATH}/output"
GUESTINPUTDIR="/inputdir/"
GUESTOUTPUTDIR="/outputdir/"
# This function performs a docker build on the image names
# passed in, to ensure that we have the latest changes from
# the dockerfiles
build_dockerfile_image()
{
local image="$1"
local dockerfile_path="$2"
local dockerfile_dir=${2%/*}
echo "docker building $image"
if ! docker build --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
die "Failed to docker build image $image"
fi
}
# This function verifies that the dockerfile version is
# equal to the test version in order to build the image or
# just run the test
check_dockerfiles_images()
{
local image="$1"
local dockerfile_path="$2"
if [ -z "$image" ] || [ -z "$dockerfile_path" ]; then
die "Missing image or dockerfile path variable"
fi
# Verify that dockerfile version is equal to test version
check_image=$(docker images "$image" -q)
if [ -n "$check_image" ]; then
# Check image label
check_image_version=$(docker image inspect $image | grep -w DOCKERFILE_VERSION | head -1 | cut -d '"' -f4)
if [ -n "$check_image_version" ]; then
echo "$image is not updated"
build_dockerfile_image "$image" "$dockerfile_path"
else
# Check dockerfile label
dockerfile_version=$(grep DOCKERFILE_VERSION $dockerfile_path | cut -d '"' -f2)
if [ "$dockerfile_version" != "$check_image_version" ]; then
echo "$dockerfile_version is not equal to $check_image_version"
build_dockerfile_image "$image" "$dockerfile_path"
fi
fi
else
build_dockerfile_image "$image" "$dockerfile_path"
fi
}
setup() {
echo "Checking subdirectories"
check_subdir="$(ls -dx ${HOSTINPUTDIR}/*/ 2> /dev/null | wc -l)"