diff --git a/metrics/collectd/collectd.bash b/metrics/collectd/collectd.bash index ba84b2a..85c400d 100644 --- a/metrics/collectd/collectd.bash +++ b/metrics/collectd/collectd.bash @@ -12,15 +12,25 @@ collectd_pod="collectd" init_stats() { local wait_time=$1 - # create collectd-config configmap + # create collectd-config configmap, delete old if there is one + kubectl get configmap collectd-config >/dev/null 2>&1 && kubectl delete configmap collectd-config kubectl create configmap collectd-config --from-file=${COLLECTD_DIR}/collectd.conf + # if there is collectd daemonset already running, delete it + # to make sure that the latest configmap will be used. + kubectl get daemonset collectd >/dev/null 2>&1 && kubectl delete daemonset --wait=true --timeout=${delete_wait_time}s "${collectd_pod}" + # Launch our stats gathering pod kubectl apply -f ${COLLECTD_DIR}/${collectd_pod}.yaml kubectl rollout status --timeout=${wait_time}s daemonset/${collectd_pod} + # clear existing collectd output + while read -u 3 name node; do + kubectl exec -ti $name -- sh -c "rm -rf /mnt/opt/collectd/run/localhost/*" + done 3< <(kubectl get pods --selector name=collectd-pods -o json | jq -r '.items[] | "\(.metadata.name) \(.spec.nodeName)"') + # attempting to provide buffer for collectd to be installed and running, - # and CPU collection to build adequate history + # and CPU collection to build adequate history sleep 12 } @@ -30,11 +40,9 @@ cleanup_stats() { # get logs before shutting down stats daemonset while read -u 3 name node; do - kubectl exec -ti $name -- sh -c "cd /opt/collectd; tar -czvf localhost.tar.gz localhost" - # make a backup on the host in-case collection fail - kubectl exec -ti $name -- sh -c "mkdir -p /mnt/opt/collectd" - kubectl exec -ti $name -- sh -c "cp /opt/collectd/localhost.tar.gz /mnt/opt/collectd/localhost.tar.gz" - kubectl cp $name:/opt/collectd/localhost.tar.gz ${RESULT_DIR}/${node}.tar.gz + kubectl exec -ti $name -- sh -c "cd /mnt/opt/collectd/run; rm -f ../localhost.tar.gz; tar -czvf ../localhost.tar.gz localhost" + kubectl cp $name:/mnt/opt/collectd/localhost.tar.gz ${RESULT_DIR}/${node}.tar.gz + kubectl exec -ti $name -- sh -c "rm -rf /mnt/opt/collectd/run" done 3< <(kubectl get pods --selector name=collectd-pods -o json | jq -r '.items[] | "\(.metadata.name) \(.spec.nodeName)"') kubectl delete daemonset --wait=true --timeout=${delete_wait_time}s "${collectd_pod}" || true diff --git a/metrics/collectd/collectd.conf b/metrics/collectd/collectd.conf index 94569ee..c2ce52f 100644 --- a/metrics/collectd/collectd.conf +++ b/metrics/collectd/collectd.conf @@ -17,7 +17,7 @@ Hostname localhost ValuesPercentage true - DataDir "/opt/collectd" + DataDir "/mnt/opt/collectd/run" StoreRates true