From f35848b8e07a9c4b92c8b2557f4fe8a4d4ffbdf6 Mon Sep 17 00:00:00 2001 From: George T Kramer Date: Thu, 24 Oct 2019 12:33:27 -0700 Subject: [PATCH] Reduce disk and CPU usage during mash creation Because the mash script will always need to run on the kojimaster server, it has local access to RPMs. Rather than copying the data, create a symlink to the data. This reduces disk usage and allows the mash hosting location to be flexible. Because the mash is re-generated every time kojira runs and creates a new koji repository, by nature, the mash is an iterative process. Existing repository metadata is always available (excpet for the first run). Pass this existing metadata to createrepo_c so that it may more efficiently diff and recalculate the updated metadata in order to reduce CPU and disk usage. By reducing disk and CPU usage, this reduces the impact of running this process on a kojimaster which is a shared system responsible for hosting many resources. Signed-off-by: George T Kramer --- koji-setup/mash.sh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/koji-setup/mash.sh b/koji-setup/mash.sh index 5550fbd..e96f8cc 100755 --- a/koji-setup/mash.sh +++ b/koji-setup/mash.sh @@ -14,7 +14,8 @@ MASH_DIR_OLD="$MASH_TRACKER_DIR".old MASH_DIR_NEW="$MASH_TRACKER_DIR".new create_dist_repos() { - local output_dir="${1}" + local source_dir="${1}" + local output_dir="${2}" local work_dir="$(mktemp -d)" @@ -36,9 +37,9 @@ create_dist_repos() { cp -f "${KOJI_REPO_PATH}/groups/comps.xml" "${comps_file}" - make_repo "${output_dir}/clear/${BUILD_ARCH}/os" "Packages" "${bin_rpm_paths}" "${comps_file}" & - make_repo "${output_dir}/clear/${BUILD_ARCH}/debug" "." "${debuginfo_rpm_paths}" & - make_repo "${output_dir}/clear/source/SRPMS" "." "${src_rpm_paths}" & + make_repo "${source_dir}" "${output_dir}" "clear/${BUILD_ARCH}/os" "Packages" "${bin_rpm_paths}" "${comps_file}" & + make_repo "${source_dir}" "${output_dir}" "clear/${BUILD_ARCH}/debug" "." "${debuginfo_rpm_paths}" & + make_repo "${source_dir}" "${output_dir}" "clear/source/SRPMS" "." "${src_rpm_paths}" & wait create_dnf_conf "${work_dir}/dnf-os.conf" "${output_dir}/clear/${BUILD_ARCH}/os" clear-os @@ -53,15 +54,19 @@ create_dist_repos() { } make_repo() { - local repo_dir="${1}" - local rpm_dir="${1}/${2}" - local file_list="${3}" - local comps_file="${4}" + local previous_repo_dir="${1}/${3}" + local repo_dir="${2}/${3}" + local rpm_dir="${repo_dir}/${4}" + local file_list="${5}" + local comps_file="${6}" local create_repo_cmd="createrepo_c --quiet --database --compress-type xz --workers $(nproc --all)" + if [[ -e "${previous_repo_dir}" ]]; then + create_repo_cmd="${create_repo_cmd} --update --update-md-path ${previous_repo_dir}" + fi mkdir -p "${rpm_dir}" - xargs -a "${file_list}" -I {} cp -f {} "${rpm_dir}" + xargs -a "${file_list}" -I {} ln -sf {} "${rpm_dir}" if [[ -z "${comps_file}" ]]; then ${create_repo_cmd} "${repo_dir}" else @@ -100,7 +105,7 @@ KOJI_BUILD_NUM="$(basename "$KOJI_REPO_PATH")" if [[ "$MASH_BUILD_NUM" -ne "$KOJI_BUILD_NUM" ]]; then rm -rf "$MASH_DIR_NEW" mkdir -p "$MASH_DIR_NEW" - create_dist_repos "$MASH_DIR_NEW" + create_dist_repos "$MASH_TRACKER_DIR" "$MASH_DIR_NEW" if [[ -e "$MASH_TRACKER_DIR" ]]; then mv "$MASH_TRACKER_DIR" "$MASH_DIR_OLD" fi