Release v51

Switches the debuginfo tarball compression format from xz to zstd.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

.gitignore

@@ -2,7 +2,6 @@
 *.o
 clr_debug_fuse
 clr_debug_daemon
-clr_debug_prepare
 .deps/
 .dirstamp
 Makefile

Makefile.am

@@ -14,7 +14,9 @@ AM_CFLAGS = \
 
 AM_CPPFLAGS = $(AM_CFLAGS) ${curl_CFLAGS} ${fuse_CFLAGS}
 
-bin_PROGRAMS = clr_debug_fuse clr_debug_daemon clr_debug_prepare
+bin_PROGRAMS = clr_debug_fuse clr_debug_daemon
+
+dist_bin_SCRIPTS = scripts/clr_debug_prepare
 
 noinst_LTLIBRARIES = \
 	libnica.la
@@ -36,12 +38,9 @@ clr_debug_daemon_CFLAGS = \
 	$(AM_CFLAGS) \
 	$(LIBSYSTEMD_CFLAGS)
 
-clr_debug_prepare_SOURCES = src/prepare.c
-
 
 clr_debug_fuse_LDADD = ${fuse_LIBS} libnica.la
 clr_debug_daemon_LDADD = ${curl_LIBS} libnica.la ${LIBSYSTEMD_LIBS}
-clr_debug_prepare_LDADD = libnica.la
 
 systemdsystemunit_DATA = clr_debug_fuse.service clr_debug_daemon.service clr_debug_daemon.socket
 

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(clr-debug-info, 49, arjan@linux.intel.com)
+AC_INIT(clr-debug-info, 51, arjan@linux.intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

scripts/clr_debug_prepare

@@ -0,0 +1,146 @@
+#!/bin/bash
+#
+# Clear Linux -- automatic debuginfo preparation
+#
+# Copyright 2019 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, version 3 or later of the License.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+if [ $# -eq 1 ] || [ $# -gt 2 ]; then
+  echo "Usage: $0 [SOURCEDIR] [DESTDIR]"
+  echo "Generates automatic debuginfo tarballs in DESTDIR using content from SOURCEDIR."
+  echo "The default SOURCEDIR and DESTDIR are /var/www/html/debuginfo.raw and"
+  echo "/var/www/html/debuginfo, respectively."
+  exit 0
+fi >&2
+
+if [ $# -eq 2 ]; then
+  if [ ! -d "$1" ]; then
+    echo "ERROR: SOURCEDIR argument must be an existing directory"
+    exit 1
+  fi
+fi >&2
+
+export SRC="${1:-/var/www/html/debuginfo.raw}"
+export DEST="${2:-/var/www/html/debuginfo}"
+
+srclist=$(mktemp -p .)
+destlist=$(mktemp -p .)
+trap "rm $srclist $destlist" EXIT
+
+set -o pipefail
+
+# First, scan the state of the SRC tree, containing all content for debuginfo RPMs.
+if ! find "$SRC" -mindepth 4 -printf "%p %T@ %y\n" | sed "s|$SRC||" | sort -k 1 > "$srclist"; then
+  echo "ERROR: expected content in $SRC"
+  exit 1
+fi
+
+# Next, scan the state of the DEST tree, containing previously generated
+# automatic debuginfo tarballs. Will be empty on first run.
+mkdir -p "$DEST"
+find "$DEST" -mindepth 2 -name '*.tar' -printf "%p %T@ %y\n" | sed "s|$DEST||;s|\.tar | |" | sort -k 1 > "$destlist"
+
+
+process_one() {
+  srcname="$1"
+  destname="$2"
+  filetype="$3"
+  srcloc="$SRC/$srcname"
+  srcdir="$SRC/$(echo "$srcname" | sed -r 's@^(/usr/(lib|share|src)/debug(/src)?).*@\1@')"
+  tarcontent="$(echo "$srcname" | sed -r 's@^/usr/(lib|share|src)/debug(/src)?@.@')"
+  destdir="$DEST/$(dirname "$destname")"
+  dest="$DEST/$destname.tar"
+
+  echo "Creating $destname.tar ..."
+
+  mkdir -p "$destdir"
+  if [ "$filetype" = "f" ]; then
+    tar --no-recursion -C "$srcdir" --zstd -cf "$dest" "$tarcontent"
+  elif [ "$filetype" = "l" ]; then
+    # We "unsymlink" non-broken symlinks as an optimization by adding the
+    # symlink target to the dest tarball, transforming the name as appropriate.
+    if target=$(realpath -e "$srcloc" --relative-to="$srcdir" 2> /dev/null); then
+      tar --no-recursion -C "$srcdir" --zstd -cf "$dest" --transform='s|.*|'"$tarcontent"'|' "$target"
+    else
+      tar --no-recursion -C "$srcdir" --zstd -cf "$dest" "$tarcontent"
+    fi
+  elif [ "$filetype" = "d" ]; then
+    tar --no-recursion -C "$srcdir" --zstd -cf "$dest" "$tarcontent"
+  fi
+}
+export -f process_one
+
+
+gawk '
+BEGIN { OFS = "\t" }
+LIST == "src" {
+  name = $1
+  srcname = name
+  ret = sub(/^\/usr\/lib\/debug/, "/lib", name)
+  if (ret == 0) {
+    ret = sub(/^\/usr\/src\/debug/, "/src", name)
+    if (ret == 0) {
+      ret = sub(/^\/usr\/share\/debug\/src/, "/src", name)
+      if (ret == 0) {
+        ret = sub(/^\/usr\/share\/debug/, "/lib", name)
+        if (ret == 0) {
+          print "Error: unexpected debuginfo file:", name
+          exit 1
+        }
+      }
+    }
+  }
+  src[name]["srcname"] = srcname
+  src[name]["time"] = $2
+  src[name]["type"] = $3
+  src[name]["done"] = 0
+}
+LIST == "dest" {
+  dest_content = 1
+  name = $1
+  time = $2
+  type = $3
+  if (name in src) {
+    # No need to recreate directory tars if they exist
+    if (src[name]["type"] == "d") {
+      src[name]["done"] = 1
+      next
+    } else {
+      # Only regenerate content if timestamp is newer
+      if (time < src[name]["time"]) {
+        print src[name]["srcname"], name, src[name]["type"]
+      }
+      src[name]["done"] = 1
+    }
+  }
+}
+END {
+  PROCINFO["sorted_in"] = "@ind_str_asc"
+  if (dest_content == 0) {
+    for (s in src) {
+      print src[s]["srcname"], s, src[s]["type"]
+    }
+  } else {
+    for (s in src) {
+      if (src[s]["done"] == 0) {
+        print src[s]["srcname"], s, src[s]["type"]
+      }
+    }
+  }
+}
+' LIST="src" "$srclist" LIST="dest" "$destlist" \
+  | parallel --colsep '\t' process_one
+
+
+# vi: ft=sh et sw=2 sts=2

src/prepare.c

@@ -1,243 +0,0 @@
-/*
- *   Fenrus Linux -- automatic debug information installation
- *   Clear Linux -- automatic debug information installation
- *   Server side preparation logic
- *
- *      Copyright (C) 2013  Arjan van de Ven
- *      Copyright (C) 2014  Intel Corporation
- *
- *   This program is free software: you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, version 2 or later of the License.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- *   Authors:
- *         Arjan van de Ven <arjanvandeven@gmail.com>
- *         Ikey Doherty <michael.i.doherty@intel.com>
- *
- */
-
-#define _GNU_SOURCE
-#include <dirent.h>
-#include <errno.h>
-#include <libgen.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include "nica/files.h"
-
-/*
-
-this program takes a directory /var/www/html/debuginfo.raw,
-in which all debuginfo rpms have previously been rpm2cpio'd, and
-creates the content for /var/www/html/debuginfo
-which contains tar'd up/compressed versions of all the files.
-As part of this symlinks will be resolved server side
-as much as possible.
-
-*/
-
-static void unsymlink(char *filename)
-{
-        char *target;
-        __nc_unused__ int r;
-
-        target = canonicalize_file_name(filename);
-        //	sleep(1);
-        //	printf("Symlink %s points to %s\n", filename, target);
-        unlink(filename);
-        r = link(target, filename);
-}
-
-static void wait_for_loadavg(void)
-{
-        FILE *file;
-        char line[4096];
-        __nc_unused__ char *ret = NULL;
-
-        while (1) {
-                double d;
-                file = fopen("/proc/loadavg", "r");
-                if (!file) {
-                        return;
-                }
-                line[0] = 0;
-                ret = fgets(line, 4096, file);
-                fclose(file);
-                d = strtod(line, NULL);
-                if (d < 50) {
-                        return;
-                }
-                sleep(5);
-        }
-}
-int tarcount;
-
-static void do_one_file(char *base1, char *base2, char *path, int isdir)
-{
-        char *dir2 = NULL;
-        autofree(char) * fullpath1, *fullpath2, *dir = NULL;
-        struct stat buf1, buf2;
-        int ret;
-        int sret;
-
-        if (asprintf(&fullpath1, "%s%s", base1, path) < 0) {
-                return;
-        }
-        if (asprintf(&fullpath2, "%s%s.tar", base2, path) < 0) {
-                return;
-        }
-
-        ret = lstat(fullpath1, &buf1);
-        if (ret) {
-                return;
-        }
-
-        if (S_ISLNK(buf1.st_mode)) {
-                //		printf("%s is a symlink\n", fullpath1);
-                ret = stat(fullpath1, &buf2);
-                if (!ret) {
-                        unsymlink(fullpath1);
-                }
-                ret = lstat(fullpath1, &buf1);
-        }
-
-        ret = lstat(fullpath2, &buf2);
-        if (ret || buf1.st_mtime > buf2.st_mtime) {
-                char *command = NULL;
-                unlink(fullpath2);
-
-                dir = strdup(fullpath2);
-
-                dir2 = dirname(dir);
-
-                if (dir2) {
-                        (void)nc_mkdir_p(dir2, 00755);
-                }
-
-                if (chdir(base1) != 0) {
-                        fprintf(stderr, "Failed to chdir: %s\n", strerror(errno));
-                        return;
-                }
-
-                if (!isdir &&
-                    asprintf(&command,
-                             "tar --no-recursion -C %s -Jcf %s %s &",
-                             base1,
-                             fullpath2,
-                             path) >= 0) {
-                        //			printf("Running -%s-\n", command);
-                        printf("Processing %s\n", fullpath2);
-                        tarcount++;
-                        if (tarcount > 50) {
-                                wait_for_loadavg();
-                                tarcount = 0;
-                        }
-                        if ((sret = system(command)) != 0) {
-                                fprintf(stderr, "Failure in command: [%d] %s\n", sret, command);
-                        }
-                        free(command);
-                }
-
-                if (isdir &&
-                    asprintf(&command,
-                             "tar --no-recursion -C %s -Jcf %s `find %s -type d` `find %s "
-                             "-maxdepth 1 -type l` &",
-                             base1,
-                             fullpath2,
-                             path,
-                             path) >= 0) {
-                        //			printf("Running -%s-\n", command);
-                        printf("Processing %s\n", fullpath2);
-                        tarcount++;
-                        if (tarcount > 50) {
-                                wait_for_loadavg();
-                                tarcount = 0;
-                        }
-                        if ((sret = system(command)) != 0) {
-                                fprintf(stderr, "Failure in command: [%d] %s\n", sret, command);
-                        }
-                        free(command);
-                }
-        }
-}
-
-static void recurse_dir(char *base1, char *base2, char *path)
-{
-        DIR *dir;
-        char *fullpath1 = NULL;
-        struct dirent *entry;
-
-        if (asprintf(&fullpath1, "%s%s", base1, path) < 0) {
-                return;
-        }
-
-        dir = opendir(fullpath1);
-
-        if (!dir) {
-                return;
-        }
-
-        while (1) {
-                autofree(char) *fullpath2 = NULL;
-                autofree(char) *newpath = NULL;
-
-                entry = readdir(dir);
-                if (!entry) {
-                        break;
-                }
-                if (strcmp(entry->d_name, ".") == 0) {
-                        continue;
-                }
-                if (strcmp(entry->d_name, "..") == 0) {
-                        continue;
-                }
-
-                newpath = NULL;
-                fullpath2 = NULL;
-                if (asprintf(&fullpath2, "%s/%s", fullpath1, entry->d_name) < 0) {
-                        return;
-                }
-                if (asprintf(&newpath, "%s/%s", path, entry->d_name) >= 0) {
-                        struct stat sb;
-                        stat(fullpath2, &sb);
-
-                        if (entry->d_type == DT_DIR || S_ISDIR(sb.st_mode)) {
-                                do_one_file(base1, base2, newpath, 1);
-                                recurse_dir(base1, base2, newpath);
-                        } else {
-                                do_one_file(base1, base2, newpath, 0);
-                        }
-                }
-        }
-        closedir(dir);
-}
-
-int main(int argc, char **argv)
-{
-        if (argc < 0)
-                (void)argv;
-        recurse_dir("/var/www/html/debuginfo.raw/usr/lib/debug/",
-                    "/var/www/html/debuginfo/lib/",
-                    ".");
-        recurse_dir("/var/www/html/debuginfo.raw/usr/src/debug/",
-                    "/var/www/html/debuginfo/src/",
-                    ".");
-
-        return EXIT_SUCCESS;
-}