Release v51

Switches the debuginfo tarball compression format from xz to zstd.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

.gitignore

@@ -2,7 +2,6 @@
 *.o
 clr_debug_fuse
 clr_debug_daemon
-clr_debug_prepare
 .deps/
 .dirstamp
 Makefile

Makefile.am

@@ -14,7 +14,9 @@ AM_CFLAGS = \
 
 AM_CPPFLAGS = $(AM_CFLAGS) ${curl_CFLAGS} ${fuse_CFLAGS}
 
-bin_PROGRAMS = clr_debug_fuse clr_debug_daemon clr_debug_prepare
+bin_PROGRAMS = clr_debug_fuse clr_debug_daemon
+
+dist_bin_SCRIPTS = scripts/clr_debug_prepare
 
 noinst_LTLIBRARIES = \
 	libnica.la
@@ -36,12 +38,9 @@ clr_debug_daemon_CFLAGS = \
 	$(AM_CFLAGS) \
 	$(LIBSYSTEMD_CFLAGS)
 
-clr_debug_prepare_SOURCES = src/prepare.c
-
 
 clr_debug_fuse_LDADD = ${fuse_LIBS} libnica.la
 clr_debug_daemon_LDADD = ${curl_LIBS} libnica.la ${LIBSYSTEMD_LIBS}
-clr_debug_prepare_LDADD = libnica.la
 
 systemdsystemunit_DATA = clr_debug_fuse.service clr_debug_daemon.service clr_debug_daemon.socket
 

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(clr-debug-info, 48, arjan@linux.intel.com)
+AC_INIT(clr-debug-info, 51, arjan@linux.intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

scripts/clr_debug_prepare

@@ -0,0 +1,146 @@
+#!/bin/bash
+#
+# Clear Linux -- automatic debuginfo preparation
+#
+# Copyright 2019 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, version 3 or later of the License.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+if [ $# -eq 1 ] || [ $# -gt 2 ]; then
+  echo "Usage: $0 [SOURCEDIR] [DESTDIR]"
+  echo "Generates automatic debuginfo tarballs in DESTDIR using content from SOURCEDIR."
+  echo "The default SOURCEDIR and DESTDIR are /var/www/html/debuginfo.raw and"
+  echo "/var/www/html/debuginfo, respectively."
+  exit 0
+fi >&2
+
+if [ $# -eq 2 ]; then
+  if [ ! -d "$1" ]; then
+    echo "ERROR: SOURCEDIR argument must be an existing directory"
+    exit 1
+  fi
+fi >&2
+
+export SRC="${1:-/var/www/html/debuginfo.raw}"
+export DEST="${2:-/var/www/html/debuginfo}"
+
+srclist=$(mktemp -p .)
+destlist=$(mktemp -p .)
+trap "rm $srclist $destlist" EXIT
+
+set -o pipefail
+
+# First, scan the state of the SRC tree, containing all content for debuginfo RPMs.
+if ! find "$SRC" -mindepth 4 -printf "%p %T@ %y\n" | sed "s|$SRC||" | sort -k 1 > "$srclist"; then
+  echo "ERROR: expected content in $SRC"
+  exit 1
+fi
+
+# Next, scan the state of the DEST tree, containing previously generated
+# automatic debuginfo tarballs. Will be empty on first run.
+mkdir -p "$DEST"
+find "$DEST" -mindepth 2 -name '*.tar' -printf "%p %T@ %y\n" | sed "s|$DEST||;s|\.tar | |" | sort -k 1 > "$destlist"
+
+
+process_one() {
+  srcname="$1"
+  destname="$2"
+  filetype="$3"
+  srcloc="$SRC/$srcname"
+  srcdir="$SRC/$(echo "$srcname" | sed -r 's@^(/usr/(lib|share|src)/debug(/src)?).*@\1@')"
+  tarcontent="$(echo "$srcname" | sed -r 's@^/usr/(lib|share|src)/debug(/src)?@.@')"
+  destdir="$DEST/$(dirname "$destname")"
+  dest="$DEST/$destname.tar"
+
+  echo "Creating $destname.tar ..."
+
+  mkdir -p "$destdir"
+  if [ "$filetype" = "f" ]; then
+    tar --no-recursion -C "$srcdir" --zstd -cf "$dest" "$tarcontent"
+  elif [ "$filetype" = "l" ]; then
+    # We "unsymlink" non-broken symlinks as an optimization by adding the
+    # symlink target to the dest tarball, transforming the name as appropriate.
+    if target=$(realpath -e "$srcloc" --relative-to="$srcdir" 2> /dev/null); then
+      tar --no-recursion -C "$srcdir" --zstd -cf "$dest" --transform='s|.*|'"$tarcontent"'|' "$target"
+    else
+      tar --no-recursion -C "$srcdir" --zstd -cf "$dest" "$tarcontent"
+    fi
+  elif [ "$filetype" = "d" ]; then
+    tar --no-recursion -C "$srcdir" --zstd -cf "$dest" "$tarcontent"
+  fi
+}
+export -f process_one
+
+
+gawk '
+BEGIN { OFS = "\t" }
+LIST == "src" {
+  name = $1
+  srcname = name
+  ret = sub(/^\/usr\/lib\/debug/, "/lib", name)
+  if (ret == 0) {
+    ret = sub(/^\/usr\/src\/debug/, "/src", name)
+    if (ret == 0) {
+      ret = sub(/^\/usr\/share\/debug\/src/, "/src", name)
+      if (ret == 0) {
+        ret = sub(/^\/usr\/share\/debug/, "/lib", name)
+        if (ret == 0) {
+          print "Error: unexpected debuginfo file:", name
+          exit 1
+        }
+      }
+    }
+  }
+  src[name]["srcname"] = srcname
+  src[name]["time"] = $2
+  src[name]["type"] = $3
+  src[name]["done"] = 0
+}
+LIST == "dest" {
+  dest_content = 1
+  name = $1
+  time = $2
+  type = $3
+  if (name in src) {
+    # No need to recreate directory tars if they exist
+    if (src[name]["type"] == "d") {
+      src[name]["done"] = 1
+      next
+    } else {
+      # Only regenerate content if timestamp is newer
+      if (time < src[name]["time"]) {
+        print src[name]["srcname"], name, src[name]["type"]
+      }
+      src[name]["done"] = 1
+    }
+  }
+}
+END {
+  PROCINFO["sorted_in"] = "@ind_str_asc"
+  if (dest_content == 0) {
+    for (s in src) {
+      print src[s]["srcname"], s, src[s]["type"]
+    }
+  } else {
+    for (s in src) {
+      if (src[s]["done"] == 0) {
+        print src[s]["srcname"], s, src[s]["type"]
+      }
+    }
+  }
+}
+' LIST="src" "$srclist" LIST="dest" "$destlist" \
+  | parallel --colsep '\t' process_one
+
+
+# vi: ft=sh et sw=2 sts=2

src/prepare.c

@@ -1,243 +0,0 @@
-/*
- *   Fenrus Linux -- automatic debug information installation
- *   Clear Linux -- automatic debug information installation
- *   Server side preparation logic
- *
- *      Copyright (C) 2013  Arjan van de Ven
- *      Copyright (C) 2014  Intel Corporation
- *
- *   This program is free software: you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, version 2 or later of the License.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- *   Authors:
- *         Arjan van de Ven <arjanvandeven@gmail.com>
- *         Ikey Doherty <michael.i.doherty@intel.com>
- *
- */
-
-#define _GNU_SOURCE
-#include <dirent.h>
-#include <errno.h>
-#include <libgen.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include "nica/files.h"
-
-/*
-
-this program takes a directory /var/www/html/debuginfo.raw,
-in which all debuginfo rpms have previously been rpm2cpio'd, and
-creates the content for /var/www/html/debuginfo
-which contains tar'd up/compressed versions of all the files.
-As part of this symlinks will be resolved server side
-as much as possible.
-
-*/
-
-static void unsymlink(char *filename)
-{
-        char *target;
-        __nc_unused__ int r;
-
-        target = canonicalize_file_name(filename);
-        //	sleep(1);
-        //	printf("Symlink %s points to %s\n", filename, target);
-        unlink(filename);
-        r = link(target, filename);
-}
-
-static void wait_for_loadavg(void)
-{
-        FILE *file;
-        char line[4096];
-        __nc_unused__ char *ret = NULL;
-
-        while (1) {
-                double d;
-                file = fopen("/proc/loadavg", "r");
-                if (!file) {
-                        return;
-                }
-                line[0] = 0;
-                ret = fgets(line, 4096, file);
-                fclose(file);
-                d = strtod(line, NULL);
-                if (d < 50) {
-                        return;
-                }
-                sleep(5);
-        }
-}
-int tarcount;
-
-static void do_one_file(char *base1, char *base2, char *path, int isdir)
-{
-        char *dir2 = NULL;
-        autofree(char) * fullpath1, *fullpath2, *dir = NULL;
-        struct stat buf1, buf2;
-        int ret;
-        int sret;
-
-        if (asprintf(&fullpath1, "%s%s", base1, path) < 0) {
-                return;
-        }
-        if (asprintf(&fullpath2, "%s%s.tar", base2, path) < 0) {
-                return;
-        }
-
-        ret = lstat(fullpath1, &buf1);
-        if (ret) {
-                return;
-        }
-
-        if (S_ISLNK(buf1.st_mode)) {
-                //		printf("%s is a symlink\n", fullpath1);
-                ret = stat(fullpath1, &buf2);
-                if (!ret) {
-                        unsymlink(fullpath1);
-                }
-                ret = lstat(fullpath1, &buf1);
-        }
-
-        ret = lstat(fullpath2, &buf2);
-        if (ret || buf1.st_mtime > buf2.st_mtime) {
-                char *command = NULL;
-                unlink(fullpath2);
-
-                dir = strdup(fullpath2);
-
-                dir2 = dirname(dir);
-
-                if (dir2) {
-                        (void)nc_mkdir_p(dir2, 00755);
-                }
-
-                if (chdir(base1) != 0) {
-                        fprintf(stderr, "Failed to chdir: %s\n", strerror(errno));
-                        return;
-                }
-
-                if (!isdir &&
-                    asprintf(&command,
-                             "tar --no-recursion -C %s -Jcf %s %s &",
-                             base1,
-                             fullpath2,
-                             path) >= 0) {
-                        //			printf("Running -%s-\n", command);
-                        printf("Processing %s\n", fullpath2);
-                        tarcount++;
-                        if (tarcount > 50) {
-                                wait_for_loadavg();
-                                tarcount = 0;
-                        }
-                        if ((sret = system(command)) != 0) {
-                                fprintf(stderr, "Failure in command: [%d] %s\n", sret, command);
-                        }
-                        free(command);
-                }
-
-                if (isdir &&
-                    asprintf(&command,
-                             "tar --no-recursion -C %s -Jcf %s `find %s -type d` `find %s "
-                             "-maxdepth 1 -type l` &",
-                             base1,
-                             fullpath2,
-                             path,
-                             path) >= 0) {
-                        //			printf("Running -%s-\n", command);
-                        printf("Processing %s\n", fullpath2);
-                        tarcount++;
-                        if (tarcount > 50) {
-                                wait_for_loadavg();
-                                tarcount = 0;
-                        }
-                        if ((sret = system(command)) != 0) {
-                                fprintf(stderr, "Failure in command: [%d] %s\n", sret, command);
-                        }
-                        free(command);
-                }
-        }
-}
-
-static void recurse_dir(char *base1, char *base2, char *path)
-{
-        DIR *dir;
-        char *fullpath1 = NULL;
-        struct dirent *entry;
-
-        if (asprintf(&fullpath1, "%s%s", base1, path) < 0) {
-                return;
-        }
-
-        dir = opendir(fullpath1);
-
-        if (!dir) {
-                return;
-        }
-
-        while (1) {
-                autofree(char) *fullpath2 = NULL;
-                autofree(char) *newpath = NULL;
-
-                entry = readdir(dir);
-                if (!entry) {
-                        break;
-                }
-                if (strcmp(entry->d_name, ".") == 0) {
-                        continue;
-                }
-                if (strcmp(entry->d_name, "..") == 0) {
-                        continue;
-                }
-
-                newpath = NULL;
-                fullpath2 = NULL;
-                if (asprintf(&fullpath2, "%s/%s", fullpath1, entry->d_name) < 0) {
-                        return;
-                }
-                if (asprintf(&newpath, "%s/%s", path, entry->d_name) >= 0) {
-                        struct stat sb;
-                        stat(fullpath2, &sb);
-
-                        if (entry->d_type == DT_DIR || S_ISDIR(sb.st_mode)) {
-                                do_one_file(base1, base2, newpath, 1);
-                                recurse_dir(base1, base2, newpath);
-                        } else {
-                                do_one_file(base1, base2, newpath, 0);
-                        }
-                }
-        }
-        closedir(dir);
-}
-
-int main(int argc, char **argv)
-{
-        if (argc < 0)
-                (void)argv;
-        recurse_dir("/var/www/html/debuginfo.raw/usr/lib/debug/",
-                    "/var/www/html/debuginfo/lib/",
-                    ".");
-        recurse_dir("/var/www/html/debuginfo.raw/usr/src/debug/",
-                    "/var/www/html/debuginfo/src/",
-                    ".");
-
-        return EXIT_SUCCESS;
-}

src/server.c

@@ -2,7 +2,7 @@
  *   Clear Linux -- automatic debug information installation
  *
  *      Copyright (C) 2013  Arjan van de Ven
- * 	Curl portions borrowed from the Fenrus Update code
+ *     Curl portions borrowed from the Fenrus Update code
  *          which in part is (C) 2012 Intel Corporation
  *      Copyright (C) 2014  Intel Corporation
  *
@@ -32,6 +32,7 @@
 #include <malloc.h>
 #include <pthread.h>
 #include <pwd.h>
+#include <libgen.h>
 #include <signal.h>
 #include <stddef.h>
 #include <stdio.h>
@@ -170,10 +171,9 @@ static int curl_get_file(const char *url, const char *prefix, time_t timestamp)
         long ret;
         long changed;
         int fd;
-        char filename[PATH_MAX];
+        autofree(char) *filename = NULL;
         CURL *curl = NULL;
         FILE *file;
-        struct stat statbuf;
 
         if (avoid_dupes(url)) {
                 return 300;
@@ -184,12 +184,15 @@ static int curl_get_file(const char *url, const char *prefix, time_t timestamp)
                 return 301;
         }
 
-        strcpy(filename, "/tmp/clr-debug-info-XXXXXX");
-
+        // fprintf(stderr, "Fetching %s, prefix %s, path %s\n", url, prefix, path);
+        if (asprintf(&filename, "/tmp/clr-debug-info-XXXXXX") < 0) {
+                curl_easy_cleanup(curl);
+                return 418;
+        }
         fd = mkstemp(filename);
         if (fd < 0) {
                 curl_easy_cleanup(curl);
-                return 500;
+                return 418;
         }
 
         file = fdopen(fd, "w");
@@ -227,7 +230,7 @@ static int curl_get_file(const char *url, const char *prefix, time_t timestamp)
 
         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &ret);
         fflush(file);
-        //	printf("HTTP return code is %i\n", ret);
+        //        printf("HTTP return code is %i\n", ret);
 
         /* HTTP 304 is returned if (a) the cached debuginfo has the same
          * timestamp or is newer than that on the server and (b) we haven't
@@ -240,6 +243,9 @@ static int curl_get_file(const char *url, const char *prefix, time_t timestamp)
         }
 
         if (ret == 200) {
+                autofree(char) *command = NULL;
+                struct stat statbuf;
+
                 /* get timestamp, if any */
                 curl_easy_getinfo(curl, CURLINFO_FILETIME, &changed);
                 if (changed >= 0) {
@@ -251,22 +257,45 @@ static int curl_get_file(const char *url, const char *prefix, time_t timestamp)
                         futimens(fd, times);
                 }
 
-                autofree(char) *command = NULL;
-                //		printf("Filename is %s\n", filename);
-
                 memset(&statbuf, 0, sizeof(statbuf));
                 stat(filename, &statbuf);
-                if (statbuf.st_size > 0 &&
-                    asprintf(&command,
-                             "tar -C /var/cache/debuginfo/%s --no-same-owner "
-                             "--no-same-permissions  -xf %s",
+                if (statbuf.st_size <= 0) {
+                        ret = 418;
+                        goto out;
+                }
+
+                /* test extraction first */
+                if (asprintf(&command, "tar -C /var/cache/debuginfo/%s --no-same-owner "
+                             "--no-same-permissions -tf %s",
                              prefix,
-                             filename) >= 0) {
-                        if (system(command) != 0) {
-                                fputs("Warning: tar extraction failed\n", stderr);
-                        }
+                             filename) < 0) {
+                        ret = 418;
+                        goto out;
+                }
+
+                if (system(command) != 0) {
+                        ret = 418;
+                        fprintf(stderr, "Error: tar validation failed\n");
+                        goto out;
+                }
+
+                free(command); /* reuse */
+                if (asprintf(&command, "tar -C /var/cache/debuginfo/%s --no-same-owner "
+                             "--no-same-permissions -xf %s",
+                             prefix,
+                             filename) < 0) {
+                        ret = 418;
+                        goto out;
+                }
+
+                if (system(command) != 0) {
+                        ret = 418;
+                        fprintf(stderr, "Error: tar extraction failed\n");
+                        goto out;
                 }
         }
+
+out:
         unlink(filename);
         curl_easy_cleanup(curl);
         fclose(file);
@@ -340,7 +369,7 @@ static void *server_thread(void *arg)
                 goto thread_end;
         }
 
-        //	printf("Getting url %s    %i:%06i\n", url, before.tv_sec, before.tv_usec);
+        //        printf("Getting url %s    %i:%06i\n", url, before.tv_sec, before.tv_usec);
         ret = curl_get_file(url, prefix, timestamp);
 
         switch (ret) {
@@ -351,7 +380,7 @@ static void *server_thread(void *arg)
                 // ignore these error codes
                 break;
         default:
-                printf("Request for %s resulted in error %i\n", url, ret);
+                fprintf(stderr, "Request for %s resulted in error %i\n", url, ret);
                 break;
         }
 
@@ -437,7 +466,7 @@ int main(__nc_unused__ int argc, __nc_unused__ char **argv)
                 /* systemd socket activation */
                 sockfd = SD_LISTEN_FDS_START + 0;
         } else if (sd_listen_fds(0) > 1) {
-                printf("Too many file descriptors received.\n");
+                fprintf(stderr, "Too many file descriptors received.\n");
                 exit(EXIT_FAILURE);
         } else {
                 sockfd = socket(AF_UNIX, SOCK_STREAM, 0);