Release v1.11.0

This release reworks the journal probe to send records for log messages
with higher log levels (LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR), and
also for messages that match EXIT_CODE="exited", which is set for
systemd services that have exited with an error code.

Also, a new opt-in feature has been added to disable certain privacy
filters in the probes. With the filters disabled, the crash probe will
collect backtraces for any crashing binary on the system, regardless of
where the binary is installed. Also, the oops probe will collect real
register values from kernel oopses instead of using "zero" or
"non-zero".

The opt in feature described above can be enabled by creating a file at:

  /etc/telemetrics/opt-in-no-privacy-filters

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([telemetrics-client], [1.10.2], [https://clearlinux.org/])
+AC_INIT([telemetrics-client], [1.11.0], [https://clearlinux.org/])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([1.14 -Wall -Werror -Wno-extra-portability foreign subdir-objects])
 AM_SILENT_RULES([yes])

src/probes/crash_probe.c

@@ -38,6 +38,7 @@
 
 #include "config.h"
 #include "log.h"
+#include "probe.h"
 #include "telemetry.h"
 
 static Dwfl *d_core = NULL;
@@ -391,6 +392,11 @@ fail:
 
 static bool in_clr_build(gchar *fullpath)
 {
+        // Global override for privacy filters
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                return false;
+        }
+
         /*
          * The build environment for Clear Linux packages is set up by 'mock',
          * and the chroot in which rpmbuild builds the packages has this prefix.
@@ -403,17 +409,22 @@ static bool in_clr_build(gchar *fullpath)
         return false;
 }
 
-static bool filter_binaries(gchar *fullpath)
+static bool is_banned_path(gchar *fullpath)
 {
-        // Anything outside of /usr/, or in /usr/local/, we consider third-party
-        if (g_regex_match_simple("^[!/]usr[!/]", fullpath, 0, 0) == FALSE) {
-                return false;
-        } else if (g_regex_match_simple("^[!/]usr[!/]local[!/]", fullpath,
-                                        0, 0) == TRUE) {
+        // Global override for privacy filters
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
                 return false;
         }
 
-        return true;
+        // Anything outside of /usr/, or in /usr/local/, we consider third-party
+        if (g_regex_match_simple("^[!/]usr[!/]", fullpath, 0, 0) == FALSE) {
+                return true;
+        } else if (g_regex_match_simple("^[!/]usr[!/]local[!/]", fullpath,
+                                        0, 0) == TRUE) {
+                return true;
+        }
+
+        return false;
 }
 
 static gchar *config_file = NULL;
@@ -506,7 +517,7 @@ int main(int argc, char **argv)
                 goto success;
         }
 
-        if (proc_path && !filter_binaries(proc_path)) {
+        if (proc_path && is_banned_path(proc_path)) {
                 telem_log(LOG_NOTICE, "Ignoring core (third-party binary)\n");
 
                 backtrace = g_string_new("Crash from third party\n");

src/probes/journal.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2015-2017 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -61,11 +61,11 @@ static inline void tm_journal_match_err(int ret)
 static void add_to_payload(const void *data, size_t length)
 {
         if (payload != NULL) {
-                g_string_append_printf(payload, "%.*s", (int)length,
+                g_string_append_printf(payload, "%.*s\n", (int)length,
                                        (char *)data);
         } else {
                 payload = g_string_new(NULL);
-                g_string_printf(payload, "%.*s", (int)length,
+                g_string_printf(payload, "%.*s\n", (int)length,
                                 (char *)data);
         }
 }
@@ -77,7 +77,7 @@ static bool send_data(char *class)
 
         if ((ret = tm_create_record(&handle, severity, class,
                                     payload_version)) < 0) {
-                telem_log(LOG_ERR, "Failed to create record: %s",
+                telem_log(LOG_ERR, "Failed to create record: %s\n",
                           strerror(-ret));
                 goto fail;
         }
@@ -86,7 +86,7 @@ static bool send_data(char *class)
         payload = NULL;
 
         if ((ret = tm_set_payload(handle, (char *)payload_str)) < 0) {
-                telem_log(LOG_ERR, "Failed to set payload: %s", strerror(-ret));
+                telem_log(LOG_ERR, "Failed to set payload: %s\n", strerror(-ret));
                 free(payload_str);
                 tm_free_record(handle);
                 goto fail;
@@ -95,7 +95,7 @@ static bool send_data(char *class)
         free(payload_str);
 
         if ((ret = tm_send_record(handle)) < 0) {
-                telem_log(LOG_ERR, "Failed to send record: %s", strerror(-ret));
+                telem_log(LOG_ERR, "Failed to send record: %s\n", strerror(-ret));
                 tm_free_record(handle);
                 goto fail;
         }
@@ -122,6 +122,16 @@ static int read_new_entries(sd_journal *journal)
 
                 add_to_payload(data, length);
 
+                // For now, we send one record per log message, in case the
+                // there is a large backlog of messages and we exceed the
+                // payload size limit (8KB). And ignore errors, hoping that it's
+                // a transient problem.
+
+                if (!send_data(error_class)) {
+                        telem_log(LOG_ERR, "Failed to send data. Ignoring.\n");
+                        return num_entries;
+                }
+
                 num_entries++;
         }
 
@@ -148,19 +158,13 @@ static bool process_existing_entries(sd_journal *journal)
                 return false;
         }
 
-        if (!read_new_entries(journal)) {
+        ret = read_new_entries(journal);
+        if (ret < 0) {
                 return false;
-        }
-
-        if (!payload) {
-                telem_log(LOG_DEBUG, "No existing entries found\n");
+        } else if (ret == 0) {
                 return true;
         }
 
-        if (!send_data(error_class)) {
-                return false;
-        }
-
         return true;
 }
 
@@ -185,6 +189,33 @@ static bool get_boot_id(char **data)
         return true;
 }
 
+#define JOURNAL_MATCH(data) \
+        do { \
+                r = sd_journal_add_match(journal, data, 0); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
+#define JOURNAL_AND \
+        do { \
+                r = sd_journal_add_conjunction(journal); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
+#define JOURNAL_OR \
+        do { \
+                r = sd_journal_add_disjunction(journal); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
 static bool add_filters(sd_journal *journal)
 {
         char *data = NULL;
@@ -194,24 +225,29 @@ static bool add_filters(sd_journal *journal)
                 return false;
         }
 
-        r = sd_journal_add_match(journal, data, 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
-        }
+        /* The semantics of how journal entry matching works is described in
+         * detail in sd_journal_add_match(3).
+         *
+         * The matches declared here correspond to the logical expression:
+         *
+         * BOOTID && ((P0 || P1 || P2 || P3) || EXITED)
+         *
+         * BOOTID is short for _BOOT_ID=VAL, where VAL is the boot ID for the
+         * current boot. P0, P1, etc stand for PRIORITY=0, etc. And EXITED is
+         * short for EXIT_CODE=exited.
+         */
+
+        JOURNAL_MATCH(data);
         free(data);
-
-        r = sd_journal_add_match(journal, "SYSLOG_IDENTIFIER=crashprobe", 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
-        }
-
-        r = sd_journal_add_match(journal, "PRIORITY=3", 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
-        }
+        JOURNAL_AND;
+        // The four highest log levels, all indicating errors
+        JOURNAL_MATCH("PRIORITY=0");
+        JOURNAL_MATCH("PRIORITY=1");
+        JOURNAL_MATCH("PRIORITY=2");
+        JOURNAL_MATCH("PRIORITY=3");
+        JOURNAL_OR;
+        // Only set for service-level error conditions
+        JOURNAL_MATCH("EXIT_CODE=exited");
 
         return true;
 }
@@ -258,10 +294,6 @@ static bool process_journal(void)
                                 } else if (r < 0) {
                                         return false;
                                 }
-
-                                if (!send_data(error_class)) {
-                                        return false;
-                                }
                         }
                 }
         }

src/probes/local.mk

@@ -44,7 +44,8 @@ endif
 
 
 %C%_crashprobe_SOURCES = \
-	%D%/crash_probe.c
+	%D%/crash_probe.c \
+	%D%/probe.h
 %C%_crashprobe_CFLAGS = \
 	$(AM_CFLAGS) \
 	$(GLIB_CFLAGS)
@@ -120,7 +121,8 @@ endif
 
 %C%_oopsprobe_SOURCES = \
         %D%/oops_probe.c \
-	%D%/oops_parser.c
+	%D%/oops_parser.c \
+	%D%/probe.h
 %C%_oopsprobe_CFLAGS = \
         $(AM_CFLAGS) \
         $(GLIB_CFLAGS)

src/probes/oops_parser.c

@@ -21,9 +21,11 @@
 #include <string.h>
 #include <ctype.h>
 #include <inttypes.h>
+#include <unistd.h>
 
 #include "oops_parser.h"
 #include "log.h"
+#include "probe.h"
 
 #define NUM_TAINTED_FLAGS 16
 
@@ -704,13 +706,14 @@ void append_registers_to_bt(GString **backtrace)
         struct reg_s *reg_entry = reg_head;
 
         while (reg_entry != NULL) {
-                #ifdef DEBUG
-                g_string_append_printf(*backtrace, "Register %s: %" PRIx64 "\n", reg_entry->reg_name,
-                                       reg_entry->reg_value);
-                #else
-                g_string_append_printf(*backtrace, "Register %s: %s\n", reg_entry->reg_name,
-                                       reg_entry->reg_value ? "Non-zero" : "Zero");
-                #endif
+                // Global override for privacy filters
+                if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                        g_string_append_printf(*backtrace, "Register %s: %" PRIx64 "\n", reg_entry->reg_name,
+                                               reg_entry->reg_value);
+                } else {
+                        g_string_append_printf(*backtrace, "Register %s: %s\n", reg_entry->reg_name,
+                                               reg_entry->reg_value ? "Non-zero" : "Zero");
+                }
                 reg_entry = reg_entry->next;
         }
 

src/probes/probe.h

@@ -0,0 +1,26 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2017 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#pragma once
+
+/**
+ * Existence of the opt-in-no-privacy-filters file disables certain privacy
+ * filters enforced in telemetry probes.
+ */
+#define TM_PRIVACY_FILTERS_OVERRIDE "/etc/telemetrics/opt-in-no-privacy-filters"
+
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */