Release v1.10.2

This release improves reliability of both the crash and oops probes.

* The crash probe will now reprocess a core file if it encounters
  missing symbols when creating backtraces the first time. This
  improves telemetry when run on Clear Linux OS, which downloads debuginfo
  on-the-fly as needed.

* The oops parsing will now capture "unreliable" stack frames for kernel
  oopses using the same notation as the kernel ("? "). Previously, the
  parsing code ignored the unreliable flag, so those frames appeared to be
  trustworthy frames.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

Makefile.am

@@ -14,6 +14,7 @@ DISTCHECK_CONFIGURE_FLAGS = \
 
 include $(top_srcdir)/build-aux/make/cflags.make
 include $(top_srcdir)/build-aux/make/ldflags.make
+include $(top_srcdir)/docs/local.mk
 include $(top_srcdir)/scripts/local.mk
 include $(top_srcdir)/src/local.mk
 include $(top_srcdir)/src/data/local.mk

README

@@ -30,9 +30,3 @@ Build dependencies
 - (optional) libsystemd, for syslog-style logging to the systemd journal, and
   socket/path activation of telemd by systemd.
 
-
-Generating HTML documentation
-=============================
-
-At toplevel, run 'doxygen docs/Doxyfile'. The documentation is generated in
-the docs/html/ directory.

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([telemetrics-client], [1.9.0], [https://clearlinux.org/])
+AC_INIT([telemetrics-client], [1.10.2], [https://clearlinux.org/])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([1.14 -Wall -Werror -Wno-extra-portability foreign subdir-objects])
 AM_SILENT_RULES([yes])
@@ -137,3 +137,27 @@ AC_CONFIG_COMMANDS([config dir],[mkdir -p src/data; touch src/data/.dirstamp])
 AC_CONFIG_FILES([Makefile])
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_OUTPUT
+
+AC_MSG_NOTICE([
+
+telemetrics-client $VERSION
+===========================
+
+prefix:                 $prefix
+exec_prefix:            $exec_prefix
+libdir:                 $libdir
+bindir:                 $bindir
+datarootdir:            $datarootdir
+
+compiler:               $CC
+cflags:                 $CFLAGS
+ldflags:                $LDFLAGS
+
+unitdir:                $unitpath
+tmpfilesdir:            $tmpfilespath
+sysctldir:              $sysctlpath
+systemconfdir:          $confpath
+socketdir:              $socketpath
+loglevel:               $loglevel
+logtype:                $logtype
+])

docs/README.md

@@ -0,0 +1,13 @@
+
+## Notes on manual pages
+
+The manual pages are generated using `ronn(1)`. To recreate them,
+run `make manpages` in the toplevel  folder. If you want to edit
+the documentation for upstream changes, make sure to edit the `*.md`
+files and not the shipped nroff output files.
+
+
+## Generating HTML documentation
+
+At toplevel, run 'doxygen docs/Doxyfile'. The documentation is
+generated in the docs/html/ directory.

docs/local.mk

@@ -0,0 +1,23 @@
+MANPAGES = \
+	docs/man/telemctl.1 \
+	docs/man/telemd.1 \
+	docs/man/telem-record-gen.1 \
+	docs/man/telemetry.3 \
+	docs/man/telemetrics.conf.5
+
+MANLINKS = \
+	docs/man/tm_create_record.3 \
+	docs/man/tm_send_record.3 \
+	docs/man/tm_set_config_file.3 \
+	docs/man/tm_set_payload.3
+
+manpages:
+	for MANPAGE in $(MANPAGES); do \
+		ronn --roff < $${MANPAGE}.md > $${MANPAGE}; \
+		ronn --html < $${MANPAGE}.md > $${MANPAGE}.html; \
+	done
+
+dist_man_MANS = \
+	$(MANPAGES) $(MANLINKS)
+
+# vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

docs/man/.gitignore

@@ -0,0 +1 @@
+*.html

docs/man/telem-record-gen.1

@@ -0,0 +1,81 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "TELEM\-RECORD\-GEN" "1" "December 2016" "" ""
+.
+.SH "NAME"
+\fBtelem\-record\-gen\fR \- Create custom telemetry record
+.
+.SH "SYNOPSIS"
+\fBtelem\-record\-gen\fR \fIoptions\fR
+.
+.SH "DESCRIPTION"
+Generate and attempt to send a telemetry record to \fBtelemd(1)\fR\. This program can be used to create a telemetry record from a shell script or from another program using e\.g\. \fBsystem()\fR\. The program connects to a local \fBtelemd(1)\fR daemon to spool the record for delivery to the actual collection service\.
+.
+.P
+The payload can be provided as standard input, as a file with the \fB\-\-payload\-file\fR option, or as a string argument with the \fB\-\-payload\fR option\. If neither the \fB\-\-payload\fR or \fB\-\-payload\-file\fR is provided, the payload is assumed to be on \fIstdin\fR\.
+.
+.P
+Each record is expected to have a class identifier, a record version number, and a severity level\. A class identifier is required to be provided\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Show help options\.
+.
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print the program version\.
+.
+.TP
+\fB\-f\fR, \fB\-\-config\-file\fR \fIfile\fR
+Path to configuration file (not implemented yet)\.
+.
+.TP
+\fB\-s\fR, \fB\-\-severity\fR
+Severity level (1\-4) \- (default 1)\.
+.
+.TP
+\fB\-c\fR, \fB\-\-class\fR
+Classification \fBlevel_1\fR/\fBlevel_2\fR/\fBlevel_3\fR\.
+.
+.TP
+\fB\-p\fR, \fB\-\-payload\fR \fIpayload\fR
+Record body (max size = 8k)\.
+.
+.TP
+\fB\-P\fR, \fB\-\-payload\-file\fR \fIfile\fR
+File to read payload from\.
+.
+.TP
+\fB\-R\fR, \fB\-\-record\-version\fR \fIversion\fR
+Version number for format of payload (default 1)\.
+.
+.SH "RETURN VALUES"
+0 on success\. A non\-zero exit code indicates a failure occurred\.
+.
+.SH "COPYRIGHT"
+.
+.IP "\(bu" 4
+Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
+.
+.IP "" 0
+.
+.SH "SEE ALSO"
+\fBtelemd(1)\fR
+.
+.P
+https://github\.com/clearlinux/telemetrics\-client
+.
+.P
+https://clearlinux\.org/documentation/
+.
+.SH "NOTES"
+Creative Commons Attribution\-ShareAlike 3\.0 Unported
+.
+.IP "\(bu" 4
+http://creativecommons\.org/licenses/by\-sa/3\.0/
+.
+.IP "" 0
+

docs/man/telem-record-gen.1.md

@@ -0,0 +1,70 @@
+telem-record-gen(1) -- Create custom telemetry record
+=====================================================
+
+## SYNOPSIS
+
+`telem-record-gen` <options>
+
+## DESCRIPTION
+
+Generate and attempt to send a telemetry record to `telemd(1)`. This
+program can be used to create a telemetry record from a shell script
+or from another program using e.g. `system()`. The program connects
+to a local `telemd(1)` daemon to spool the record for delivery to
+the actual collection service.
+
+The payload can be provided as standard input, as a file with the
+`--payload-file` option, or as a string argument with the `--payload`
+option. If neither the `--payload` or `--payload-file` is provided,
+the payload is assumed to be on <stdin>.
+
+Each record is expected to have a class identifier, a record version
+number, and a severity level. A class identifier is required to be
+provided.
+
+## OPTIONS
+ * `-h`, `--help`:
+   Show help options.
+
+ * `-V`, `--version`:
+   Print the program version.
+
+ * `-f`, `--config-file` <file>:
+   Path to configuration file (not implemented yet).
+
+ * `-s`, `--severity`:
+   Severity level (1-4) - (default 1).
+
+ * `-c`, `--class`:
+   Classification `level_1`/`level_2`/`level_3`.
+
+ * `-p`, `--payload` <payload>:
+   Record body (max size = 8k).
+
+ * `-P`, `--payload-file` <file>:
+   File to read payload from.
+
+ * `-R`, `--record-version` <version>:
+   Version number for format of payload (default 1).
+
+## RETURN VALUES
+
+0 on success. A non-zero exit code indicates a failure occurred.
+
+## COPYRIGHT
+
+ * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
+
+## SEE ALSO
+
+`telemd(1)`
+
+https://github.com/clearlinux/telemetrics-client
+
+https://clearlinux.org/documentation/
+
+## NOTES
+
+Creative Commons Attribution-ShareAlike 3.0 Unported
+
+ * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemctl.1

@@ -0,0 +1,58 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "TELEMCTL" "1" "December 2016" "" ""
+.
+.SH "NAME"
+\fBtelemctl\fR \- Telemetry service administration tool
+.
+.SH "SYNOPSIS"
+\fBtelemctl\fR
+.
+.P
+\fB/etc/telemetrics/opt\-out\fR
+.
+.SH "DESCRIPTION"
+Control actions for telemetry services\. The command can be used to start, restart, or stop \fBtelemd(1)\fR, or to opt\-in or opt\-out of telemetry delivery of records to a central telemetry service\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fBstart\fR|\fBstop\fR|\fBrestart\fR
+Starts, stops or restarts all running telemetry services\.
+.
+.TP
+\fBopt\-in\fR
+Opts in to telemetry, and starts telemetry services\. The opt\-out file \fB/etc/telemetrics/opt\-out\fR is removed\.
+.
+.TP
+\fBopt\-out\fR
+Opts out of telemetry, and stops telemetry services\. The opt\-out file \fB/etc/telemetrics/opt\-out\fR is created\.
+.
+.SH "RETURN VALUES"
+0 on success\. A non\-zero exit code indicates a failure occurred\.
+.
+.SH "COPYRIGHT"
+.
+.IP "\(bu" 4
+Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
+.
+.IP "" 0
+.
+.SH "SEE ALSO"
+\fBtelemd(1)\fR
+.
+.P
+https://github\.com/clearlinux/telemetrics\-client
+.
+.P
+https://clearlinux\.org/documentation/
+.
+.SH "NOTES"
+Creative Commons Attribution\-ShareAlike 3\.0 Unported
+.
+.IP "\(bu" 4
+http://creativecommons\.org/licenses/by\-sa/3\.0/
+.
+.IP "" 0
+

docs/man/telemctl.1.md

@@ -0,0 +1,49 @@
+telemctl(1) -- Telemetry service administration tool
+====================================================
+
+## SYNOPSIS
+
+`telemctl`
+
+`/etc/telemetrics/opt-out`
+
+## DESCRIPTION
+
+Control actions for telemetry services. The command can be used to start,
+restart, or stop `telemd(1)`, or to opt-in or opt-out of telemetry delivery
+of records to a central telemetry service.
+
+## OPTIONS
+
+ * `start`|`stop`|`restart`:
+   Starts, stops or restarts all running telemetry services.
+
+ * `opt-in`: 
+   Opts in to telemetry, and starts telemetry services. The opt-out file
+   `/etc/telemetrics/opt-out` is removed.
+
+ * `opt-out`: 
+   Opts out of telemetry, and stops telemetry services. The opt-out file
+   `/etc/telemetrics/opt-out` is created.
+
+## RETURN VALUES
+
+0 on success. A non-zero exit code indicates a failure occurred.
+
+## COPYRIGHT
+
+ * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
+
+## SEE ALSO
+
+`telemd(1)`
+
+https://github.com/clearlinux/telemetrics-client
+
+https://clearlinux.org/documentation/
+
+## NOTES
+
+Creative Commons Attribution-ShareAlike 3.0 Unported
+
+ * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemd.1

@@ -0,0 +1,55 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "TELEMD" "1" "December 2016" "" ""
+.
+.SH "NAME"
+\fBtelemd\fR \- Telemetry client service
+.
+.SH "SYNOPSIS"
+\fBtelemd\fR \fIflags\fR \fB/etc/telemetrics/telemetrics\.conf\fR
+.
+.SH "DESCRIPTION"
+The \fBtelemd\fR program delivers locally generated telemetry records to a remote telemetry service\. Telemetry data can be in any format, and is relayed as\-is\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-f\fR, \fB\-\-config_file\fR [\fIfile\fR]
+Configuration file\. This overides the other parameters\.
+.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Display this help message\.
+.
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print the program version\.
+.
+.SH "EXIT STATUS"
+0 when no errors occurred\. A non\-zero exit status indicates a failure occurred\.
+.
+.SH "COPYRIGHT"
+.
+.IP "\(bu" 4
+Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
+.
+.IP "" 0
+.
+.SH "SEE ALSO"
+\fBtelemetry(3)\fR \fBtelemetrics\-client\.conf(5)\fR
+.
+.P
+https://github\.com/clearlinux/telemetrics\-client
+.
+.P
+https://clearlinux\.org/documentation/
+.
+.SH "NOTES"
+Creative Commons Attribution\-ShareAlike 3\.0 Unported
+.
+.IP "\(bu" 4
+http://creativecommons\.org/licenses/by\-sa/3\.0/
+.
+.IP "" 0
+

docs/man/telemd.1.md

@@ -0,0 +1,47 @@
+telemd(1) -- Telemetry client service
+=====================================
+
+## SYNOPSIS
+
+`telemd` <flags> 
+`/etc/telemetrics/telemetrics.conf`
+
+## DESCRIPTION
+
+The `telemd` program delivers locally generated telemetry records to a remote
+telemetry service. Telemetry data can be in any format, and is relayed as-is.
+
+## OPTIONS
+
+  * `-f`, `--config_file` [<file>]:
+    Configuration file. This overides the other parameters.
+
+  * `-h`, `--help`:
+    Display this help message.
+
+  * `-V`, `--version`:
+    Print the program version.
+
+
+## EXIT STATUS
+
+0 when no errors occurred. A non-zero exit status indicates a failure occurred.
+
+## COPYRIGHT
+
+ * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
+
+## SEE ALSO
+
+`telemetry(3)`
+`telemetrics-client.conf(5)`
+
+https://github.com/clearlinux/telemetrics-client
+
+https://clearlinux.org/documentation/
+
+## NOTES
+
+Creative Commons Attribution-ShareAlike 3.0 Unported
+
+ * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemetrics.conf.5

@@ -0,0 +1,105 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "TELEMETRICS\.CONF" "5" "December 2016" "" ""
+.
+.SH "NAME"
+\fBtelemetrics\.conf\fR \- Telemetry service configuration file
+.
+.SH "SYNOPSIS"
+.
+.nf
+
+/etc/telemetrics/telemetrics\.conf
+/usr/share/defaults/telemetrics/telemetrics\.conf
+.
+.fi
+.
+.SH "DESCRIPTION"
+This file contains configuration parameters for the \fBtelemd(1)\fR telemetry service daemon\. The daemon reads this file at startup if it exists\.
+.
+.SH "SYNTAX"
+The configuration file contains \fBkey=value\fR pairs, formatted as plain text, one option per line\. Comments can be added by preceding them with the \fB#\fR character\. All configuration options should be in a section marked with \fB[settings]\fR\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fBserver\fR=\fIurl\fR
+Server URL including protocol designator\.
+.
+.TP
+\fBsocket_path\fR=\fIpath\fR
+Path to the socket that \fBtelemd\fR will listen on\.
+.
+.TP
+\fBcainfo\fR=\fIpath\fR
+Certificate file to use for validation of SSL endpoint\.
+.
+.TP
+\fBtidheader\fR=\fIheader\fR
+Telemetry ID in header format, usually \fBtidheader=X\-Telemetry\-TID:<uuid>\fR post header used to group records in ingestion service, which may be ingesting for more than one set of clients\. Can be set to any string\.
+.
+.TP
+\fBrecord_expiry\fR=\fIminutes\fR
+Record expiry time in minutes\.
+.
+.TP
+\fBspool_dir\fR=\fI\fBdir\fR\fR
+Local spool directory used to store records being processed\.
+.
+.TP
+\fBspool_max_size\fR=\fIkB\fR
+maximum size of the spool directory in kB\. A value of \fB\-1\fR indicates no limit\. The block size of the files in this directory is considered, and not the actual size of the record itself\.
+.
+.TP
+\fBspool_process_time\fR=\fIseconds\fR
+Time in seconds for processing spool\. Valid range: 120\.\.3600\. Values outside this range are clamped\.
+.
+.TP
+\fBrate_limit_enabled\fR=\fI\fBtrue\fR|\fBfalse\fR\fR
+Enable rate limiting\. If this is set to false then all rate\-limiting disabled\. It is possible to disable each rate\-limit individually below\.
+.
+.TP
+\fBrecord_burst_limit\fR=\fIlimit\fR
+Rate limiting record burst limit\. Valid Range: 0\.\.\fBINT_MAX\fR, \-1 = disabled\.
+.
+.TP
+\fBrecord_window_length\fR=\fIminutes\fR
+Rate limiting record window length in minutes\. Valid Range: 0\.\.59\.
+.
+.TP
+\fBbyte_burst_limit\fR=\fIlimit\fR
+Rate limiting byte burst limit\. Valid Range: 0\.\.\fBINT_MAX\fR, \-1 = disabled\.
+.
+.TP
+\fBbyte_window_length\fR=\fIminutes\fR
+Rate limiting byte window length in minutes\. Valid Range: 0\.\.59\.
+.
+.TP
+\fBrate_limit_strategy\fR=\fIstrategy\fR
+Rate limit strategy \- what to do with record if rate\-limiting prevents delivery over network\. Valid stategies: \fBspool\fR, \fBdrop\fR\.
+.
+.SH "COPYRIGHT"
+.
+.IP "\(bu" 4
+Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
+.
+.IP "" 0
+.
+.SH "SEE ALSO"
+\fBtelemd(1)\fR
+.
+.P
+https://github\.com/clearlinux/telemetrics\-client
+.
+.P
+https://clearlinux\.org/documentation/
+.
+.SH "NOTES"
+Creative Commons Attribution\-ShareAlike 3\.0 Unported
+.
+.IP "\(bu" 4
+http://creativecommons\.org/licenses/by\-sa/3\.0/
+.
+.IP "" 0
+

docs/man/telemetrics.conf.5.md

@@ -0,0 +1,92 @@
+telemetrics.conf(5) -- Telemetry service configuration file
+===========================================================
+
+## SYNOPSIS
+
+```
+/etc/telemetrics/telemetrics.conf
+/usr/share/defaults/telemetrics/telemetrics.conf
+```
+
+## DESCRIPTION
+
+This file contains configuration parameters for the `telemd(1)` telemetry
+service daemon. The daemon reads this file at startup if it exists.
+
+## SYNTAX
+
+The configuration file contains `key=value` pairs, formatted as plain
+text, one option per line. Comments can be added by preceding them with the
+`#` character. All configuration options should be in a section marked
+with `[settings]`.
+
+## OPTIONS
+
+ * `server`=<url>:
+   Server URL including protocol designator.
+
+ * `socket_path`=<path>:
+   Path to the socket that `telemd` will listen on.
+
+ * `cainfo`=<path>:
+  Certificate file to use for validation of SSL endpoint.
+
+ * `tidheader`=<header>:
+  Telemetry ID in header format, usually
+    `tidheader=X-Telemetry-TID:<uuid>`
+  post header used to group records in ingestion service, which may be
+  ingesting for more than one set of clients. Can be set to any string.
+
+ * `record_expiry`=<minutes>:
+  Record expiry time in minutes.
+
+ * `spool_dir`=<`dir`>:
+  Local spool directory used to store records being processed.
+
+ * `spool_max_size`=<kB>:
+  maximum size of the spool directory in kB. A value of `-1` indicates
+  no limit. The block size of the files in this directory is considered,
+  and not the actual size of the record itself.
+
+ * `spool_process_time`=<seconds>:
+  Time in seconds for processing spool. Valid range: 120..3600. Values
+  outside this range are clamped.
+
+ * `rate_limit_enabled`=<`true`|`false`>:
+  Enable rate limiting. If this is set to false then all rate-limiting
+  disabled. It is possible to disable each rate-limit individually below.
+
+ * `record_burst_limit`=<limit>:
+  Rate limiting record burst limit. Valid Range:  0..`INT_MAX`, -1 = disabled.
+
+ * `record_window_length`=<minutes>:
+  Rate limiting record window length in minutes. Valid Range: 0..59.
+
+ * `byte_burst_limit`=<limit>:
+  Rate limiting byte burst limit. Valid Range:  0..`INT_MAX`, -1 = disabled.
+
+ * `byte_window_length`=<minutes>:
+  Rate limiting byte window length in minutes. Valid Range: 0..59.
+
+ * `rate_limit_strategy`=<strategy>:
+  Rate limit strategy - what to do with record if rate-limiting prevents
+  delivery over network. Valid stategies: `spool`, `drop`.
+
+
+## COPYRIGHT
+
+ * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
+
+## SEE ALSO
+
+`telemd(1)`
+
+https://github.com/clearlinux/telemetrics-client
+
+https://clearlinux.org/documentation/
+
+## NOTES
+
+Creative Commons Attribution-ShareAlike 3.0 Unported
+
+ * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemetry.3

@@ -1,15 +1,69 @@
-.TH TELEMETRY 3 "April 2015" Telemetrics
-.SH NAME
-telemetry \- Telemetry record creation and sending
-.SH SYNOPSIS
-#include <telemetry.h>
-.sp
-tm_create_record()
-.br
-tm_set_payload()
-.br
-tm_send_record()
-.br
-tm_set_config_File()
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "TELEMETRY" "3" "December 2016" "" ""
+.
+.SH "NAME"
+\fBtelemetry\fR \- C programming interface for telemetrics\-client
+.
+.SH "SYNOPSIS"
+.
+.nf
 
+#include "telemetry\.h"
+
+struct telem_ref {
+        struct telem_record *record;
+};
+
+int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
+                     char *classification, uint32_t payload_version)
+int tm_set_payload(struct telem_ref *t_ref, char *payload)
+int tm_send_record(struct telem_ref *t_ref)
+void tm_free_record(struct telem_ref *t_ref)
+void tm_set_config_file(char *c_file)
+.
+.fi
+.
+.SH "DESCRIPTION"
+The functions in the telemetry library facilitate the delivery of telemetry data to the \fBtelemd(1)\fR service\.
+.
+.P
+The function \fBtm_create_record()\fR initializes a telemetry record and sets the severity and classification of that record, as well as the payload version number\. The memory needed to store the telemetry record is allocated and should be freed with \fBtm_free_record()\fR when no longer needed\.
+.
+.P
+The function \fBtm_set_payload()\fR attaches the provided telemetry record data to the telemetry record\. The current maximum payload size is 8192b\.
+.
+.P
+The function \fBtm_send_record()\fR delivers the record to the local \fBtelemd(1)\fR service\.
+.
+.P
+The function \fBtm_set_config_file()\fR can be used to provide an alternate configuration path to the telemetry library\.
+.
+.SH "RETURN VALUES"
+All these functions return \fB0\fR on success, or a non\-zero return value if an error occurred\. The functions \fBtm_free_record()\fR and \fBtm_set_config_file()\fR do not return any values\.
+.
+.SH "COPYRIGHT"
+.
+.IP "\(bu" 4
+Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
+.
+.IP "" 0
+.
+.SH "SEE ALSO"
+\fBtelemd(1)\fR
+.
+.P
+https://github\.com/clearlinux/telemetrics\-client
+.
+.P
+https://clearlinux\.org/documentation/
+.
+.SH "NOTES"
+Creative Commons Attribution\-ShareAlike 3\.0 Unported
+.
+.IP "\(bu" 4
+http://creativecommons\.org/licenses/by\-sa/3\.0/
+.
+.IP "" 0
 

docs/man/telemetry.3.md

@@ -0,0 +1,63 @@
+telemetry(3) -- C programming interface for telemetrics-client
+==============================================================
+
+## SYNOPSIS
+
+```
+#include "telemetry.h"
+
+struct telem_ref {
+        struct telem_record *record;
+};
+
+int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
+                     char *classification, uint32_t payload_version)
+int tm_set_payload(struct telem_ref *t_ref, char *payload)
+int tm_send_record(struct telem_ref *t_ref)
+void tm_free_record(struct telem_ref *t_ref)
+void tm_set_config_file(char *c_file)
+```
+
+## DESCRIPTION
+
+The functions in the telemetry library facilitate the delivery of
+telemetry data to the `telemd(1)` service.
+
+The function `tm_create_record()` initializes a telemetry record and
+sets the severity and classification of that record, as well as the
+payload version number. The memory needed to store the telemetry record
+is allocated and should be freed with `tm_free_record()` when no longer
+needed.
+
+The function `tm_set_payload()` attaches the provided telemetry record
+data to the telemetry record. The current maximum payload size is 8192b.
+
+The function `tm_send_record()` delivers the record to the local
+`telemd(1)` service.
+
+The function `tm_set_config_file()` can be used to provide an alternate
+configuration path to the telemetry library.
+
+## RETURN VALUES
+
+All these functions return `0` on success, or a non-zero return value
+if an error occurred. The functions `tm_free_record()` and `tm_set_config_file()`
+do not return any values.
+
+## COPYRIGHT
+
+ * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
+
+## SEE ALSO
+
+`telemd(1)`
+
+https://github.com/clearlinux/telemetrics-client
+
+https://clearlinux.org/documentation/
+
+## NOTES
+
+Creative Commons Attribution-ShareAlike 3.0 Unported
+
+ * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/tm_create_record.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

docs/man/tm_send_record.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

docs/man/tm_set_config_file.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

docs/man/tm_set_payload.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

src/configuration.c

@@ -44,7 +44,8 @@ const char *config_key_int[] = { NULL, "record_expiry", "spool_max_size",
                                  "byte_window_length", "record_burst_limit",
                                  "byte_burst_limit", NULL };
 
-const char *config_key_bool[] = { NULL, "rate_limit_enabled", NULL };
+const char *config_key_bool[] = { NULL, "rate_limit_enabled",
+				  "daemon_recycling_enabled", NULL };
 
 static struct configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
 
@@ -334,4 +335,10 @@ const char *rate_limit_strategy_config()
 
         return val;
 }
+
+bool daemon_recycling_enabled_config(void)
+{
+	initialize_config();
+	return config.boolValues[CONF_DAEMON_RECYCLING_ENABLED];
+}
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/configuration.h

@@ -46,6 +46,7 @@ enum config_int_keys {
 enum config_bool_keys {
         CONF_BOOL_MIN = 0,
         CONF_RATE_LIMIT_ENABLED,
+	CONF_DAEMON_RECYCLING_ENABLED,
         CONF_BOOL_MAX
 };
 
@@ -113,4 +114,7 @@ const char *get_cainfo_config(void);
 /* Gets tidheader */
 const char *get_tidheader_config(void);
 
+/* Gets whether recycling is enabled */
+bool daemon_recycling_enabled_config(void);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/data/example.conf

@@ -35,3 +35,6 @@ rate_limit_enabled=true
 
 #rate limit strategy if record not send
 rate_limit_strategy=spool
+
+#daemon recycling enabled
+daemon_recycling_enabled=true

src/data/telemetrics.conf.in

@@ -51,3 +51,7 @@ byte_window_length=20
 # Valid stategies: spool, drop
 rate_limit_strategy=spool
 
+# daemon recycling enabled - if daemon has been running for a while (2 hours),
+# has not any client nor spool data, then it exits.
+# this is to ensure that latest code runs.
+daemon_recycling_enabled=true

src/log.h

@@ -20,6 +20,7 @@
 
 #include "config.h"
 
+#ifdef TM_LOG_SYSTEMD
 #ifdef HAVE_SYSTEMD_SD_JOURNAL_H
 
 /* Certain static analysis tools do not understand GCC's __INCLUDE_LEVEL__
@@ -32,6 +33,7 @@
 #endif
 
 #include <systemd/sd-journal.h>
+#endif
 #else
 #include <errno.h>
 #include <string.h>
@@ -88,10 +90,10 @@
 #define telem_perror(msg) sd_journal_perror(msg)
 #endif
 #ifdef TM_LOG_SYSLOG
-#define telem_perror(msg) syslog(LOG_ERR, msg ": %s", strerror(errno))
+#define telem_perror(msg) syslog(LOG_ERR, msg ": %s\n", strerror(errno))
 #endif
 #ifdef TM_LOG_STDERR
-#define telem_perror(msg) fprintf(stderr, ": %s", msg)
+#define telem_perror(msg) fprintf(stderr, "ERROR: " msg ": %s\n", strerror(errno))
 #endif
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/main.c

@@ -232,6 +232,8 @@ int main(int argc, char **argv)
         time_t last_spool_run_time = time(NULL);
         time_t last_daemon_start_time = time(NULL);
 
+	bool daemon_recycling_enabled = daemon_recycling_enabled_config();
+
         ret = update_machine_id();
         if (ret == -1) {
                 telem_log(LOG_ERR, "Unable to update machine id\n");
@@ -336,7 +338,7 @@ int main(int argc, char **argv)
                 } else {
                         time_t now = time(NULL);
                         /* If spool is empty and time to recycle the daemon has elapsed*/
-                        if ((daemon.current_spool_size == 0) &&
+			if (daemon_recycling_enabled && (daemon.current_spool_size == 0) &&
                             difftime(now, last_daemon_start_time) >= TM_DAEMON_EXIT_TIME) {
                                 /* Exit */
                                 telem_log(LOG_INFO, "Daemon exiting for recycling\n");

src/probes/crash_probe.c

@@ -176,6 +176,55 @@ static int temp_core_file(void)
         return tmp;
 }
 
+/* There are a number of initialization routines required to initialize the Elf
+ * and Dwfl objects used by libdwfl to later process a core file. The argument
+ * e_core is the address of an Elf pointer declared by the caller, and core_fd
+ * is the open file descriptor for the core file.
+ */
+static int prepare_corefile(Elf **e_core, int core_fd)
+{
+        // Cleanup previous corefile processing if needed
+        if (d_core) {
+                dwfl_end(d_core);
+        }
+        if (*e_core) {
+                elf_end(*e_core);
+        }
+
+        // Boilerplate initialization for elfutils (libdwfl)
+        if (!(*e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
+                tm_elf_err("Failed to get file descriptor for ELF core file");
+                goto fail;
+        }
+
+        if (!(d_core = dwfl_begin(&cb))) {
+                tm_dwfl_err("Failed to start new libdwfl session");
+                goto fail;
+        }
+
+        int core_modules = 0;
+        core_modules = dwfl_core_file_report(d_core, *e_core, NULL);
+        if (core_modules == -1) {
+                tm_dwfl_err("Failed to report modules for ELF core file");
+                goto fail;
+        }
+
+        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
+                tm_dwfl_err("Failed to finish reporting modules");
+                goto fail;
+        }
+
+        if ((core_for_pid = dwfl_core_file_attach(d_core, *e_core)) < 0) {
+                tm_dwfl_err("Failed to prepare libdwfl session for thread"
+                            " iteration");
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
 /* This callback is invoked for every frame in a thread. From a Dwfl_Frame, we
  * are able to extract the program counter (PC), and from that, the procedure
  * name via a Dwfl_Module.
@@ -305,6 +354,41 @@ fail:
         return false;
 }
 
+/* This is the entry point for libdwfl to unwind the backtrace from the core
+ * file. All data necessary for processing is referenced by d_core (a *Dwfl),
+ * which must be initialized prior to calling this function.  The callback
+ * function for processing each core thread is passed as the second argument to
+ * dwfl_getthreads(). The backtrace argument is the address of a pointer to a
+ * GString object declared by the caller which stores the backtrace data as a
+ * string.
+ */
+static int process_corefile(GString **backtrace)
+{
+        if (*backtrace) {
+                g_string_free(*backtrace, TRUE);
+        }
+        *backtrace = g_string_new(NULL);
+
+        if (dwfl_getthreads(d_core, thread_cb, backtrace) != DWARF_CB_OK) {
+                /* When errors occur during the unwinding, we reach this point.
+                 * If an error string is set for the particular error, send an
+                 * "error" record to capture at least a partial backtrace if
+                 * some frames were processed.
+                 */
+                if (errorstr != NULL) {
+                        telem_log(LOG_ERR, "%s", errorstr);
+                        g_string_append_printf(header, "Error: %s", errorstr);
+                        g_string_prepend(*backtrace, header->str);
+                        send_data(backtrace, error_severity, error_class);
+                }
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
 static bool in_clr_build(gchar *fullpath)
 {
         /*
@@ -373,13 +457,20 @@ static void free_glib_strings(void)
 int main(int argc, char **argv)
 {
         Elf *e_core = NULL;
-        int core_modules = 0;
         int ret = EXIT_FAILURE;
         int core_fd = STDIN_FILENO;
         GString *backtrace = NULL;
         GError *error = NULL;
         GOptionContext *context;
 
+        if (fcntl(STDERR_FILENO, F_GETFL) < 0) {
+                // redirect stderr to avoid bad things to happen with
+                // fd 2 when launched from kernel core pattern
+                if (freopen("/dev/null", "w", stderr) == NULL) {
+                        exit(EXIT_FAILURE);
+                }
+        }
+
         // Since this program handles core files, make sure it does not produce
         // core files itself, or else it will be invoked endlessly...
         prctl(PR_SET_DUMPABLE, 0);
@@ -460,30 +551,7 @@ int main(int argc, char **argv)
 
         elf_version(EV_CURRENT);
 
-        if (!(e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
-                tm_elf_err("Failed to get file descriptor for ELF core file");
-                goto fail;
-        }
-
-        if (!(d_core = dwfl_begin(&cb))) {
-                tm_dwfl_err("Failed to start new libdwfl session");
-                goto fail;
-        }
-
-        core_modules = dwfl_core_file_report(d_core, e_core, NULL);
-        if (core_modules == -1) {
-                tm_dwfl_err("Failed to report modules for ELF core file");
-                goto fail;
-        }
-
-        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
-                tm_dwfl_err("Failed to finish reporting modules");
-                goto fail;
-        }
-
-        if ((core_for_pid = dwfl_core_file_attach(d_core, e_core)) < 0) {
-                tm_dwfl_err("Failed to prepare libdwfl session for thread"
-                            " iteration");
+        if (prepare_corefile(&e_core, core_fd) < 0) {
                 goto fail;
         }
 
@@ -496,17 +564,27 @@ int main(int argc, char **argv)
                 g_string_append_printf(header, "Signal: %d\n", signal_num);
         }
 
-        backtrace = g_string_new(NULL);
-        if (dwfl_getthreads(d_core, thread_cb, &backtrace) != DWARF_CB_OK) {
-                if (errorstr != NULL) {
-                        telem_log(LOG_ERR, "%s", errorstr);
-                        g_string_append_printf(header, "Error: %s", errorstr);
-                        g_string_prepend(backtrace, header->str);
-                        send_data(&backtrace, error_severity, error_class);
-                }
+        if (process_corefile(&backtrace) < 0) {
                 goto fail;
         }
 
+        /* On Clear Linux OS, missing symbols may appear if automatic debuginfo
+         * downloads are still in flight. So if any missing symbols appear on
+         * the first run (indicated by the presence of "??? - ["), wait 10
+         * seconds and try again.
+         */
+        if (strstr(backtrace->str, "??? - [")) {
+                sleep(10);
+
+                if (prepare_corefile(&e_core, core_fd) < 0) {
+                        goto fail;
+                }
+
+                if (process_corefile(&backtrace) < 0) {
+                        goto fail;
+                }
+        }
+
         g_string_prepend(backtrace, header->str);
 
         if (!send_data(&backtrace, default_severity, clr_class)) {

src/probes/oops_parser.c

@@ -477,13 +477,6 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
                 start++;
         }
 
-        if (*start && *start == '?') {
-                start++;
-                if (*start && isspace(*start)) {
-                        start++;
-                }
-        }
-
         offset_ptr = strchr(start, '+');
         end = offset_ptr ? offset_ptr : (start + strlen(start));
         frame->function = strndup(start, (size_t)(end - start));

src/probes/telem_record_gen.c

@@ -56,6 +56,20 @@ static GOptionEntry options[] = {
         { NULL }
 };
 
+const unsigned int count_chars(const char *check, const char character)
+{
+        unsigned int count = 0U;
+        if (check == NULL) {
+                return count;
+        }
+        for (; *check != '\0'; ++check) {
+                if (*check == character) {
+                        ++count;
+                }
+        }
+        return count;
+}
+
 static void free_glib_strings(void)
 {
         if (config_file) {
@@ -122,6 +136,12 @@ int validate_opts(void)
                 return ret;
         }
 
+        if (count_chars(opt_class, '/') != 2) {
+                fprintf(stderr, "Error: Classification needs to be in "
+                        "most/to/least specific format, 2 \'/\' required.\n");
+                return ret;
+        }
+
         /* Severity */
         if ((severity) < 1 || (severity > 4)) {
                 fprintf(stderr, "Error: Valid range for severity is 1-4\n");

tests/check_config.c

@@ -50,6 +50,7 @@ START_TEST(check_read_valid_config)
         ck_assert_str_eq(config.strValues[CONF_CAINFO], "/tmp/cacert.crt");
         ck_assert_str_eq(config.strValues[CONF_TIDHEADER],
                          "X-Telemetry-TID: 6907c830-eed9-4ce9-81ae-76daf8d88f0f");
+	ck_assert(config.boolValues[CONF_DAEMON_RECYCLING_ENABLED] == true);
 
 }
 END_TEST
@@ -75,6 +76,7 @@ START_TEST(check_config_initialised)
         ck_assert_str_eq(get_cainfo_config(), "/tmp/cacert.crt");
         ck_assert_str_eq(get_tidheader_config(),
                          "X-Telemetry-TID: 6907c830-eed9-4ce9-81ae-76daf8d88f0f");
+	ck_assert(daemon_recycling_enabled_config() == true);
 }
 END_TEST
 

tests/check_probes.c

@@ -100,23 +100,23 @@ START_TEST(watchdog_payload)
         ck_assert(strstr(pl->str, "#1 dump_stack"));
         ck_assert(strstr(pl->str, "#2 warn_slowpath_common"));
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
-        ck_assert(strstr(pl->str, "#4 __queue_work"));
+        ck_assert(strstr(pl->str, "#4 ? __queue_work"));
         ck_assert(strstr(pl->str, "#5 dev_watchdog"));
-        ck_assert(strstr(pl->str, "#6 dev_deactivate_queue.constprop.30"));
+        ck_assert(strstr(pl->str, "#6 ? dev_deactivate_queue.constprop.30"));
         ck_assert(strstr(pl->str, "#7 call_timer_fn"));
-        ck_assert(strstr(pl->str, "#8 dev_deactivate_queue.constprop.30"));
+        ck_assert(strstr(pl->str, "#8 ? dev_deactivate_queue.constprop.30"));
         ck_assert(strstr(pl->str, "#9 run_timer_softirq"));
         ck_assert(strstr(pl->str, "#10 __do_softirq"));
         ck_assert(strstr(pl->str, "#11 irq_exit"));
         ck_assert(strstr(pl->str, "#12 smp_apic_timer_interrupt"));
         ck_assert(strstr(pl->str, "#13 apic_timer_interrupt"));
-        ck_assert(strstr(pl->str, "#14 __hrtimer_start_range_ns"));
-        ck_assert(strstr(pl->str, "#15 cpuidle_enter_state"));
-        ck_assert(strstr(pl->str, "#16 cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#14 ? __hrtimer_start_range_ns"));
+        ck_assert(strstr(pl->str, "#15 ? cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#16 ? cpuidle_enter_state"));
         ck_assert(strstr(pl->str, "#17 cpuidle_idle_call"));
         ck_assert(strstr(pl->str, "#18 arch_cpu_idle"));
         ck_assert(strstr(pl->str, "#19 cpu_startup_entry"));
-        ck_assert(strstr(pl->str, "#20 clockevents_register_device"));
+        ck_assert(strstr(pl->str, "#20 ? clockevents_register_device"));
         ck_assert(strstr(pl->str, "#21 start_secondary"));
 
         g_string_free(pl, true);
@@ -141,8 +141,8 @@ START_TEST(alsa_bug_payload)
         ck_assert(strstr(pl->str, "Tainted : Not tainted"));
 
         ck_assert(strstr(pl->str, "#1 warn_on_slowpath"));
-        ck_assert(strstr(pl->str, "#2 vsnprintf"));
-        ck_assert(strstr(pl->str, "#3 _spin_unlock_irqrestore"));
+        ck_assert(strstr(pl->str, "#2 ? vsnprintf"));
+        ck_assert(strstr(pl->str, "#3 ? _spin_unlock_irqrestore"));
         ck_assert(strstr(pl->str, "#10 start_kernel"));
         g_string_free(pl, true);
 }
@@ -173,14 +173,14 @@ START_TEST(warning_payload)
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
         ck_assert(strstr(pl->str, "#4 preempt_notifier_register"));
         ck_assert(strstr(pl->str, "#5 VBoxHost_RTThreadCtxHooksRegister"));
-        ck_assert(strstr(pl->str, "#6 update_curr"));
-        ck_assert(strstr(pl->str, "#7 supdrvIOCtlFast"));
-        ck_assert(strstr(pl->str, "#8 VBoxDrvLinuxIOCtl_4_3_28"));
-        ck_assert(strstr(pl->str, "#9 put_prev_entity"));
-        ck_assert(strstr(pl->str, "#10 do_vfs_ioctl"));
-        ck_assert(strstr(pl->str, "#11 pick_next_task_rt"));
-        ck_assert(strstr(pl->str, "#12 SyS_ioctl"));
-        ck_assert(strstr(pl->str, "#13 entry_SYSCALL_64_fastpath"));
+        ck_assert(strstr(pl->str, "#6 ? update_curr"));
+        ck_assert(strstr(pl->str, "#7 ? supdrvIOCtlFast"));
+        ck_assert(strstr(pl->str, "#8 ? VBoxDrvLinuxIOCtl_4_3_28"));
+        ck_assert(strstr(pl->str, "#9 ? put_prev_entity"));
+        ck_assert(strstr(pl->str, "#10 ? do_vfs_ioctl"));
+        ck_assert(strstr(pl->str, "#11 ? pick_next_task_rt"));
+        ck_assert(strstr(pl->str, "#12 ? SyS_ioctl"));
+        ck_assert(strstr(pl->str, "#13 ? entry_SYSCALL_64_fastpath"));
 
         g_string_free(pl, true);
 }
@@ -208,16 +208,16 @@ START_TEST(warn_on_payload)
         ck_assert(strstr(pl->str, "#2 warn_slowpath_common"));
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
         ck_assert(strstr(pl->str, "#4 dev_watchdog"));
-        ck_assert(strstr(pl->str, "#5 qdisc_rcu_free"));
+        ck_assert(strstr(pl->str, "#5 ? qdisc_rcu_free"));
         ck_assert(strstr(pl->str, "#6 call_timer_fn"));
-        ck_assert(strstr(pl->str, "#7 qdisc_rcu_free"));
+        ck_assert(strstr(pl->str, "#7 ? qdisc_rcu_free"));
         ck_assert(strstr(pl->str, "#8 run_timer_softirq"));
         ck_assert(strstr(pl->str, "#9 __do_softirq"));
         ck_assert(strstr(pl->str, "#10 irq_exit"));
         ck_assert(strstr(pl->str, "#11 smp_apic_timer_interrupt"));
         ck_assert(strstr(pl->str, "#12 apic_timer_interrupt"));
-        ck_assert(strstr(pl->str, "#13 cpuidle_enter_state"));
-        ck_assert(strstr(pl->str, "#14 cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#13 ? cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#14 ? cpuidle_enter_state"));
         ck_assert(strstr(pl->str, "#15 cpuidle_enter"));
         ck_assert(strstr(pl->str, "#16 cpu_startup_entry"));
         ck_assert(strstr(pl->str, "#17 start_secondary"));
@@ -285,20 +285,20 @@ START_TEST(sysctl2_payload)
         ck_assert(strstr(pl->str, "Kernel Version : 2.6.32.53-bigmem-5-openvz #28"));
         ck_assert(strstr(pl->str, "Tainted : P"));
 
-        ck_assert(strstr(pl->str, "#1 set_fail"));
-        ck_assert(strstr(pl->str, "#2 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#3 sysctl_check_lookup"));
-        ck_assert(strstr(pl->str, "#4 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#5 sysctl_check_lookup"));
-        ck_assert(strstr(pl->str, "#6 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#7 __kmalloc"));
-        ck_assert(strstr(pl->str, "#8 __register_sysctl_paths"));
-        ck_assert(strstr(pl->str, "#9 proc_register"));
-        ck_assert(strstr(pl->str, "#10 register_net_sysctl_table"));
-        ck_assert(strstr(pl->str, "#11 nf_conntrack_net_init"));
-        ck_assert(strstr(pl->str, "#12 setup_net"));
-        ck_assert(strstr(pl->str, "#13 copy_net_ns"));
-        ck_assert(strstr(pl->str, "#14 create_new_namespaces"));
+        ck_assert(strstr(pl->str, "#1 ? set_fail"));
+        ck_assert(strstr(pl->str, "#2 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#3 ? sysctl_check_lookup"));
+        ck_assert(strstr(pl->str, "#4 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#5 ? sysctl_check_lookup"));
+        ck_assert(strstr(pl->str, "#6 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#7 ? __kmalloc"));
+        ck_assert(strstr(pl->str, "#8 ? __register_sysctl_paths"));
+        ck_assert(strstr(pl->str, "#9 ? proc_register"));
+        ck_assert(strstr(pl->str, "#10 ? register_net_sysctl_table"));
+        ck_assert(strstr(pl->str, "#11 ? nf_conntrack_net_init"));
+        ck_assert(strstr(pl->str, "#12 ? setup_net"));
+        ck_assert(strstr(pl->str, "#13 ? copy_net_ns"));
+        ck_assert(strstr(pl->str, "#14 ? create_new_namespaces"));
 
         g_string_free(pl, true);
 }
@@ -327,7 +327,7 @@ START_TEST(softlockup_payload)
         ck_assert(strstr(pl->str, "#2 btrfs_tree_read_lock"));
         ck_assert(strstr(pl->str, "#3 btrfs_read_lock_root_node"));
         ck_assert(strstr(pl->str, "#4 btrfs_search_slot"));
-        ck_assert(strstr(pl->str, "#5 crypto_shash_update"));
+        ck_assert(strstr(pl->str, "#5 ? crypto_shash_update"));
         ck_assert(strstr(pl->str, "#6 btrfs_lookup_xattr"));
         ck_assert(strstr(pl->str, "#7 __btrfs_getxattr"));
         ck_assert(strstr(pl->str, "#8 btrfs_getxattr"));
@@ -356,19 +356,19 @@ START_TEST(rtnl_payload)
 
         ck_assert(strstr(pl->str, "Tainted : Not tainted"));
 
-        ck_assert(strstr(pl->str, "#1 dump_stack"));
-        ck_assert(strstr(pl->str, "#2 bond_set_rx_mode"));
-        ck_assert(strstr(pl->str, "#3 __dev_mc_add"));
-        ck_assert(strstr(pl->str, "#4 igmp6_group_added"));
-        ck_assert(strstr(pl->str, "#5 sock_def_readable"));
-        ck_assert(strstr(pl->str, "#6 ipv6_dev_mc_inc"));
-        ck_assert(strstr(pl->str, "#7 ipv6_sock_mc_join"));
-        ck_assert(strstr(pl->str, "#8 do_ipv6_setsockopt.isra.6"));
-        ck_assert(strstr(pl->str, "#9 pipe_write"));
-        ck_assert(strstr(pl->str, "#10 SYSC_sendto"));
-        ck_assert(strstr(pl->str, "#11 ipv6_setsockopt"));
-        ck_assert(strstr(pl->str, "#12 SyS_setsockopt"));
-        ck_assert(strstr(pl->str, "#13 system_call_fastpath"));
+        ck_assert(strstr(pl->str, "#1 ? dump_stack"));
+        ck_assert(strstr(pl->str, "#2 ? bond_set_rx_mode"));
+        ck_assert(strstr(pl->str, "#3 ? __dev_mc_add"));
+        ck_assert(strstr(pl->str, "#4 ? igmp6_group_added"));
+        ck_assert(strstr(pl->str, "#5 ? sock_def_readable"));
+        ck_assert(strstr(pl->str, "#6 ? ipv6_dev_mc_inc"));
+        ck_assert(strstr(pl->str, "#7 ? ipv6_sock_mc_join"));
+        ck_assert(strstr(pl->str, "#8 ? do_ipv6_setsockopt.isra.6"));
+        ck_assert(strstr(pl->str, "#9 ? pipe_write"));
+        ck_assert(strstr(pl->str, "#10 ? SYSC_sendto"));
+        ck_assert(strstr(pl->str, "#11 ? ipv6_setsockopt"));
+        ck_assert(strstr(pl->str, "#12 ? SyS_setsockopt"));
+        ck_assert(strstr(pl->str, "#13 ? system_call_fastpath"));
 
         g_string_free(pl, true);
 }
@@ -454,49 +454,49 @@ START_TEST(irq_payload)
         ck_assert(strstr(pl->str, "#1 dump_stack"));
         ck_assert(strstr(pl->str, "#2 __report_bad_irq"));
         ck_assert(strstr(pl->str, "#3 note_interrupt"));
-        ck_assert(strstr(pl->str, "#4 __do_softirq"));
+        ck_assert(strstr(pl->str, "#4 ? __do_softirq"));
         ck_assert(strstr(pl->str, "#5 handle_irq_event_percpu"));
         ck_assert(strstr(pl->str, "#6 handle_irq_event"));
-        ck_assert(strstr(pl->str, "#7 handle_nested_irq"));
+        ck_assert(strstr(pl->str, "#7 ? handle_nested_irq"));
         ck_assert(strstr(pl->str, "#8 handle_level_irq"));
-        ck_assert(strstr(pl->str, "#9 do_IRQ"));
-        ck_assert(strstr(pl->str, "#10 common_interrupt"));
-        ck_assert(strstr(pl->str, "#11 __do_softirq"));
-        ck_assert(strstr(pl->str, "#12 handle_irq"));
-        ck_assert(strstr(pl->str, "#13 irq_exit"));
-        ck_assert(strstr(pl->str, "#14 do_IRQ"));
-        ck_assert(strstr(pl->str, "#15 pci_write"));
-        ck_assert(strstr(pl->str, "#16 common_interrupt"));
-        ck_assert(strstr(pl->str, "#17 audit_make_tree"));
-        ck_assert(strstr(pl->str, "#18 usbdev_ioctl"));
-        ck_assert(strstr(pl->str, "#19 _raw_spin_unlock_irqrestore"));
-        ck_assert(strstr(pl->str, "#20 __setup_irq"));
-        ck_assert(strstr(pl->str, "#21 kmem_cache_alloc_trace"));
-        ck_assert(strstr(pl->str, "#22 vsnprintf"));
-        ck_assert(strstr(pl->str, "#23 request_threaded_irq"));
-        ck_assert(strstr(pl->str, "#24 usb_hcd_platform_shutdown"));
-        ck_assert(strstr(pl->str, "#25 request_threaded_irq"));
-        ck_assert(strstr(pl->str, "#26 usb_add_hcd"));
-        ck_assert(strstr(pl->str, "#27 usb_hcd_pci_probe"));
-        ck_assert(strstr(pl->str, "#28 rpm_resume"));
-        ck_assert(strstr(pl->str, "#29 pci_device_probe"));
-        ck_assert(strstr(pl->str, "#30 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#31 pci_match_device"));
-        ck_assert(strstr(pl->str, "#32 __driver_attach"));
-        ck_assert(strstr(pl->str, "#33 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#34 bus_for_each_dev"));
-        ck_assert(strstr(pl->str, "#35 driver_attach"));
-        ck_assert(strstr(pl->str, "#36 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#37 bus_add_driver"));
-        ck_assert(strstr(pl->str, "#38 pci_match_id"));
-        ck_assert(strstr(pl->str, "#39 driver_register"));
-        ck_assert(strstr(pl->str, "#40 ohci_hcd_mod_init"));
-        ck_assert(strstr(pl->str, "#41 ohci_hcd_mod_init"));
-        ck_assert(strstr(pl->str, "#42 __pci_register_driver"));
-        ck_assert(strstr(pl->str, "#43 uhci_hcd_init"));
-        ck_assert(strstr(pl->str, "#44 do_one_initcall"));
-        ck_assert(strstr(pl->str, "#45 kernel_init_freeable"));
-        ck_assert(strstr(pl->str, "#46 do_early_param"));
+        ck_assert(strstr(pl->str, "#9 ? do_IRQ"));
+        ck_assert(strstr(pl->str, "#10 ? common_interrupt"));
+        ck_assert(strstr(pl->str, "#11 ? __do_softirq"));
+        ck_assert(strstr(pl->str, "#12 ? handle_irq"));
+        ck_assert(strstr(pl->str, "#13 ? irq_exit"));
+        ck_assert(strstr(pl->str, "#14 ? do_IRQ"));
+        ck_assert(strstr(pl->str, "#15 ? pci_write"));
+        ck_assert(strstr(pl->str, "#16 ? common_interrupt"));
+        ck_assert(strstr(pl->str, "#17 ? audit_make_tree"));
+        ck_assert(strstr(pl->str, "#18 ? usbdev_ioctl"));
+        ck_assert(strstr(pl->str, "#19 ? _raw_spin_unlock_irqrestore"));
+        ck_assert(strstr(pl->str, "#20 ? __setup_irq"));
+        ck_assert(strstr(pl->str, "#21 ? kmem_cache_alloc_trace"));
+        ck_assert(strstr(pl->str, "#22 ? vsnprintf"));
+        ck_assert(strstr(pl->str, "#23 ? request_threaded_irq"));
+        ck_assert(strstr(pl->str, "#24 ? usb_hcd_platform_shutdown"));
+        ck_assert(strstr(pl->str, "#25 ? request_threaded_irq"));
+        ck_assert(strstr(pl->str, "#26 ? usb_add_hcd"));
+        ck_assert(strstr(pl->str, "#27 ? usb_hcd_pci_probe"));
+        ck_assert(strstr(pl->str, "#28 ? rpm_resume"));
+        ck_assert(strstr(pl->str, "#29 ? pci_device_probe"));
+        ck_assert(strstr(pl->str, "#30 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#31 ? pci_match_device"));
+        ck_assert(strstr(pl->str, "#32 ? __driver_attach"));
+        ck_assert(strstr(pl->str, "#33 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#34 ? bus_for_each_dev"));
+        ck_assert(strstr(pl->str, "#35 ? driver_attach"));
+        ck_assert(strstr(pl->str, "#36 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#37 ? bus_add_driver"));
+        ck_assert(strstr(pl->str, "#38 ? pci_match_id"));
+        ck_assert(strstr(pl->str, "#39 ? driver_register"));
+        ck_assert(strstr(pl->str, "#40 ? ohci_hcd_mod_init"));
+        ck_assert(strstr(pl->str, "#41 ? ohci_hcd_mod_init"));
+        ck_assert(strstr(pl->str, "#42 ? __pci_register_driver"));
+        ck_assert(strstr(pl->str, "#43 ? uhci_hcd_init"));
+        ck_assert(strstr(pl->str, "#44 ? do_one_initcall"));
+        ck_assert(strstr(pl->str, "#45 ? kernel_init_freeable"));
+        ck_assert(strstr(pl->str, "#46 ? do_early_param"));
 
         g_string_free(pl, true);
 
@@ -521,12 +521,12 @@ START_TEST(general_protection_fault_payload)
         ck_assert(strstr(pl->str, "Modules : tun 8021q garp mrp bridge stp llc bonding ipt_REJECT xt_comment xt_multiport"));
         ck_assert(strstr(pl->str, "lpc_ich iw_cm mfd_core nfsd auth_rpcgss wmi nfs_acl lockd ipmi_si ipmi_devintf"));
 
-        ck_assert(strstr(pl->str, "#1 alloc_pipe_info"));
+        ck_assert(strstr(pl->str, "#1 ? alloc_pipe_info"));
         ck_assert(strstr(pl->str, "#2 alloc_pipe_info"));
         ck_assert(strstr(pl->str, "#3 create_pipe_files"));
         ck_assert(strstr(pl->str, "#4 __do_pipe_flags"));
         ck_assert(strstr(pl->str, "#5 SyS_pipe2"));
-        ck_assert(strstr(pl->str, "#6 __audit_syscall_exit"));
+        ck_assert(strstr(pl->str, "#6 ? __audit_syscall_exit"));
         ck_assert(strstr(pl->str, "#7 SyS_pipe"));
         ck_assert(strstr(pl->str, "#8 system_call_fastpath"));
 
@@ -594,29 +594,29 @@ START_TEST(bug_kernel_handle_payload)
         ck_assert(strstr(pl->str, "Modules : nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE be2iscsi"));
         ck_assert(strstr(pl->str, "drm_kms_helper firewire_ohci drm firewire_core tg3 crc_itu_t i2c_core video usb_storage sunrpc"));
 
-        ck_assert(strstr(pl->str, "#1 _nv007312rm"));
-        ck_assert(strstr(pl->str, "#2 _nv007847rm"));
-        ck_assert(strstr(pl->str, "#3 _nv004049rm"));
-        ck_assert(strstr(pl->str, "#4 _nv004049rm"));
-        ck_assert(strstr(pl->str, "#5 _nv010019rm"));
-        ck_assert(strstr(pl->str, "#6 _nv014983rm"));
-        ck_assert(strstr(pl->str, "#7 _nv001097rm"));
-        ck_assert(strstr(pl->str, "#8 rm_init_adapter"));
-        ck_assert(strstr(pl->str, "#9 nv_kern_open"));
-        ck_assert(strstr(pl->str, "#10 chrdev_open"));
-        ck_assert(strstr(pl->str, "#11 do_dentry_open"));
-        ck_assert(strstr(pl->str, "#12 cdev_put"));
-        ck_assert(strstr(pl->str, "#13 finish_open"));
-        ck_assert(strstr(pl->str, "#14 do_last"));
-        ck_assert(strstr(pl->str, "#15 inode_permission"));
-        ck_assert(strstr(pl->str, "#16 link_path_walk"));
-        ck_assert(strstr(pl->str, "#17 path_openat"));
-        ck_assert(strstr(pl->str, "#18 do_filp_open"));
-        ck_assert(strstr(pl->str, "#19 __alloc_fd"));
-        ck_assert(strstr(pl->str, "#20 do_sys_open"));
-        ck_assert(strstr(pl->str, "#21 __audit_syscall_entry"));
-        ck_assert(strstr(pl->str, "#22 sys_open"));
-        ck_assert(strstr(pl->str, "#23 system_call_fastpath"));
+        ck_assert(strstr(pl->str, "#1 ? _nv007312rm"));
+        ck_assert(strstr(pl->str, "#2 ? _nv007847rm"));
+        ck_assert(strstr(pl->str, "#3 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#4 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#5 ? _nv010019rm"));
+        ck_assert(strstr(pl->str, "#6 ? _nv014983rm"));
+        ck_assert(strstr(pl->str, "#7 ? _nv001097rm"));
+        ck_assert(strstr(pl->str, "#8 ? rm_init_adapter"));
+        ck_assert(strstr(pl->str, "#9 ? nv_kern_open"));
+        ck_assert(strstr(pl->str, "#10 ? chrdev_open"));
+        ck_assert(strstr(pl->str, "#11 ? do_dentry_open"));
+        ck_assert(strstr(pl->str, "#12 ? cdev_put"));
+        ck_assert(strstr(pl->str, "#13 ? finish_open"));
+        ck_assert(strstr(pl->str, "#14 ? do_last"));
+        ck_assert(strstr(pl->str, "#15 ? inode_permission"));
+        ck_assert(strstr(pl->str, "#16 ? link_path_walk"));
+        ck_assert(strstr(pl->str, "#17 ? path_openat"));
+        ck_assert(strstr(pl->str, "#18 ? do_filp_open"));
+        ck_assert(strstr(pl->str, "#19 ? __alloc_fd"));
+        ck_assert(strstr(pl->str, "#20 ? do_sys_open"));
+        ck_assert(strstr(pl->str, "#21 ? __audit_syscall_entry"));
+        ck_assert(strstr(pl->str, "#22 ? sys_open"));
+        ck_assert(strstr(pl->str, "#23 ? system_call_fastpath"));
 
         g_string_free(pl, true);