Release v1.16.1

This release contains:

* Fix for failing unit tests during build when the system where the
package is being built has a telemetry client running.

* Added documentation to probes.

* Documenting local debugging features.

Signed-off-by: Alex Jaramillo <alex.v.jaramillo@intel.com>

.gitignore

@@ -38,6 +38,7 @@ autoscan.log
 configure.scan
 tprobe
 hprobe
+bertprobe
 crashprobe
 journalprobe
 telem-record-gen
@@ -45,6 +46,7 @@ klogscanner
 oopsprobe
 pstoreclean
 pstoreprobe
+telem_journal
 make.out
 types_c.taghl
 src/recomp.sh

.travis.yml

@@ -0,0 +1,35 @@
+dist: trusty
+
+compiler:
+    - gcc
+
+os:
+    - linux
+
+before_script:
+    ./autogen.sh
+
+language: c
+
+addons:
+    apt:
+        sources:
+            - ubuntu-toolchain-r-test
+        packages:
+            - libcurl4-gnutls-dev
+            - valgrind
+            - autoconf 
+            - automake
+            - check
+            - libelf-dev
+            - libdw-dev
+
+install:
+        - wget https://github.com/libcheck/check/releases/download/0.12.0/check-0.12.0.tar.gz
+        - tar -xvf check-0.12.0.tar.gz
+        - pushd check-0.12.0 && ./configure --prefix=/usr --libdir=/usr/lib/x86_64-linux-gnu && make -j48 && sudo make install && popd
+
+script:
+    - ./configure && make && make check
+
+after_failure: cat test-suite.log

CODING-STYLE

@@ -62,7 +62,7 @@ Coding Style
 - To make type casts stand out more, include no space between the cast and the
   variable, etc. to cast:
 
-        GString **foo = (GString **)bar;
+        nc_string **foo = (nc_string **)bar;
 
 - Whenever arithmetic operators are used, always include one space before and
   after each operator:

Makefile.am

@@ -18,7 +18,9 @@ include $(top_srcdir)/docs/local.mk
 include $(top_srcdir)/scripts/local.mk
 include $(top_srcdir)/src/local.mk
 include $(top_srcdir)/src/data/local.mk
+include $(top_srcdir)/src/nica/local.mk
 include $(top_srcdir)/src/probes/local.mk
+include $(top_srcdir)/src/journal/local.mk
 include $(top_srcdir)/tests/local.mk
 
 release:

README

@@ -1,32 +0,0 @@
-telemetrics-client
-==================
-
-This package provides the front end component of a complete telemetrics
-solution for Linux-based operating systems. Specifically, the front end
-component includes:
-
-- telemetrics probes that collect specific types of data from the operating
-  system.
-
-- a library, libtelemetry, that telemetrics probes use to create telemetrics
-  records and send them to the daemon for further processing.
-
-- a daemon, telemd, that prepares the records to send to a telemetrics server
-  (not included in this source tree), or spools the records on disk in case
-  it's unable to successfully deliver them.
-
-
-Build dependencies
-==================
-
-- libcheck
-
-- libcurl
-
-- glib
-
-- elfutils, which provides libelf and libdwfl libraries..
-
-- (optional) libsystemd, for syslog-style logging to the systemd journal, and
-  socket/path activation of telemd by systemd.
-

README.md

@@ -0,0 +1,312 @@
+Telemetrics-client
+==================
+
+This package provides the front end component of a complete telemetrics
+solution for Linux-based operating systems. Specifically, the front end
+component includes:
+
+- telemetrics probes that collect specific types of data from the operating
+  system. For more info on probes go [here](./src/probes).
+
+- a library, libtelemetry, that telemetrics probes use to create telemetrics
+  records and send them to the daemon for further processing.
+
+- a daemon, telemd, that prepares the records to send to a telemetrics server
+  (not included in this source tree), or spools the records on disk in case
+  it's unable to successfully deliver them.
+
+A telemetrics server implementation that works with this component is available
+from
+[clearlinux/telemetrics-backend](https://github.com/clearlinux/telemetrics-backend).
+
+
+Build dependencies
+---------------------
+
+- libcheck
+
+- libcurl
+
+- elfutils, which provides libelf and libdwfl libraries..
+
+- (optional) libsystemd, for syslog-style logging to the systemd journal, and
+  socket/path activation of telemd by systemd.
+
+
+Build and installation
+---------------------
+
+```{r, engine='bash', count_lines}
+$ ./autogen.sh
+$ ./configure
+$ make
+```
+
+Set up
+---------------------
+
+There is a config file installed at
+
+```{r, engine='bash', count_lines}
+/usr/share/defaults/telemetrics/telemetrics.conf
+```
+
+To make modifications, copy that file to
+
+```{r, engine='bash', count_lines}
+/etc/telemetrics/telemetrics.conf
+```
+and modify the /etc version.
+
+Descriptions of config options are listed below in the Usage section.
+
+Starting the daemon
+---------------------
+
+Method 1 (recommended):
+
+```{r, engine='bash', count_lines}
+ systemctl start telemd.socket telemd.path
+```
+
+Note: the above invocation technically readies the service for both socket and
+path activation, so you may not see the daemon start right away.
+
+Method 2:
+
+```{r, engine='bash', count_lines}
+systemctl start telemd.service
+```
+
+Method 3:
+
+```{r, engine='bash', count_lines}
+telemd &
+```
+
+Configure the daemon to autostart at boot
+---------------------
+
+Method 1 (recommended):
+
+Enable the socket-activated service and path unit:
+
+```{r, engine='bash', count_lines}
+systemctl enable telemd.socket telemd.path
+```
+
+Method 2:
+
+Enable the service itself, which automatically enables the socket and path
+units as well:
+
+```{r, engine='bash', count_lines}
+systemctl enable telemd.service
+```
+
+Usage
+---------------------
+
+Once the daemon is running, the telemetrics probes will be ready to use.
+
+Available probes:
+
+* hprobe: A test program that utilizes libtelemetry to ensure
+that telemetrics-client works. It sends a "hello world" record to the server.
+
+* crash probe: A handler for core files that sends the corresponding backtraces
+to the server.
+
+The daemon uses the following configuration options from the configuration file:
+
+* server: This specifies the web server that the daemon sends the telemetry records to
+* socket_path: This specifies the path of the unix domain socket that the
+  daemon listens on for connections from the probes
+* spool_dir: This config option is related to spooling. If the daemon is not
+  able to send the telemetry records to the backend server due to reasons such
+  as the network availability, then it stores the records in a spool directory.
+  This option specifies that path of the spool directory. This directory should
+  be owned by the same user that the daemon is running as.
+
+```{r, engine='bash', count_lines}
+    mkdir -p /var/spool/telemetry
+    chown -R telemetry:telemetry /var/spool/telemetry
+    systemctl restart telemd.service
+```
+
+* record_expiry: This is the time in minutes after which the records in the
+  spool directory are deleted by the daemon.
+* spool_max_size: This specifies the maximum size of the spool directory. When
+  the size of the spool directory reaches this limit, new telemetry records are
+  dropped by the daemon.
+* spool_process_time: This specifies the time interval in seconds that the
+  daemon waits for before checking the spool directory for records. The daemon
+  picks up the records in the order of modification date and tries to send the
+  record to the server. It sends a maximum of 10 records at a time. If it was
+  able to send a record successfully, it deletes the record from the spool.  If
+  the daemon finds a record older than the "record_expiry" time, then it
+  deletes that record. The daemon looks at a maximum of 20 records in a single
+  spool run loop.
+* rate_limit_enabled: This determines whether rate-limiting is enabled or
+  disabled. When enabled, there is a threshold on both records sent within a
+  window of time, and record bytes sent within a window a time.
+* record_burst_limit: This is the maximum amount of records allowed to be
+  passed by the daemon within the record_window_length of time. If set to -1,
+  the rate-limiting for record bursts is disabled.
+* record_window_length: This is the time, in minutes (0-59), that establishes
+  the window length for the record_burst_limit. EX: if record_burst_window=1000
+  and record_window_length=15, then no more than 1000 records can be passed
+  within any given fifteen minute window.
+* byte_burst_limit: This is the maximum amount of bytes that can be passed by
+  the daemon within the byte_window_length of time. If set to -1, the
+  rate-limiting for byte bursts is disabled.
+* byte_window_length: This is the time, in minutes (0-59), that establishes the
+  window length for the byte_burst_limit.
+* rate_limit_strategy: This is the strategy chosen once the rate-limiting
+  threshold has been reached. Currently the options are 'drop' or 'spool', with
+  spool being the default. If spool is chosen, records will be spooled and sent
+  at a later time.
+* record_retention_enabled: When this key is enabled (true) the daemon saves a
+  copy of the payload on disk from all valid records. To avoid the excessive use
+  of disk space only the latest 100 records are kept. The default value for this
+  configuration key is false.
+* record_server_delivery_enabled: This key controls the delivery of records to
+  ```server```, when enabled (default value) the record will be posted to the
+  address in the configuration file. If this configuration key is disabled (false)
+  records will not be spooled or posted to backend. This configuration key can
+  be used in combination with ```record_retention_enabled``` to keep copies of
+  telemetry records locally only.
+
+
+Data reported
+---------------------
+
+The data reported by the telemetry client could be understood as two main sets:
+metadata and a payload.
+
+The metadata is used to report details of a machine's architecture. The
+following are the metadata values currently collected (Record Format Version 4):
+
+* record_format_version: version of the record, currently is 'Version 4'. This
+  value changes when new metadata is added.
+* classification: this field is used to identify the type of record sent by a
+  specific client probe; classifications use the format DOMAIN/PROBE/REST, where
+  DOMAIN is the vendor of the probe, PROBE is the probe name, and REST is a
+  probe-defined field to classify what is contained in the payload.
+* severity: this is an integer value between 1 and 4 where 1 is "low" and 4 is
+  "critical"
+* machine_id: a machine identifier that is rotate every 3 days for privacy reasons.
+* creation_timestamp: timestamp when the record was collected.
+* arch: a string describing machine architecture i.e. 'x86_64'.
+* host_type: a string with the combination of 'System Vendor', 'Product Name', and
+  'Product Version' read from dmi file system.
+* build: OS build number.
+* kernel_version: Kernel version.
+* payload_format_version: version of the payload, currently is 'Version 1'.
+* system_name: the value after 'ID=' from '/etc/os-release' (or distribution
+  provided folder)
+* board_name: a string read from dmi file system that combines 'Board Name' and
+  'Board Vendor'.
+* cpu_model: cpu model name extracted from '/proc/cpuinfo'.
+* bios_version: BIOS version.
+* event_id: an id to group multiple records if these were generated by a single
+  event occurrence.
+
+The payload as mentioned above is reported by probes. The telemetry library adds
+the metadata to the payload (done programatically when using library API) for
+more information about probes go [here](./src/probes).
+
+
+Machine id
+---------------------
+
+The machine id reported by the telemetry client is rotated every 3 days for
+privacy reasons. If you wish to have a static machine id for testing purposes,
+you can opt in by creating a static machine id file named
+"opt-in-static-machine-id" under the directory  "/etc/telemetrics/". Where
+"unique machine ID" is your desired static machine ID:
+
+```{r, engine='bash', count_lines}
+# mkdir -p /etc/telemetrics
+# echo "unique machine ID" > /etc/telemetrics/opt-in-static-machine-id
+```
+
+The telemetry daemon reads, at most, the first 32 characters from this file
+uses it for the machine id. You can put a string like 'my-machine-name' in this
+file to easily identify your machine.  Restart telemd for the machine id
+changes to take effect by running:
+
+```{r, engine='bash', count_lines}
+# systemctl restart telemd.service
+```
+
+You can switch back to the rotating machine id by deleting the override file
+and restarting the daemon.  You can do a quick test to check that your
+machine-id has changed by running "hprobe" and verifying that a record has
+landed on your backend telemetrics server, with the specified machine id.
+
+Event Id
+----------------------
+
+This is a 32 character lowercase hexadecimal string i.e. '5de9de8d5f3c6a7d445d75ba01cc3322'.
+This header is used to group multiple records by an event id. Before this header
+every single record could have been thought of an event, however this is not always
+the case. There are "events" that trigger the creation of multiple records (i.e.
+updates). The ```event_id``` header was added for probes with the capability to
+detect events and group records based on such events. This header was added to
+```telem-record-gen``` and can be specified using the ```-e``` (```--event-id```
+long form) switch.
+
+```{r, engine=bash, count_lines}
+  -e, --event-id        Event id to use in the record
+```
+
+Debugging locally with telemetrics-client
+-----------------------------------------------
+
+The function of the telemetrics-client is to handle the transport of information
+reported by a probe to a backend (see ```server``` key in configuration). This
+information is helpful for developers to debug and fix reported crashes, however
+developers not always have access to the backend in these case users can leverage
+features added for local debugging. The following is a list of steps to enable
+local debug:
+
+* *Enabling record retention*: this step configures telemd to keep
+copies of telemetry records locally. To enable record retention set the value of
+```record_retention_enabled``` from ```false``` to ```true```. Optionally set
+```record_server_delivery_enabled`` to ```false``` to keep records local only.
+Remember to restart the daemon after configuration values are updated
+(```telemctl restart```).
+
+* *Creating a record*: run ```hprobe``` command to create a record for the purposes
+of this step by step guide. Once we have the record or records that you need to
+capture locally you can display the data.
+
+* *Displaying record metadata*: telemd keeps metadata of any valid record,
+to display this data a new option to telemctl was added ```telemctl journal```.
+Assuming that the last record created was the record from previous step `hprobe` we
+can use `tail -n 1` to print the last created record only, i.e.
+
+```
+$ sudo telemctl journal | tail -n 1
+$ org.clearlinux/hello/world     Mon 2018-04-02 17:48:01 UTC a19a0d41ba16788881e274b19b8a1be4 5de9de8d5f3c6a7d445d75ba01cc3322 60c014cd-4693-40f1-b334-548cd932949b
+```
+
+The headers for the metadata (along with other information) can be printed using the
+```-V``` switch with ```telemctl journal``` command, i.e.
+
+```
+$ sudo telemctl journal -V | head -n 1
+$ Classification             Time stamp              Record ID                    Event ID                     Boot ID
+```
+
+* *Displaying record payload*: to print the content of a record payload you can use
+the ```-i``` (```--include_record``` long format) option to ```telemctl journal```
+command. To print the specific record you created you can use the option ```-r```
+(```--record_id``` long format) with the Record Id of the generated record, i.e.
+
+```
+$ sudo telemctl journal --record_id a19a0d41ba16788881e274b19b8a1be4 --include_record
+$ org.clearlinux/hello/world     Mon 2018-04-02 17:48:01 UTC a19a0d41ba16788881e274b19b8a1be4 5de9de8d5f3c6a7d445d75ba01cc3322 60c014cd-4693-40f1-b334-548cd932949b
+$ hello
+```

build-aux/make/cflags.make

@@ -2,7 +2,6 @@ AM_CFLAGS = \
 	-std=gnu99 \
 	-pedantic \
 	-Wall \
-	-fstack-protector \
 	-Wformat \
 	-Wformat-security \
 	-Wimplicit-function-declaration \
@@ -12,7 +11,6 @@ AM_CFLAGS = \
 	-Wconversion \
 	-Wunreachable-code \
 	-funsigned-char \
-	-fstack-protector-strong \
 	-fPIE \
 	-fPIC
 

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([telemetrics-client], [1.10.1], [https://clearlinux.org/])
+AC_INIT([telemetrics-client], [1.16.1], [https://clearlinux.org/])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([1.14 -Wall -Werror -Wno-extra-portability foreign subdir-objects])
 AM_SILENT_RULES([yes])
@@ -21,9 +21,10 @@ AC_PROG_LN_S
 AC_PROG_MAKE_SET
 
 # Checks for libraries.
-PKG_CHECK_MODULES([CHECK], [check >= 0.9.12])
+
+# check >= 0.9.12 is required for TAP output
+PKG_CHECK_MODULES([CHECK], [check >= 0.12])
 PKG_CHECK_MODULES([CURL], [libcurl])
-PKG_CHECK_MODULES([GLIB], [glib-2.0])
 AC_CHECK_LIB([elf], [elf_begin], [have_elflib=yes], [AC_MSG_ERROR([Unable to find libelf from elfutils])])
 AC_CHECK_LIB([dw], [dwfl_begin], [have_dwlib=yes], [AC_MSG_ERROR([Unable to find libdw from elfutils])])
 AS_IF([test "x$have_elflib" = "xyes" -a "x$have_dwlib" = "xyes"],
@@ -97,6 +98,11 @@ AC_ARG_WITH([systemdsysctldir], AS_HELP_STRING([--with-systemdsysctldir=DIR],
 test -z "${sysctlpath}" && sysctlpath=/usr/lib/sysctl.d
 AC_SUBST(SYSTEMD_SYSCTLDIR, [${sysctlpath}])
 
+AC_ARG_WITH([systemdsystemctldir], AS_HELP_STRING([--with-systemdsystemctldir=DIR],
+            [path to systemd systemctl dir @<:@default=/usr/bin@:>@]), [systemctlpath=${withval}])
+test -z "${systemctlpath}" && systemctlpath=/usr/bin
+AC_SUBST(SYSTEMD_SYSTEMCTLDIR, [${systemctlpath}])
+
 # systemd.pc gives the /etc conf dir, so we can't use PKG_CONFIG for this one
 AC_ARG_WITH([systemdsystemconfdir], AS_HELP_STRING([--with-systemdsystemconfdir=DIR],
 	    [path to systemd system conf dir @<:@default=/usr/lib/systemd/system.conf.d@:>@]), [confpath=${withval}],
@@ -137,3 +143,27 @@ AC_CONFIG_COMMANDS([config dir],[mkdir -p src/data; touch src/data/.dirstamp])
 AC_CONFIG_FILES([Makefile])
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_OUTPUT
+
+AC_MSG_NOTICE([
+
+telemetrics-client $VERSION
+===========================
+
+prefix:                 $prefix
+exec_prefix:            $exec_prefix
+libdir:                 $libdir
+bindir:                 $bindir
+datarootdir:            $datarootdir
+
+compiler:               $CC
+cflags:                 $CFLAGS
+ldflags:                $LDFLAGS
+
+unitdir:                $unitpath
+tmpfilesdir:            $tmpfilespath
+sysctldir:              $sysctlpath
+systemconfdir:          $confpath
+socketdir:              $socketpath
+loglevel:               $loglevel
+logtype:                $logtype
+])

docs/local.mk

@@ -13,8 +13,8 @@ MANLINKS = \
 
 manpages:
 	for MANPAGE in $(MANPAGES); do \
-		ronn --roff < $${MANPAGE}.md > $${MANPAGE}; \
-		ronn --html < $${MANPAGE}.md > $${MANPAGE}.html; \
+		rst2man.py $${MANPAGE}.rst > $${MANPAGE}; \
+		rst2html.py $${MANPAGE}.rst > $${MANPAGE}.html; \
 	done
 
 dist_man_MANS = \

docs/man/telem-record-gen.1

@@ -1,81 +1,98 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.\" Man page generated from reStructuredText.
 .
-.TH "TELEM\-RECORD\-GEN" "1" "December 2016" "" ""
+.TH TELEM-RECORD-GEN 1 "" "" ""
+.SH NAME
+telem-record-gen \- Create custom telemetry record
 .
-.SH "NAME"
-\fBtelem\-record\-gen\fR \- Create custom telemetry record
+.nr rst2man-indent-level 0
 .
-.SH "SYNOPSIS"
-\fBtelem\-record\-gen\fR \fIoptions\fR
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelem\-record\-gen\fP <options>
+.SH DESCRIPTION
+.sp
+Generate and attempt to send a telemetry record to \fBtelemd\fP(1). This
+program can be used to create a telemetry record from a shell script
+or from another program using e.g. \fBsystem()\fP\&. The program connects
+to a local \fBtelemd\fP(1) daemon to spool the record for delivery to
+the actual collection service.
+.sp
+The payload can be provided as standard input, as a file with the
+\fB\-\-payload\-file\fP option, or as a string argument with the \fB\-\-payload\fP
+option. If neither the \fB\-\-payload\fP or \fB\-\-payload\-file\fP is provided,
+the payload is assumed to be on <stdin>.
+.sp
+Each record is expected to have a class identifier, a record version
+number, and a severity level. A class identifier is required to be
+provided.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fB\-h\fP, \fB\-\-help\fP:
+Show help options.
+.IP \(bu 2
+\fB\-V\fP, \fB\-\-version\fP:
+Print the program version.
+.IP \(bu 2
+\fB\-f\fP, \fB\-\-config\-file\fP <file>:
+Path to configuration file (not implemented yet).
+.IP \(bu 2
+\fB\-s\fP, \fB\-\-severity\fP:
+Severity level (1\-4) \- (default 1).
+.IP \(bu 2
+\fB\-c\fP, \fB\-\-class\fP:
+Classification \fBlevel_1\fP/\fBlevel_2\fP/\fBlevel_3\fP\&.
+.IP \(bu 2
+\fB\-p\fP, \fB\-\-payload\fP <payload>:
+Record body (max size = 8k).
+.IP \(bu 2
+\fB\-P\fP, \fB\-\-payload\-file\fP <file>:
+File to read payload from.
+.IP \(bu 2
+\fB\-R\fP, \fB\-\-record\-version\fP <version>:
+Version number for format of payload (default 1).
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH RETURN VALUES
+.sp
+0 on success. A non\-zero exit code indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
 .
-.SH "DESCRIPTION"
-Generate and attempt to send a telemetry record to \fBtelemd(1)\fR\. This program can be used to create a telemetry record from a shell script or from another program using e\.g\. \fBsystem()\fR\. The program connects to a local \fBtelemd(1)\fR daemon to spool the record for delivery to the actual collection service\.
-.
-.P
-The payload can be provided as standard input, as a file with the \fB\-\-payload\-file\fR option, or as a string argument with the \fB\-\-payload\fR option\. If neither the \fB\-\-payload\fR or \fB\-\-payload\-file\fR is provided, the payload is assumed to be on \fIstdin\fR\.
-.
-.P
-Each record is expected to have a class identifier, a record version number, and a severity level\. A class identifier is required to be provided\.
-.
-.SH "OPTIONS"
-.
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-Show help options\.
-.
-.TP
-\fB\-V\fR, \fB\-\-version\fR
-Print the program version\.
-.
-.TP
-\fB\-f\fR, \fB\-\-config\-file\fR \fIfile\fR
-Path to configuration file (not implemented yet)\.
-.
-.TP
-\fB\-s\fR, \fB\-\-severity\fR
-Severity level (1\-4) \- (default 1)\.
-.
-.TP
-\fB\-c\fR, \fB\-\-class\fR
-Classification \fBlevel_1\fR/\fBlevel_2\fR/\fBlevel_3\fR\.
-.
-.TP
-\fB\-p\fR, \fB\-\-payload\fR \fIpayload\fR
-Record body (max size = 8k)\.
-.
-.TP
-\fB\-P\fR, \fB\-\-payload\-file\fR \fIfile\fR
-File to read payload from\.
-.
-.TP
-\fB\-R\fR, \fB\-\-record\-version\fR \fIversion\fR
-Version number for format of payload (default 1)\.
-.
-.SH "RETURN VALUES"
-0 on success\. A non\-zero exit code indicates a failure occurred\.
-.
-.SH "COPYRIGHT"
-.
-.IP "\(bu" 4
-Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
-.
-.IP "" 0
-.
-.SH "SEE ALSO"
-\fBtelemd(1)\fR
-.
-.P
-https://github\.com/clearlinux/telemetrics\-client
-.
-.P
-https://clearlinux\.org/documentation/
-.
-.SH "NOTES"
-Creative Commons Attribution\-ShareAlike 3\.0 Unported
-.
-.IP "\(bu" 4
-http://creativecommons\.org/licenses/by\-sa/3\.0/
-.
-.IP "" 0
-

docs/man/telem-record-gen.1.md

@@ -1,70 +0,0 @@
-telem-record-gen(1) -- Create custom telemetry record
-=====================================================
-
-## SYNOPSIS
-
-`telem-record-gen` <options>
-
-## DESCRIPTION
-
-Generate and attempt to send a telemetry record to `telemd(1)`. This
-program can be used to create a telemetry record from a shell script
-or from another program using e.g. `system()`. The program connects
-to a local `telemd(1)` daemon to spool the record for delivery to
-the actual collection service.
-
-The payload can be provided as standard input, as a file with the
-`--payload-file` option, or as a string argument with the `--payload`
-option. If neither the `--payload` or `--payload-file` is provided,
-the payload is assumed to be on <stdin>.
-
-Each record is expected to have a class identifier, a record version
-number, and a severity level. A class identifier is required to be
-provided.
-
-## OPTIONS
- * `-h`, `--help`:
-   Show help options.
-
- * `-V`, `--version`:
-   Print the program version.
-
- * `-f`, `--config-file` <file>:
-   Path to configuration file (not implemented yet).
-
- * `-s`, `--severity`:
-   Severity level (1-4) - (default 1).
-
- * `-c`, `--class`:
-   Classification `level_1`/`level_2`/`level_3`.
-
- * `-p`, `--payload` <payload>:
-   Record body (max size = 8k).
-
- * `-P`, `--payload-file` <file>:
-   File to read payload from.
-
- * `-R`, `--record-version` <version>:
-   Version number for format of payload (default 1).
-
-## RETURN VALUES
-
-0 on success. A non-zero exit code indicates a failure occurred.
-
-## COPYRIGHT
-
- * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
-
-## SEE ALSO
-
-`telemd(1)`
-
-https://github.com/clearlinux/telemetrics-client
-
-https://clearlinux.org/documentation/
-
-## NOTES
-
-Creative Commons Attribution-ShareAlike 3.0 Unported
-
- * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telem-record-gen.1.rst

@@ -0,0 +1,78 @@
+================
+telem-record-gen
+================
+
+------------------------------
+Create custom telemetry record
+------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telem-record-gen`` <options>
+
+
+DESCRIPTION
+===========
+
+Generate and attempt to send a telemetry record to ``telemd``\(1). This
+program can be used to create a telemetry record from a shell script
+or from another program using e.g. ``system()``. The program connects
+to a local ``telemd``\(1) daemon to spool the record for delivery to
+the actual collection service.
+
+The payload can be provided as standard input, as a file with the
+``--payload-file`` option, or as a string argument with the ``--payload``
+option. If neither the ``--payload`` or ``--payload-file`` is provided,
+the payload is assumed to be on <stdin>.
+
+Each record is expected to have a class identifier, a record version
+number, and a severity level. A class identifier is required to be
+provided.
+
+
+OPTIONS
+=======
+
+ * ``-h``, ``--help``:
+   Show help options.
+
+ * ``-V``, ``--version``:
+   Print the program version.
+
+ * ``-f``, ``--config-file`` <file>:
+   Path to configuration file (not implemented yet).
+
+ * ``-s``, ``--severity``:
+   Severity level (1-4) - (default 1).
+
+ * ``-c``, ``--class``:
+   Classification ``level_1``/``level_2``/``level_3``.
+
+ * ``-p``, ``--payload`` <payload>:
+   Record body (max size = 8k).
+
+ * ``-P``, ``--payload-file`` <file>:
+   File to read payload from.
+
+ * ``-R``, ``--record-version`` <version>:
+   Version number for format of payload (default 1).
+
+
+RETURN VALUES
+=============
+
+0 on success. A non-zero exit code indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

docs/man/telemctl.1

@@ -1,58 +1,76 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.\" Man page generated from reStructuredText.
 .
-.TH "TELEMCTL" "1" "December 2016" "" ""
+.TH TELEMCTL 1 "" "" ""
+.SH NAME
+telemctl \- Telemetry service administration tool
 .
-.SH "NAME"
-\fBtelemctl\fR \- Telemetry service administration tool
+.nr rst2man-indent-level 0
 .
-.SH "SYNOPSIS"
-\fBtelemctl\fR
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelemctl\fP
+.sp
+\fB/etc/telemetrics/opt\-out\fP
+.SH DESCRIPTION
+.sp
+Control actions for telemetry services. The command can be used to start,
+restart, or stop \fBtelemd\fP(1), or to opt\-in or opt\-out of telemetry delivery
+of records to a central telemetry service.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fBstart\(ga\(ga|\(ga\(gastop\(ga\(ga|\(ga\(garestart\fP:
+Starts, stops or restarts all running telemetry services.
+.IP \(bu 2
+\fBopt\-in\fP:
+Opts in to telemetry, and starts telemetry services. The opt\-out file
+\fB/etc/telemetrics/opt\-out\fP is removed.
+.IP \(bu 2
+\fBopt\-out\fP:
+Opts out of telemetry, and stops telemetry services. The opt\-out file
+\fB/etc/telemetrics/opt\-out\fP is created.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH RETURN VALUES
+.sp
+0 on success. A non\-zero exit code indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
 .
-.P
-\fB/etc/telemetrics/opt\-out\fR
-.
-.SH "DESCRIPTION"
-Control actions for telemetry services\. The command can be used to start, restart, or stop \fBtelemd(1)\fR, or to opt\-in or opt\-out of telemetry delivery of records to a central telemetry service\.
-.
-.SH "OPTIONS"
-.
-.TP
-\fBstart\fR|\fBstop\fR|\fBrestart\fR
-Starts, stops or restarts all running telemetry services\.
-.
-.TP
-\fBopt\-in\fR
-Opts in to telemetry, and starts telemetry services\. The opt\-out file \fB/etc/telemetrics/opt\-out\fR is removed\.
-.
-.TP
-\fBopt\-out\fR
-Opts out of telemetry, and stops telemetry services\. The opt\-out file \fB/etc/telemetrics/opt\-out\fR is created\.
-.
-.SH "RETURN VALUES"
-0 on success\. A non\-zero exit code indicates a failure occurred\.
-.
-.SH "COPYRIGHT"
-.
-.IP "\(bu" 4
-Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
-.
-.IP "" 0
-.
-.SH "SEE ALSO"
-\fBtelemd(1)\fR
-.
-.P
-https://github\.com/clearlinux/telemetrics\-client
-.
-.P
-https://clearlinux\.org/documentation/
-.
-.SH "NOTES"
-Creative Commons Attribution\-ShareAlike 3\.0 Unported
-.
-.IP "\(bu" 4
-http://creativecommons\.org/licenses/by\-sa/3\.0/
-.
-.IP "" 0
-

docs/man/telemctl.1.md

@@ -1,49 +0,0 @@
-telemctl(1) -- Telemetry service administration tool
-====================================================
-
-## SYNOPSIS
-
-`telemctl`
-
-`/etc/telemetrics/opt-out`
-
-## DESCRIPTION
-
-Control actions for telemetry services. The command can be used to start,
-restart, or stop `telemd(1)`, or to opt-in or opt-out of telemetry delivery
-of records to a central telemetry service.
-
-## OPTIONS
-
- * `start`|`stop`|`restart`:
-   Starts, stops or restarts all running telemetry services.
-
- * `opt-in`: 
-   Opts in to telemetry, and starts telemetry services. The opt-out file
-   `/etc/telemetrics/opt-out` is removed.
-
- * `opt-out`: 
-   Opts out of telemetry, and stops telemetry services. The opt-out file
-   `/etc/telemetrics/opt-out` is created.
-
-## RETURN VALUES
-
-0 on success. A non-zero exit code indicates a failure occurred.
-
-## COPYRIGHT
-
- * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
-
-## SEE ALSO
-
-`telemd(1)`
-
-https://github.com/clearlinux/telemetrics-client
-
-https://clearlinux.org/documentation/
-
-## NOTES
-
-Creative Commons Attribution-ShareAlike 3.0 Unported
-
- * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemctl.1.rst

@@ -0,0 +1,56 @@
+========
+telemctl
+========
+
+-------------------------------------
+Telemetry service administration tool
+-------------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telemctl``
+
+``/etc/telemetrics/opt-out``
+
+
+DESCRIPTION
+===========
+
+Control actions for telemetry services. The command can be used to start,
+restart, or stop ``telemd``\(1), or to opt-in or opt-out of telemetry delivery
+of records to a central telemetry service.
+
+
+OPTIONS
+=======
+
+ * ``start``|``stop``|``restart``:
+   Starts, stops or restarts all running telemetry services.
+
+ * ``opt-in``: 
+   Opts in to telemetry, and starts telemetry services. The opt-out file
+   ``/etc/telemetrics/opt-out`` is removed.
+
+ * ``opt-out``: 
+   Opts out of telemetry, and stops telemetry services. The opt-out file
+   ``/etc/telemetrics/opt-out`` is created.
+
+
+RETURN VALUES
+=============
+
+0 on success. A non-zero exit code indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

docs/man/telemd.1

@@ -1,55 +1,106 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.\" Man page generated from reStructuredText.
 .
-.TH "TELEMD" "1" "December 2016" "" ""
+.TH TELEMD 1 "" "" ""
+.SH NAME
+telemd \- Telemetry client service
 .
-.SH "NAME"
-\fBtelemd\fR \- Telemetry client service
+.nr rst2man-indent-level 0
 .
-.SH "SYNOPSIS"
-\fBtelemd\fR \fIflags\fR \fB/etc/telemetrics/telemetrics\.conf\fR
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelemd\fP <flags>
+.sp
+\fB/etc/telemetrics/telemetrics.conf\fP
+.sp
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.sp
+\fB/etc/telemetrics/opt\-in\-static\-machine\-id\fP
+.SH DESCRIPTION
+.sp
+The \fBtelemd\fP program delivers locally generated telemetry records to a remote
+telemetry service. Telemetry data can be in any format, and is relayed as\-is.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fB\-f\fP, \fB\-\-config_file\fP [<file>]:
+Configuration file. This overides the other parameters.
+.IP \(bu 2
+\fB\-h\fP, \fB\-\-help\fP:
+Display this help message.
+.IP \(bu 2
+\fB\-V\fP, \fB\-\-version\fP:
+Print the program version.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH FILES
+.INDENT 0.0
+.IP \(bu 2
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+If no custom configuration file is found, \fBtelemd\fP uses the
+settings in this file.
+.UNINDENT
+.UNINDENT
+.IP \(bu 2
+\fB/etc/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+Custom configuration file that \fBtelemd\fP reads. See \fBtelemetrics.conf\fP(5).
+.UNINDENT
+.UNINDENT
+.IP \(bu 2
+\fB/etc/telemetrics/opt\-in\-static\-machine\-id\fP
+.INDENT 2.0
+.INDENT 3.5
+If this file exists, the first line of this file will be sent to
+the server as the machine ID. Otherwise, the machine ID will be
+random and changed regularly.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH EXIT STATUS
+.sp
+0 when no errors occurred. A non\-zero exit status indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemetry\fP(3)
+.IP \(bu 2
+\fBtelemetrics.conf\fP(5)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
 .
-.SH "DESCRIPTION"
-The \fBtelemd\fR program delivers locally generated telemetry records to a remote telemetry service\. Telemetry data can be in any format, and is relayed as\-is\.
-.
-.SH "OPTIONS"
-.
-.TP
-\fB\-f\fR, \fB\-\-config_file\fR [\fIfile\fR]
-Configuration file\. This overides the other parameters\.
-.
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-Display this help message\.
-.
-.TP
-\fB\-V\fR, \fB\-\-version\fR
-Print the program version\.
-.
-.SH "EXIT STATUS"
-0 when no errors occurred\. A non\-zero exit status indicates a failure occurred\.
-.
-.SH "COPYRIGHT"
-.
-.IP "\(bu" 4
-Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
-.
-.IP "" 0
-.
-.SH "SEE ALSO"
-\fBtelemetry(3)\fR \fBtelemetrics\-client\.conf(5)\fR
-.
-.P
-https://github\.com/clearlinux/telemetrics\-client
-.
-.P
-https://clearlinux\.org/documentation/
-.
-.SH "NOTES"
-Creative Commons Attribution\-ShareAlike 3\.0 Unported
-.
-.IP "\(bu" 4
-http://creativecommons\.org/licenses/by\-sa/3\.0/
-.
-.IP "" 0
-

docs/man/telemd.1.md

@@ -1,47 +0,0 @@
-telemd(1) -- Telemetry client service
-=====================================
-
-## SYNOPSIS
-
-`telemd` <flags> 
-`/etc/telemetrics/telemetrics.conf`
-
-## DESCRIPTION
-
-The `telemd` program delivers locally generated telemetry records to a remote
-telemetry service. Telemetry data can be in any format, and is relayed as-is.
-
-## OPTIONS
-
-  * `-f`, `--config_file` [<file>]:
-    Configuration file. This overides the other parameters.
-
-  * `-h`, `--help`:
-    Display this help message.
-
-  * `-V`, `--version`:
-    Print the program version.
-
-
-## EXIT STATUS
-
-0 when no errors occurred. A non-zero exit status indicates a failure occurred.
-
-## COPYRIGHT
-
- * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
-
-## SEE ALSO
-
-`telemetry(3)`
-`telemetrics-client.conf(5)`
-
-https://github.com/clearlinux/telemetrics-client
-
-https://clearlinux.org/documentation/
-
-## NOTES
-
-Creative Commons Attribution-ShareAlike 3.0 Unported
-
- * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemd.1.rst

@@ -0,0 +1,78 @@
+======
+telemd
+======
+
+------------------------
+Telemetry client service
+------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telemd`` \<flags\>
+
+``/etc/telemetrics/telemetrics.conf``
+
+``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+``/etc/telemetrics/opt-in-static-machine-id``
+
+
+DESCRIPTION
+===========
+
+The ``telemd`` program delivers locally generated telemetry records to a remote
+telemetry service. Telemetry data can be in any format, and is relayed as-is.
+
+
+OPTIONS
+=======
+
+  * ``-f``, ``--config_file`` \[\<file\>\]:
+    Configuration file. This overides the other parameters.
+
+  * ``-h``, ``--help``:
+    Display this help message.
+
+  * ``-V``, ``--version``:
+    Print the program version.
+
+
+FILES
+=====
+
+* ``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+    If no custom configuration file is found, ``telemd`` uses the
+    settings in this file.
+
+* ``/etc/telemetrics/telemetrics.conf``
+
+    Custom configuration file that ``telemd`` reads. See ``telemetrics.conf``\(5).
+
+
+* ``/etc/telemetrics/opt-in-static-machine-id``
+
+    If this file exists, the first line of this file will be sent to
+    the server as the machine ID. Otherwise, the machine ID will be
+    random and changed regularly.
+
+
+EXIT STATUS
+===========
+
+0 when no errors occurred. A non-zero exit status indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemetry``\(3)
+* ``telemetrics.conf``\(5)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

docs/man/telemetrics.conf.5

@@ -1,105 +1,126 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.\" Man page generated from reStructuredText.
 .
-.TH "TELEMETRICS\.CONF" "5" "December 2016" "" ""
+.TH TELEMETRICS.CONF 5 "" "" ""
+.SH NAME
+telemetrics.conf \- Telemetry service configuration file
 .
-.SH "NAME"
-\fBtelemetrics\.conf\fR \- Telemetry service configuration file
+.nr rst2man-indent-level 0
 .
-.SH "SYNOPSIS"
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fB/etc/telemetrics/telemetrics.conf\fP
+.sp
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.SH DESCRIPTION
+.sp
+This file contains configuration parameters for the \fBtelemd\fP(1) telemetry
+service daemon. The daemon reads this file at startup if it exists.
+.SH SYNTAX
+.sp
+The configuration file contains \fBkey=value\fP pairs, formatted as plain
+text, one option per line. Comments can be added by preceding them with the
+\fB#\fP character. All configuration options should be in a section marked
+with \fB[settings]\fP\&.
+.SH OPTIONS
+.INDENT 0.0
+.IP \(bu 2
+\fBserver=<url>\fP
+.sp
+Server URL including protocol designator.
+.IP \(bu 2
+\fBsocket_path=<path>\fP
+.sp
+Path to the socket that \fItelemd\fP will listen on.
+.IP \(bu 2
+\fBcainfo=<path>\fP
+.sp
+Certificate file to use for validation of SSL endpoint.
+.IP \(bu 2
+\fBtidheader=<header>\fP
+.sp
+Telemetry ID in header format, usually \fBtidheader=X\-Telemetry\-TID:<uuid>\fP
+post header used to group records in ingestion service, which may be
+ingesting for more than one set of clients. Can be set to any string.
+.IP \(bu 2
+\fBrecord_expiry=<minutes>\fP
+.sp
+Record expiry time in minutes.
+.IP \(bu 2
+\fBspool_dir=<dir>\fP
+.sp
+Local spool directory used to store records being processed.
+.IP \(bu 2
+\fBspool_max_size=<kB>\fP
+.sp
+maximum size of the spool directory in kB. A value of \fB\-1\fP indicates
+no limit. The block size of the files in this directory is considered,
+and not the actual size of the record itself.
+.IP \(bu 2
+\fBspool_process_time=<seconds>\fP
+.sp
+Time in seconds for processing spool. Valid range: 120..3600. Values
+outside this range are clamped.
+.IP \(bu 2
+\fBrate_limit_enabled=<true|false>\fP
+.sp
+Enable rate limiting. If this is set to false then all rate\-limiting
+disabled. It is possible to disable each rate\-limit individually below.
+.IP \(bu 2
+\fBrecord_burst_limit=<limit>\fP
+.sp
+Rate limiting record burst limit. Valid Range:  0..\(ga\(gaINT_MAX\(ga\(ga, \-1 = disabled.
+.IP \(bu 2
+\fBrecord_window_length=<minutes>\fP
+.sp
+Rate limiting record window length in minutes. Valid Range: 0..59.
+.IP \(bu 2
+\fBbyte_burst_limit=<limit>\fP
+.sp
+Rate limiting byte burst limit. Valid Range:  0..\(gaINT_MAX\(ga, \-1 = disabled.
+.IP \(bu 2
+\fBbyte_window_length=<minutes>\fP
+.sp
+Rate limiting byte window length in minutes. Valid Range: 0..59.
+.IP \(bu 2
+\fBrate_limit_strategy=<strategy>\fP
+.sp
+Rate limit strategy \- what to do with record if rate\-limiting prevents
+delivery over network. Valid stategies: \fBspool\fP, \fBdrop\fP\&.
+.UNINDENT
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
 .
-.nf
-
-/etc/telemetrics/telemetrics\.conf
-/usr/share/defaults/telemetrics/telemetrics\.conf
-.
-.fi
-.
-.SH "DESCRIPTION"
-This file contains configuration parameters for the \fBtelemd(1)\fR telemetry service daemon\. The daemon reads this file at startup if it exists\.
-.
-.SH "SYNTAX"
-The configuration file contains \fBkey=value\fR pairs, formatted as plain text, one option per line\. Comments can be added by preceding them with the \fB#\fR character\. All configuration options should be in a section marked with \fB[settings]\fR\.
-.
-.SH "OPTIONS"
-.
-.TP
-\fBserver\fR=\fIurl\fR
-Server URL including protocol designator\.
-.
-.TP
-\fBsocket_path\fR=\fIpath\fR
-Path to the socket that \fBtelemd\fR will listen on\.
-.
-.TP
-\fBcainfo\fR=\fIpath\fR
-Certificate file to use for validation of SSL endpoint\.
-.
-.TP
-\fBtidheader\fR=\fIheader\fR
-Telemetry ID in header format, usually \fBtidheader=X\-Telemetry\-TID:<uuid>\fR post header used to group records in ingestion service, which may be ingesting for more than one set of clients\. Can be set to any string\.
-.
-.TP
-\fBrecord_expiry\fR=\fIminutes\fR
-Record expiry time in minutes\.
-.
-.TP
-\fBspool_dir\fR=\fI\fBdir\fR\fR
-Local spool directory used to store records being processed\.
-.
-.TP
-\fBspool_max_size\fR=\fIkB\fR
-maximum size of the spool directory in kB\. A value of \fB\-1\fR indicates no limit\. The block size of the files in this directory is considered, and not the actual size of the record itself\.
-.
-.TP
-\fBspool_process_time\fR=\fIseconds\fR
-Time in seconds for processing spool\. Valid range: 120\.\.3600\. Values outside this range are clamped\.
-.
-.TP
-\fBrate_limit_enabled\fR=\fI\fBtrue\fR|\fBfalse\fR\fR
-Enable rate limiting\. If this is set to false then all rate\-limiting disabled\. It is possible to disable each rate\-limit individually below\.
-.
-.TP
-\fBrecord_burst_limit\fR=\fIlimit\fR
-Rate limiting record burst limit\. Valid Range: 0\.\.\fBINT_MAX\fR, \-1 = disabled\.
-.
-.TP
-\fBrecord_window_length\fR=\fIminutes\fR
-Rate limiting record window length in minutes\. Valid Range: 0\.\.59\.
-.
-.TP
-\fBbyte_burst_limit\fR=\fIlimit\fR
-Rate limiting byte burst limit\. Valid Range: 0\.\.\fBINT_MAX\fR, \-1 = disabled\.
-.
-.TP
-\fBbyte_window_length\fR=\fIminutes\fR
-Rate limiting byte window length in minutes\. Valid Range: 0\.\.59\.
-.
-.TP
-\fBrate_limit_strategy\fR=\fIstrategy\fR
-Rate limit strategy \- what to do with record if rate\-limiting prevents delivery over network\. Valid stategies: \fBspool\fR, \fBdrop\fR\.
-.
-.SH "COPYRIGHT"
-.
-.IP "\(bu" 4
-Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
-.
-.IP "" 0
-.
-.SH "SEE ALSO"
-\fBtelemd(1)\fR
-.
-.P
-https://github\.com/clearlinux/telemetrics\-client
-.
-.P
-https://clearlinux\.org/documentation/
-.
-.SH "NOTES"
-Creative Commons Attribution\-ShareAlike 3\.0 Unported
-.
-.IP "\(bu" 4
-http://creativecommons\.org/licenses/by\-sa/3\.0/
-.
-.IP "" 0
-

docs/man/telemetrics.conf.5.md

@@ -1,92 +0,0 @@
-telemetrics.conf(5) -- Telemetry service configuration file
-===========================================================
-
-## SYNOPSIS
-
-```
-/etc/telemetrics/telemetrics.conf
-/usr/share/defaults/telemetrics/telemetrics.conf
-```
-
-## DESCRIPTION
-
-This file contains configuration parameters for the `telemd(1)` telemetry
-service daemon. The daemon reads this file at startup if it exists.
-
-## SYNTAX
-
-The configuration file contains `key=value` pairs, formatted as plain
-text, one option per line. Comments can be added by preceding them with the
-`#` character. All configuration options should be in a section marked
-with `[settings]`.
-
-## OPTIONS
-
- * `server`=<url>:
-   Server URL including protocol designator.
-
- * `socket_path`=<path>:
-   Path to the socket that `telemd` will listen on.
-
- * `cainfo`=<path>:
-  Certificate file to use for validation of SSL endpoint.
-
- * `tidheader`=<header>:
-  Telemetry ID in header format, usually
-    `tidheader=X-Telemetry-TID:<uuid>`
-  post header used to group records in ingestion service, which may be
-  ingesting for more than one set of clients. Can be set to any string.
-
- * `record_expiry`=<minutes>:
-  Record expiry time in minutes.
-
- * `spool_dir`=<`dir`>:
-  Local spool directory used to store records being processed.
-
- * `spool_max_size`=<kB>:
-  maximum size of the spool directory in kB. A value of `-1` indicates
-  no limit. The block size of the files in this directory is considered,
-  and not the actual size of the record itself.
-
- * `spool_process_time`=<seconds>:
-  Time in seconds for processing spool. Valid range: 120..3600. Values
-  outside this range are clamped.
-
- * `rate_limit_enabled`=<`true`|`false`>:
-  Enable rate limiting. If this is set to false then all rate-limiting
-  disabled. It is possible to disable each rate-limit individually below.
-
- * `record_burst_limit`=<limit>:
-  Rate limiting record burst limit. Valid Range:  0..`INT_MAX`, -1 = disabled.
-
- * `record_window_length`=<minutes>:
-  Rate limiting record window length in minutes. Valid Range: 0..59.
-
- * `byte_burst_limit`=<limit>:
-  Rate limiting byte burst limit. Valid Range:  0..`INT_MAX`, -1 = disabled.
-
- * `byte_window_length`=<minutes>:
-  Rate limiting byte window length in minutes. Valid Range: 0..59.
-
- * `rate_limit_strategy`=<strategy>:
-  Rate limit strategy - what to do with record if rate-limiting prevents
-  delivery over network. Valid stategies: `spool`, `drop`.
-
-
-## COPYRIGHT
-
- * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
-
-## SEE ALSO
-
-`telemd(1)`
-
-https://github.com/clearlinux/telemetrics-client
-
-https://clearlinux.org/documentation/
-
-## NOTES
-
-Creative Commons Attribution-ShareAlike 3.0 Unported
-
- * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemetrics.conf.5.rst

@@ -0,0 +1,110 @@
+================
+telemetrics.conf
+================
+
+------------------------------------
+Telemetry service configuration file
+------------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 5
+
+
+SYNOPSIS
+========
+
+``/etc/telemetrics/telemetrics.conf``
+
+``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+
+DESCRIPTION
+===========
+
+This file contains configuration parameters for the ``telemd``\(1) telemetry
+service daemon. The daemon reads this file at startup if it exists.
+
+
+SYNTAX
+======
+
+The configuration file contains ``key=value`` pairs, formatted as plain
+text, one option per line. Comments can be added by preceding them with the
+``#`` character. All configuration options should be in a section marked
+with ``[settings]``.
+
+
+OPTIONS
+=======
+
+-  ``server=<url>``
+
+   Server URL including protocol designator.
+
+-  ``socket_path=<path>``
+
+   Path to the socket that `telemd` will listen on.
+
+-  ``cainfo=<path>``
+
+   Certificate file to use for validation of SSL endpoint.
+
+-  ``tidheader=<header>``
+
+   Telemetry ID in header format, usually ``tidheader=X-Telemetry-TID:<uuid>``
+   post header used to group records in ingestion service, which may be
+   ingesting for more than one set of clients. Can be set to any string.
+
+-  ``record_expiry=<minutes>``
+
+   Record expiry time in minutes.
+
+-  ``spool_dir=<dir>``
+
+   Local spool directory used to store records being processed.
+
+-  ``spool_max_size=<kB>``
+
+   maximum size of the spool directory in kB. A value of ``-1`` indicates
+   no limit. The block size of the files in this directory is considered,
+   and not the actual size of the record itself.
+
+-  ``spool_process_time=<seconds>``
+
+   Time in seconds for processing spool. Valid range: 120..3600. Values
+   outside this range are clamped.
+
+-  ``rate_limit_enabled=<true|false>``
+
+   Enable rate limiting. If this is set to false then all rate-limiting
+   disabled. It is possible to disable each rate-limit individually below.
+
+-  ``record_burst_limit=<limit>``
+
+   Rate limiting record burst limit. Valid Range:  0..``INT_MAX``, -1 = disabled.
+
+-  ``record_window_length=<minutes>``
+
+   Rate limiting record window length in minutes. Valid Range: 0..59.
+
+-  ``byte_burst_limit=<limit>``
+
+   Rate limiting byte burst limit. Valid Range:  0..`INT_MAX`, -1 = disabled.
+
+-  ``byte_window_length=<minutes>``
+
+   Rate limiting byte window length in minutes. Valid Range: 0..59.
+
+-  ``rate_limit_strategy=<strategy>``
+
+   Rate limit strategy - what to do with record if rate-limiting prevents
+   delivery over network. Valid stategies: ``spool``, ``drop``.
+
+
+SEE ALSO
+========
+
+* ``telemd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

docs/man/telemetry.3

@@ -1,69 +1,84 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.\" Man page generated from reStructuredText.
 .
-.TH "TELEMETRY" "3" "December 2016" "" ""
+.TH TELEMETRY 3 "" "" ""
+.SH NAME
+telemetry \- C programming interface for telemetrics-client
 .
-.SH "NAME"
-\fBtelemetry\fR \- C programming interface for telemetrics\-client
+.nr rst2man-indent-level 0
 .
-.SH "SYNOPSIS"
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fB#include "telemetry.h"\fP
+.sp
+\fBstruct telem_ref { struct telem_record *record; };\fP
+.sp
+\fBint tm_create_record(struct telem_ref **t_ref, uint32_t severity, char *classification, uint32_t payload_version)\fP
+.sp
+\fBint tm_set_payload(struct telem_ref *t_ref, char *payload)\fP
+.sp
+\fBint tm_send_record(struct telem_ref *t_ref)\fP
+.sp
+\fBvoid tm_free_record(struct telem_ref *t_ref)\fP
+.sp
+\fBvoid tm_set_config_file(char *c_file)\fP
+.SH DESCRIPTION
+.sp
+The functions in the telemetry library facilitate the delivery of
+telemetry data to the \fBtelemd\fP(1) service.
+.sp
+The function \fBtm_create_record()\fP initializes a telemetry record and
+sets the severity and classification of that record, as well as the
+payload version number. The memory needed to store the telemetry record
+is allocated and should be freed with \fBtm_free_record()\fP when no longer
+needed.
+.sp
+The function \fBtm_set_payload()\fP attaches the provided telemetry record
+data to the telemetry record. The current maximum payload size is 8192b.
+.sp
+The function \fBtm_send_record()\fP delivers the record to the local
+\fBtelemd\fP(1) service.
+.sp
+The function \fBtm_set_config_file()\fP can be used to provide an alternate
+configuration path to the telemetry library.
+.SH RETURN VALUES
+.sp
+All these functions return \fB0\fP on success, or a non\-zero return value
+if an error occurred. The functions \fBtm_free_record()\fP and \fBtm_set_config_file()\fP
+do not return any values.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
 .
-.nf
-
-#include "telemetry\.h"
-
-struct telem_ref {
-        struct telem_record *record;
-};
-
-int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
-                     char *classification, uint32_t payload_version)
-int tm_set_payload(struct telem_ref *t_ref, char *payload)
-int tm_send_record(struct telem_ref *t_ref)
-void tm_free_record(struct telem_ref *t_ref)
-void tm_set_config_file(char *c_file)
-.
-.fi
-.
-.SH "DESCRIPTION"
-The functions in the telemetry library facilitate the delivery of telemetry data to the \fBtelemd(1)\fR service\.
-.
-.P
-The function \fBtm_create_record()\fR initializes a telemetry record and sets the severity and classification of that record, as well as the payload version number\. The memory needed to store the telemetry record is allocated and should be freed with \fBtm_free_record()\fR when no longer needed\.
-.
-.P
-The function \fBtm_set_payload()\fR attaches the provided telemetry record data to the telemetry record\. The current maximum payload size is 8192b\.
-.
-.P
-The function \fBtm_send_record()\fR delivers the record to the local \fBtelemd(1)\fR service\.
-.
-.P
-The function \fBtm_set_config_file()\fR can be used to provide an alternate configuration path to the telemetry library\.
-.
-.SH "RETURN VALUES"
-All these functions return \fB0\fR on success, or a non\-zero return value if an error occurred\. The functions \fBtm_free_record()\fR and \fBtm_set_config_file()\fR do not return any values\.
-.
-.SH "COPYRIGHT"
-.
-.IP "\(bu" 4
-Copyright (C) 2016 Intel Corporation, License: CC\-BY\-SA\-3\.0
-.
-.IP "" 0
-.
-.SH "SEE ALSO"
-\fBtelemd(1)\fR
-.
-.P
-https://github\.com/clearlinux/telemetrics\-client
-.
-.P
-https://clearlinux\.org/documentation/
-.
-.SH "NOTES"
-Creative Commons Attribution\-ShareAlike 3\.0 Unported
-.
-.IP "\(bu" 4
-http://creativecommons\.org/licenses/by\-sa/3\.0/
-.
-.IP "" 0
-

docs/man/telemetry.3.md

@@ -1,63 +0,0 @@
-telemetry(3) -- C programming interface for telemetrics-client
-==============================================================
-
-## SYNOPSIS
-
-```
-#include "telemetry.h"
-
-struct telem_ref {
-        struct telem_record *record;
-};
-
-int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
-                     char *classification, uint32_t payload_version)
-int tm_set_payload(struct telem_ref *t_ref, char *payload)
-int tm_send_record(struct telem_ref *t_ref)
-void tm_free_record(struct telem_ref *t_ref)
-void tm_set_config_file(char *c_file)
-```
-
-## DESCRIPTION
-
-The functions in the telemetry library facilitate the delivery of
-telemetry data to the `telemd(1)` service.
-
-The function `tm_create_record()` initializes a telemetry record and
-sets the severity and classification of that record, as well as the
-payload version number. The memory needed to store the telemetry record
-is allocated and should be freed with `tm_free_record()` when no longer
-needed.
-
-The function `tm_set_payload()` attaches the provided telemetry record
-data to the telemetry record. The current maximum payload size is 8192b.
-
-The function `tm_send_record()` delivers the record to the local
-`telemd(1)` service.
-
-The function `tm_set_config_file()` can be used to provide an alternate
-configuration path to the telemetry library.
-
-## RETURN VALUES
-
-All these functions return `0` on success, or a non-zero return value
-if an error occurred. The functions `tm_free_record()` and `tm_set_config_file()`
-do not return any values.
-
-## COPYRIGHT
-
- * Copyright (C) 2016 Intel Corporation, License: CC-BY-SA-3.0
-
-## SEE ALSO
-
-`telemd(1)`
-
-https://github.com/clearlinux/telemetrics-client
-
-https://clearlinux.org/documentation/
-
-## NOTES
-
-Creative Commons Attribution-ShareAlike 3.0 Unported
-
- * http://creativecommons.org/licenses/by-sa/3.0/

docs/man/telemetry.3.rst

@@ -0,0 +1,67 @@
+=========
+telemetry
+=========
+
+----------------------------------------------
+C programming interface for telemetrics-client
+----------------------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 3
+
+
+SYNOPSIS
+========
+
+``#include "telemetry.h"``
+  
+``struct telem_ref { struct telem_record *record; };``
+  
+``int tm_create_record(struct telem_ref **t_ref, uint32_t severity, char *classification, uint32_t payload_version)``
+
+``int tm_set_payload(struct telem_ref *t_ref, char *payload)``
+
+``int tm_send_record(struct telem_ref *t_ref)``
+
+``void tm_free_record(struct telem_ref *t_ref)``
+
+``void tm_set_config_file(char *c_file)``
+
+
+DESCRIPTION
+===========
+
+The functions in the telemetry library facilitate the delivery of
+telemetry data to the ``telemd``\(1) service.
+
+The function ``tm_create_record()`` initializes a telemetry record and
+sets the severity and classification of that record, as well as the
+payload version number. The memory needed to store the telemetry record
+is allocated and should be freed with ``tm_free_record()`` when no longer
+needed.
+
+The function ``tm_set_payload()`` attaches the provided telemetry record
+data to the telemetry record. The current maximum payload size is 8192b.
+
+The function ``tm_send_record()`` delivers the record to the local
+``telemd``\(1) service.
+
+The function ``tm_set_config_file()`` can be used to provide an alternate
+configuration path to the telemetry library.
+
+
+RETURN VALUES
+=============
+
+All these functions return ``0`` on success, or a non-zero return value
+if an error occurred. The functions ``tm_free_record()`` and ``tm_set_config_file()``
+do not return any values.
+
+
+SEE ALSO
+========
+
+* ``telemd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

scripts/source-indent.sh

@@ -14,6 +14,6 @@ if [ "$?" -ne 0 ]; then
   exit 1
 fi
 
-find .. -path "../*/*.[ch]" -exec uncrustify {} -c tm.cfg --no-backup \;
+find .. -path "../*/*.[ch]" ! -path "../src/nica/*" -exec uncrustify {} -c tm.cfg --no-backup \;
 
 # vi: ts=8 sw=2 sts=2 et tw=80

scripts/telemctl

@@ -18,6 +18,17 @@ declare -a SERVICES=(
 SCRIPT="$0"
 TELEM_DIR=/etc/telemetrics
 OPT_OUT_FILE=${TELEM_DIR}/opt-out
+TELEM_WRK_DIRS_CONF=/usr/lib/tmpfiles.d/telemetrics-dirs.conf
+
+create_work_dirs() {
+  # Creates dirs if missing, adjust ownership if exists
+  systemd-tmpfiles --create ${TELEM_WRK_DIRS_CONF}
+}
+
+telem_remove_work_dirs() {
+  # Remove dirs
+  awk '/^d/{print $2}' ${TELEM_WRK_DIRS_CONF} | xargs rm -rf;
+}
 
 exit_ok() {
   echo "$1" > /dev/stderr
@@ -50,15 +61,23 @@ telem_stop() {
 
 telem_start() {
   [ -f $OPT_OUT_FILE ] && exit_err "Opt out is enabled. Cannot start services."
+  # trigger systemd-tmpfiles work dirs creation
+  create_work_dirs
   # the units in SERVICES are activated as needed, so no need to start them
   for_each_service "start" ${SPECIAL_UNITS[@]}
 }
 
+telem_is_active() {
+  # check only activation units
+  systemctl is-active telemd.socket
+}
+
 telem_opt_out() {
   [ -f $OPT_OUT_FILE ] && exit_ok "Already opted out. Nothing to do."
   mkdir -p $TELEM_DIR || exit_err "Failed to create ${TELEM_DIR}."
   touch $OPT_OUT_FILE || exit_err "Failed to create ${OPT_OUT_FILE}."
   telem_stop
+  telem_remove_work_dirs
 }
 
 telem_opt_in() {
@@ -72,20 +91,29 @@ telem_restart() {
   telem_start
 }
 
+telem_journal_cli() {
+  telem_journal "$@"
+}
+
 usage() {
-  printf "$SCRIPT - Control actions for telemetry services\n"
+  format='  %-10s %s\n'
   printf "\n"
-  printf "%15s   %-65s\n" "stop" "Stops all running telemetry services"
-  printf "%15s   %-65s\n" "start" "Starts all telemetry services"
-  printf "%15s   %-65s\n" "restart" "Restarts all telemetry services"
-  printf "%15s   %-65s\n" "opt-in" "Opts in to telemetry, and starts telemetry services"
-  printf "%15s   %-65s\n" "opt-out" "Opts out of telemetry, and stops telemetry services"
+  printf "%s - Control actions for telemetry services\n" "$SCRIPT"
+  printf "\n"
+  printf "$format" "stop" "Stops all running telemetry services"
+  printf "$format" "start" "Starts all telemetry services"
+  printf "$format" "restart" "Restarts all telemetry services"
+  printf "$format" "is-active" "Checks if telemd is active"
+  printf "$format" "opt-in" "Opts in to telemetry, and starts telemetry services"
+  printf "$format" "opt-out" "Opts out of telemetry, and stops telemetry services"
+  printf "$format" "journal" "Prints telemd journal contents. Use -h argument with"
+  printf "$format" ""        "command for more options"
   printf "\n"
   exit 2
 }
 
-if [ $# -ne 1 ]; then
-  usage
+if [ "$1" != "journal" ] && [ $# -ne 1 ]; then
+     usage
 fi
 
 if [ $EUID -ne 0 ]; then
@@ -105,6 +133,10 @@ case $SUBCOMMAND in
     telem_start ;;
   restart)
     telem_restart ;;
+  is-active)
+    telem_is_active ;;
+  journal)
+    telem_journal_cli "$@" ;;
   *)
     notice "Unknown command passed to $SCRIPT"
     usage ;;

src/common.c

@@ -33,7 +33,11 @@ static const char *header_names[] = {
         TM_SYSTEM_BUILD_STR,
         TM_KERNEL_VERSION_STR,
         TM_PAYLOAD_VERSION_STR,
-        TM_SYSTEM_NAME_STR
+        TM_SYSTEM_NAME_STR,
+        TM_BOARD_NAME_STR,
+        TM_CPU_MODEL_STR,
+        TM_BIOS_VERSION_STR,
+        TM_EVENT_ID_STR
 };
 
 const char *get_header_name(int ind)

src/common.h

@@ -35,6 +35,10 @@
 #define TM_KERNEL_VERSION 8
 #define TM_PAYLOAD_VERSION 9
 #define TM_SYSTEM_NAME 10
+#define TM_BOARD_NAME 11
+#define TM_CPU_MODEL 12
+#define TM_BIOS_VERSION 13
+#define TM_EVENT_ID 14
 
 #define TM_RECORD_VERSION_STR "record_format_version"
 #define TM_CLASSIFICATION_STR "classification"
@@ -47,14 +51,26 @@
 #define TM_KERNEL_VERSION_STR "kernel_version"
 #define TM_PAYLOAD_VERSION_STR "payload_format_version"
 #define TM_SYSTEM_NAME_STR "system_name"
+#define TM_BOARD_NAME_STR "board_name"
+#define TM_CPU_MODEL_STR "cpu_model"
+#define TM_BIOS_VERSION_STR "bios_version"
+#define TM_EVENT_ID_STR "event_id"
 
-#define NUM_HEADERS 11
+#define NUM_HEADERS 15
+
+#define EVENT_ID_ALPHAB "0123456789abcdef"
+#define EVENT_ID_LEN 32
+
+/* Journal */
+#define JOURNAL_PATH "/var/log/telemetry/journal"
+#define JOURNAL_TMPDIR  "/var/log/telemetry/"
+#define RECORD_RETENTION_DIR "/var/log/telemetry/records"
 
 /* For internal library usage. Bump the version whenever we change the record
  * structure (e.g. adding or removing a header field). Note that the value
  * should be an unsigned int.
  */
-static const uint32_t RECORD_FORMAT_VERSION = 2;
+static const uint32_t RECORD_FORMAT_VERSION = 4;
 
 #define TM_SITE_VERSION_FILE "/etc/os-release"
 #define TM_DIST_VERSION_FILE "/usr/lib/os-release"

src/configuration.c

@@ -16,22 +16,25 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <glib.h>
 #include <errno.h>
 #include <limits.h>
 #include <dirent.h>
 #include <string.h>
 #include <ctype.h>
 #include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
 
 #include "configuration.h"
 #include "util.h"
 #include "common.h"
 
+#include "nica/inifile.h"
+
 static char *config_file = NULL;
 static char *default_config_file = DATADIR "/defaults/telemetrics/telemetrics.conf";
 static char *etc_config_file = "/etc/telemetrics/telemetrics.conf";
-static GKeyFile *keyfile = NULL;
+static NcHashmap *keyfile = NULL;
 static bool cmd_line_cfg = false;
 
 /* Conf strings, integers, and booleans expected in the conf file */
@@ -44,8 +47,8 @@ const char *config_key_int[] = { NULL, "record_expiry", "spool_max_size",
                                  "byte_window_length", "record_burst_limit",
                                  "byte_burst_limit", NULL };
 
-const char *config_key_bool[] = { NULL, "rate_limit_enabled",
-				  "daemon_recycling_enabled", NULL };
+const char *config_key_bool[] = { NULL, "rate_limit_enabled", "daemon_recycling_enabled",
+                                  "record_retention_enabled", "record_server_delivery_enabled", NULL };
 
 static struct configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
 
@@ -81,42 +84,70 @@ void set_config_file(char *filename)
 
 bool read_config_from_file(char *config_file, struct configuration *config)
 {
-        GError *error = NULL;
-        GKeyFileFlags flags = G_KEY_FILE_NONE;
-
         if (keyfile != NULL) {
-                g_key_file_free(keyfile);
+                nc_hashmap_free(keyfile);
         }
 
-        keyfile = g_key_file_new();
-
-        if (!g_key_file_load_from_file(keyfile, (gchar *)config_file, flags, &error)) {
+        keyfile = nc_ini_file_parse(config_file);
+        if (!keyfile) {
 #ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to read config file: %s\n",
-                        error->message);
+                fprintf(stderr, "ERR: Failed to read config file\n");
 #endif
                 return false;
         } else {
                 for (int i = CONF_STR_MIN + 1; i < CONF_STR_MAX; i++) {
-                        config->strValues[i] = g_key_file_get_string(keyfile,
-                                                                     "settings", config_key_str[i], &error);
-                        if (error != NULL) {
+                        char *ptr;
+                        ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_str[i]);
+                        if (ptr) {
+                                config->strValues[i] = strdup(ptr);
+                                if (config->strValues[i] == NULL) {
+                                        return false;
+                                }
+                        } else {
+                                fprintf(stderr, "ERR: missing key with string value: %s\n", config_key_str[i]);
                                 return false;
                         }
                 }
                 for (int i = CONF_INT_MIN + 1; i < CONF_INT_MAX; i++) {
-                        config->intValues[i] = g_key_file_get_int64(keyfile,
-                                                                    "settings", config_key_int[i], &error);
-                        if (error != NULL) {
+                        char *ptr;
+                        ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_int[i]);
+                        if (ptr) {
+                                config->intValues[i] = strtoll(ptr, NULL, 10);
+                        } else {
+                                fprintf(stderr, "ERR: missing key with integer value: %s\n", config_key_int[i]);
                                 return false;
                         }
                 }
 
                 for (int i = CONF_BOOL_MIN + 1; i < CONF_BOOL_MAX; i++) {
-                        config->boolValues[i] = g_key_file_get_boolean(keyfile,
-                                                                       "settings", config_key_bool[i], &error);
-                        if (error != NULL) {
-                                return false;
+                        char *ptr;
+                        ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_bool[i]);
+                        config->boolValues[i] = false;
+
+                        if (ptr) {
+                                if (strcasecmp(ptr, "TRUE") == 0) {
+                                        config->boolValues[i] = true;
+                                }
+                                if (strcasecmp(ptr, "1") == 0) {
+                                        config->boolValues[i] = true;
+                                }
+                        } else {
+                                /* New configuration keys, CONF_RECORD_RETENTION_ENABLED and CONF_RECORD_SERVER_DELIVERY_ENABLED
+                                 * default values, otherwise update will require changes in custom configurations */
+                                if (i == CONF_RECORD_RETENTION_ENABLED) {
+#ifdef DEBUG
+                                        fprintf(stderr, "WARN: missing boolean optional key: %s\n", config_key_bool[i]);
+#endif
+                                        config->boolValues[i] = RECORD_RETENTION_ENABLED_DEFAULT;
+                                } else if (i == CONF_RECORD_SERVER_DELIVERY_ENABLED) {
+#ifdef DEBUG
+                                        fprintf(stderr, "WARN: missing boolean optional key: %s\n", config_key_bool[i]);
+#endif
+                                        config->boolValues[i] = RECORD_SERVER_DELIVERY_ENABLED_DEFAULT;
+                                } else {
+                                        fprintf(stderr, "ERR: missing key with boolean value: %s\n", config_key_bool[i]);
+                                        return false;
+                                }
                         }
                 }
         }
@@ -133,11 +164,11 @@ void initialize_config(void)
 
         /* No config file provided on command line */
         if (!config_file) {
-                if (g_file_test(etc_config_file, G_FILE_TEST_IS_REGULAR)) {
+                if (access(etc_config_file, R_OK) == 0) {
                         config_file = etc_config_file;
                 } else {
-                        if (g_file_test(default_config_file,
-                                        G_FILE_TEST_IS_REGULAR)) {
+                        if (access(default_config_file,
+                                   R_OK) == 0) {
                                 config_file = default_config_file;
                         } else {
                                 /* If there is no default config, exit with failure */
@@ -179,8 +210,6 @@ void free_configuration(void)
                 free(config.strValues[i]);
         }
 
-        g_key_file_free(keyfile);
-
         if (cmd_line_cfg) {
                 free(config_file);
         }
@@ -338,7 +367,19 @@ const char *rate_limit_strategy_config()
 
 bool daemon_recycling_enabled_config(void)
 {
-	initialize_config();
-	return config.boolValues[CONF_DAEMON_RECYCLING_ENABLED];
+        initialize_config();
+        return config.boolValues[CONF_DAEMON_RECYCLING_ENABLED];
+}
+
+bool record_retention_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_RETENTION_ENABLED];
+}
+
+bool record_server_delivery_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED];
 }
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/configuration.h

@@ -19,6 +19,8 @@
 #include <stdint.h>
 
 #define TM_MAX_WINDOW_LENGTH (1 /*h*/ * 60 /*m*/)
+#define RECORD_RETENTION_ENABLED_DEFAULT false
+#define RECORD_SERVER_DELIVERY_ENABLED_DEFAULT true
 
 enum config_str_keys {
         CONF_STR_MIN = 0,
@@ -46,7 +48,9 @@ enum config_int_keys {
 enum config_bool_keys {
         CONF_BOOL_MIN = 0,
         CONF_RATE_LIMIT_ENABLED,
-	CONF_DAEMON_RECYCLING_ENABLED,
+        CONF_DAEMON_RECYCLING_ENABLED,
+        CONF_RECORD_RETENTION_ENABLED,
+        CONF_RECORD_SERVER_DELIVERY_ENABLED,
         CONF_BOOL_MAX
 };
 
@@ -117,4 +121,10 @@ const char *get_tidheader_config(void);
 /* Gets whether recycling is enabled */
 bool daemon_recycling_enabled_config(void);
 
+/* Gets whether local retention of records is enabled */
+bool record_retention_enabled_config(void);
+
+/* Gets whether records should be sent to server_addr */
+bool record_server_delivery_enabled_config(void);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/data/bert-probe.service.in

@@ -0,0 +1,13 @@
+[Unit]
+Description=Telemetrics BERT Probe
+Requires=telemd.socket
+After=telemd.socket
+ConditionPathExists=!/etc/telemetrics/opt-out
+ConditionPathExists=/sys/firmware/acpi/tables/data/BERT
+
+[Service]
+ExecStart=@bindir@/bertprobe
+User=telemetry
+
+[Install]
+WantedBy=multi-user.target

src/data/example.1.conf

@@ -0,0 +1,46 @@
+[settings]
+server=http://127.0.0.1
+
+cainfo=/tmp/cacert.crt
+
+tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
+
+socket_path=/tmp/test_telem_socket
+
+#record expiry time in minutes
+record_expiry=1200
+
+spool_dir=/tmp/spool
+
+#maximum size of the spool directory in KB
+spool_max_size=1024
+
+#time in seconds for processing spool
+spool_process_time=900
+
+#rate limiting record burst limit
+record_burst_limit=100
+
+#rate limiting record window length
+record_window_length = 15
+
+#rate limiting byte burst limit
+byte_burst_limit = 1000
+
+#rate limiting byte window length
+byte_window_length=20
+
+#rate limit enabled
+rate_limit_enabled=true
+
+#rate limit strategy if record not send
+rate_limit_strategy=spool
+
+#daemon recycling enabled
+daemon_recycling_enabled=true
+
+#server delivery disable
+record_server_delivery_enabled=false
+
+#local copy enabled
+record_retention_enabled=true

src/data/local.mk

@@ -4,14 +4,17 @@ pathfix = @sed \
 	-e 's|@libdir[@]|$(libdir)|g' \
 	-e 's|@localstatedir[@]|$(localstatedir)|g' \
 	-e 's|@PACKAGE_VERSION[@]|$(PACKAGE_VERSION)|g' \
-	-e 's|@SOCKETDIR[@]|$(SOCKETDIR)|g'
+	-e 's|@SOCKETDIR[@]|$(SOCKETDIR)|g' \
+	-e 's|@systemctldir[@]|$(SYSTEMD_SYSTEMCTLDIR)|g'
 
 EXTRA_DIST += \
 	%D%/40-core-ulimit.conf \
 	%D%/40-crash-probe.conf.in \
 	%D%/example.conf \
+	%D%/example.1.conf \
 	%D%/hprobe.service.in \
 	%D%/hprobe.timer \
+	%D%/bert-probe.service.in \
 	%D%/journal-probe.service.in \
 	%D%/oops-probe.service.in \
 	%D%/pstore-probe.service.in \
@@ -21,6 +24,7 @@ EXTRA_DIST += \
 	%D%/telemd.path.in \
 	%D%/telemd.service.in \
 	%D%/telemd.socket.in \
+	%D%/telemd-update-trigger.service.in \
 	%D%/telemetrics-dirs.conf.in \
 	%D%/telemetrics.conf.in
 
@@ -46,6 +50,7 @@ systemdunitdir = @SYSTEMD_UNITDIR@
 systemdunit_DATA = \
 	%D%/hprobe.service \
 	%D%/hprobe.timer \
+	%D%/bert-probe.service \
 	%D%/journal-probe.service \
 	%D%/oops-probe.service \
 	%D%/pstore-probe.service \
@@ -53,11 +58,15 @@ systemdunit_DATA = \
 	%D%/pstore-clean.service \
 	%D%/telemd.service \
 	%D%/telemd.socket \
+	%D%/telemd-update-trigger.service \
 	%D%/telemd.path
 
 %D%/hprobe.service: %D%/hprobe.service.in
 	$(pathfix) < $< > $@
 
+%D%/bert-probe.service: %D%/bert-probe.service.in
+	$(pathfix) < $< > $@
+
 %D%/journal-probe.service: %D%/journal-probe.service.in
 	$(pathfix) < $< > $@
 
@@ -82,6 +91,9 @@ systemdunit_DATA = \
 %D%/telemd.path: %D%/telemd.path.in
 	$(pathfix) < $< > $@
 
+%D%/telemd-update-trigger.service: %D%/telemd-update-trigger.service.in
+	$(pathfix) < $< > $@
+
 sysctldir = @SYSTEMD_SYSCTLDIR@
 sysctl_DATA = %D%/40-crash-probe.conf
 
@@ -95,6 +107,7 @@ clean-local:
 	-rm -f  %D%/telemd.service \
 		%D%/telemd.socket \
 		%D%/telemd.path \
+		%D%/telemd-update-trigger.service \
 		%D%/telemetrics.conf \
 		%D%/telemetrics-dirs.conf \
 		%D%/libtelemetry.pc \
@@ -105,4 +118,5 @@ clean-local:
 		%D%/klogscanner.service \
 		%D%/pstore-clean.service \
 		%D%/hprobe.service \
+		%D%/bert-probe.service \
 		%D%/.dirstamp

src/data/telemd-update-trigger.service.in

@@ -0,0 +1,7 @@
+[Unit]
+Description=Restart Telemetrics Daemon On Update
+BindsTo=update-triggers.target
+
+[Service]
+Type=oneshot
+ExecStart=@systemctldir@/systemctl try-restart telemd.service

src/data/telemetrics-dirs.conf.in

@@ -1,5 +1,7 @@
 d @localstatedir@/lib/telemetry 0755 telemetry telemetry -
 d @localstatedir@/spool/telemetry 0750 telemetry telemetry -
+d @localstatedir@/log/telemetry 0750 telemetry telemetry -
+d @localstatedir@/log/telemetry/records 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry/oops 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry/pstore 0750 telemetry telemetry -

src/data/telemetrics.conf.in

@@ -55,3 +55,13 @@ rate_limit_strategy=spool
 # has not any client nor spool data, then it exits.
 # this is to ensure that latest code runs.
 daemon_recycling_enabled=true
+
+# record server delivery enabled - when enabled records will be delivered
+# to server otherwise records will be ignored. This configuration can be used
+# with 'record_retention_enable' configuration value to keep records local only.
+record_server_delivery_enabled=true
+
+# record retention enabled - when enabled a copy of reported telemetry records
+# will be kept locally. This configuration combined with 'record_server_delivery_enabled'
+# value can be used to keep records local only.
+record_retention_enabled=false

src/journal/cli.c

@@ -0,0 +1,112 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <getopt.h>
+
+#include "common.h"
+#include "journal.h"
+
+void print_usage(void)
+{
+        printf(" Usage\n");
+        printf("  -r,  --record_id        Print record with specific record_id\n");
+        printf("  -e,  --event_id         Print records with specific event_id\n");
+        printf("  -c,  --classification   Print records with specific classification\n");
+        printf("  -b,  --boot_id          Print records with specific boot_id\n");
+        printf("  -i,  --include_record   Include record content\n");
+        printf("  -V,  --verbose          Verbose output\n");
+        printf("  -h,  --help             Display this help message\n");
+}
+
+int main(int argc, char **argv)
+{
+
+        int rc = EXIT_SUCCESS;
+        int count = 0;
+        int verbose_output = 0;
+        int record = 0;
+        char *boot_id = NULL;
+        char *record_id = NULL;
+        char *event_id = NULL;
+        char *classification = NULL;
+        struct TelemJournal *telem_journal = NULL;
+
+        /** opts */
+        int c;
+        int opt_index = 0;
+        struct option opts[] = {
+                { "record_id", 1, NULL, 'r' },
+                { "event_id", 1, NULL, 'e' },
+                { "classification", 1, NULL, 'c' },
+                { "boot_id", 1, NULL, 'b' },
+                { "verbose", 0, NULL, 'V' },
+                { "include_record", 0, NULL, 'i' },
+                { "help", 0, NULL, 'h' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "r:e:c:b:Vih", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'r':
+                                record_id = optarg;
+                                break;
+                        case 'e':
+                                event_id = optarg;
+                                break;
+                        case 'c':
+                                classification = optarg;
+                                break;
+                        case 'b':
+                                boot_id = optarg;
+                                break;
+                        case 'V':
+                                verbose_output = 1;
+                                break;
+                        case 'i':
+                                record = 1;
+                                break;
+                        case 'h':
+                                print_usage();
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                /** default */
+                                print_usage();
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        if ((telem_journal = open_journal(JOURNAL_PATH))) {
+                if (verbose_output) {
+                        fprintf(stdout, "%-30s %-27s %-32s %-32s %-36s\n", "Classification", "Time stamp",
+                                "Record ID", "Event ID", "Boot ID");
+                }
+                count = print_journal(telem_journal, classification, record_id, event_id, boot_id, record);
+                if (verbose_output) {
+                        fprintf(stdout, "Total records: %d\n", count);
+                }
+                close_journal(telem_journal);
+        } else {
+                fprintf(stderr, "Unable to open journal\n");
+                rc = EXIT_FAILURE;
+        }
+
+        return rc;
+}

src/journal/journal.c

@@ -0,0 +1,709 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#define ID_LEN 32
+#define BOOTID_LEN 37 // Includes the \n character at the end
+#define BOOTID_FILE "/proc/sys/kernel/random/boot_id"
+#define MID_BUFF 1024
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/uio.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include "log.h"
+#include "util.h"
+#include "common.h"
+#include "journal.h"
+
+/*
+ * Packs journal entry values into one line string separated
+ * by '\036' RS.
+ *
+ * @param entry A pointer to a JournalEntry struct with values
+ *              to serialize.
+ * @param buff A pointer to be initialized and set with the
+ *             serialized values.
+ *
+ * @return 0 on success, -1 on failure.
+ */
+static int serialize_journal_entry(struct JournalEntry *entry, char **buff)
+{
+        int rc = 0;
+
+        if (entry == NULL) {
+                return -1;
+        }
+
+        if (asprintf(buff, "%s\036%lld\036%s\036%s\036%s", entry->record_id, (long long int)entry->timestamp,
+                     entry->classification, entry->event_id, entry->boot_id) < 0) {
+#ifdef DEBUG
+                fprintf(stderr, "Error: unable to serialize data in buff\n");
+#endif
+                rc = -1;
+        }
+
+        return rc;
+}
+
+/**
+ *  Frees journal entry struct members and journal entry pointer.
+ *
+ *  @param the pointer to the entry structure to free.
+ */
+static void free_journal_entry(struct JournalEntry *entry)
+{
+        if (entry) {
+                free(entry->classification);
+                free(entry->record_id);
+                free(entry->event_id);
+                free(entry->boot_id);
+                free(entry);
+        }
+}
+
+/**
+ * Unpacks a record that was stored in file and initializes a
+ * JournalEntry struct with parsed values.
+ *
+ * @param line A pointer to the data to unpack.
+ * @param entry A pointer to be initialized and set with values
+ *        parsed from parameter line.
+ *
+ * @return 0 on success, -1 on failure.
+ */
+static int deserialize_journal_entry(char *line, struct JournalEntry **entry)
+{
+        int rc = -1;
+        size_t len = 0;
+        size_t llen = 0;
+        size_t offset = 0;
+        struct JournalEntry *e = NULL;
+
+        if (line == NULL) {
+                return -1;
+        }
+
+        e = malloc(sizeof(struct JournalEntry));
+        if (!e) {
+                return -1;
+        }
+
+        llen = strlen(line);
+
+        for (int i = 0; i < 5; i++) {
+                char *out = NULL;
+                if (offset > llen) {
+                        rc = -1;
+                        break;
+                }
+                len = strcspn(line + offset, "\036\n");
+                out = strndup((line + offset), len);
+                offset += len + 1;
+                if (out) {
+                        switch (i) {
+                                case 0:
+                                        e->record_id = out;
+                                        break;
+                                case 1:
+                                        e->timestamp = atoll(out);
+                                        free(out);
+                                        break;
+                                case 2:
+                                        e->classification = out;
+                                        break;
+                                case 3:
+                                        e->event_id = out;
+                                        break;
+                                case 4:
+                                        e->boot_id = out;
+                                        rc = 0;
+                                        break;
+                                default:
+                                        assert(i < 0 && i > 4);
+                                        free(out);
+                                        rc = 1;
+                        }
+                } else {
+                        rc = -1;
+                        break;
+                }
+        }
+
+        if (rc == 0) {
+                *entry = e;
+        } else {
+                free_journal_entry(e);
+        }
+
+        return rc;
+}
+
+/**
+ * Counts the number of records in journal.
+ *
+ * @param telem_journal Pointer to a telemetry journal
+ *        to get record count.
+ *
+ * @return record count
+ *
+ */
+static int get_record_count(struct TelemJournal *telem_journal)
+{
+        int count = 0;
+        size_t len;
+        ssize_t read;
+        char *line = NULL;
+
+        rewind(telem_journal->fptr);
+        // read until the end line by line
+        while ((read = getline(&line, &len, telem_journal->fptr)) != -1) {
+                count++;
+        }
+        free(line);
+
+        return count;
+}
+
+/**
+ * Serializes JournalEntry and saves it to file.
+ *
+ * @param fptr A FILE pointer.
+ * @param entry A pointer to data.
+ *
+ * @return 0 on succes, !=0 on failure
+ */
+static int save_entry(FILE *fptr, struct JournalEntry *entry)
+{
+        int rc = -1;
+        char *serialized_data = NULL;
+
+        if (fptr == NULL || entry == NULL) {
+                return rc;
+        }
+
+        if ((rc = serialize_journal_entry(entry, &serialized_data)) == 0) {
+#ifdef DEBUG
+                printf("Saving: %s\n", serialized_data);
+#endif
+                fprintf(fptr, "%s\n", serialized_data);
+                fflush(fptr);
+                free(serialized_data);
+        }
+
+        return rc;
+}
+
+/**
+ * Reads the boot unique identifier from BOOTID_FILE
+ *
+ * @param buff pointer to a BOOTID_LEN allocated
+ *
+ * @return 0 on success, -1 on failure
+ */
+static int read_boot_id(char buff[])
+{
+        int rc = -1;
+        FILE *fs = NULL;
+
+        fs = fopen(BOOTID_FILE, "r");
+        if (!fs) {
+#ifdef DEBUG
+                fprintf(stderr, "Unable to open %s for reading: %d\n", BOOTID_FILE, errno);
+#endif
+                return rc;
+        }
+
+        if (fgets(buff, BOOTID_LEN, fs)) {
+                rc = 0;
+        }
+        fclose(fs);
+
+        return rc;
+}
+
+/**
+ * Moves the file position indicator n lines.
+ *
+ * @param n Number of lines to skip.
+ * @param fptr A file pointer.
+ * @param found_record A pointer to a function to be called
+ *                     every time a valid record is found, if
+ *                     pointer is not set to null.
+ *
+ * @returns 0 on success, on failure -1 if n > existing
+ *            lines in file or errno if failure happens
+ *            during file operation.
+ */
+static int skip_n_lines(int n, FILE *fptr, int (*found_record)(char *))
+{
+        int rc = 0;
+        size_t len = 0;
+        char *line = NULL;
+        struct JournalEntry *entry = NULL;
+
+        if (fptr == NULL) {
+#ifdef DEBUG
+                fprintf(stderr, "fptr argument to skip_n_lines is invalid\n");
+#endif
+                // File descriptor in bad state
+                return EBADFD;
+        }
+
+        if (fseek(fptr, 0, 0) != 0) {
+                return errno;
+        }
+
+        for (int i = 0; i < n; i++) {
+                if (getline(&line, &len, fptr) == -1) {
+                        rc = -1;
+                        break;
+                }
+                if (found_record != NULL) {
+                        deserialize_journal_entry(line, &entry);
+                        if (entry) {
+                                found_record(entry->record_id);
+                                free_journal_entry(entry);
+                                entry = NULL;
+                        }
+                }
+        }
+        free(line);
+
+        return rc;
+}
+
+/**
+ * Copies contents of fptr to a known location.
+ *
+ * @param fptr A file pointer from source file.
+ *
+ * @returns 0 on success, errno on failure
+ */
+static int copy_to_tmp(FILE *fptr, char *tmp_path)
+{
+        int rc = 0;
+        size_t len;
+        char *line = NULL;
+        FILE *fptr_tmp = NULL;
+
+        if (fptr == NULL) {
+                return -1;
+        }
+
+        fptr_tmp = fopen(tmp_path, "w");
+        if (!fptr_tmp) {
+                return errno;
+        }
+
+        // Copy lines to new file
+        while (getline(&line, &len, fptr) > -1) {
+                if (fprintf(fptr_tmp, "%s", line) < 0) {
+                        rc = errno;
+                        break;
+                }
+        }
+
+        if (fclose(fptr_tmp) != 0) {
+                rc = errno;
+#ifdef DEBUG
+                perror("Error: ");
+#endif
+        }
+        free(line);
+
+        return rc;
+}
+
+/* Exported function */
+TelemJournal *open_journal(const char *journal_file)
+{
+        FILE *fptr = NULL;
+        char boot_id[BOOTID_LEN] = { '\0' };
+        struct TelemJournal *telem_journal;
+
+        // Use default location if journal_file parameter is NULL
+        if (journal_file == NULL) {
+                fptr = fopen(JOURNAL_PATH, "a+");
+        } else {
+                fptr = fopen(journal_file, "a+");
+        }
+        if (fptr == NULL) {
+#ifdef DEBUG
+                perror("Error while opening journal file: ");
+#endif
+                return NULL;
+        }
+
+        if (read_boot_id(boot_id) != 0) {
+#ifdef DEBUG
+                perror("Error while reading boot_id: ");
+#endif
+                return NULL;
+        }
+
+        telem_journal = malloc(sizeof(struct TelemJournal));
+        if (!telem_journal) {
+                fprintf(stderr, "Unable to allocate more memory\n");
+                return NULL;
+        }
+
+        telem_journal->fptr = fptr;
+        telem_journal->journal_file = \
+                (journal_file == NULL) ? strdup(JOURNAL_PATH) : strdup(journal_file);
+        /* boot_id includes \n at the end, strip RC during duplication */
+        telem_journal->boot_id = strndup(boot_id, BOOTID_LEN - 1);
+        telem_journal->record_count = get_record_count(telem_journal);
+        telem_journal->record_count_limit = RECORD_LIMIT;
+        telem_journal->latest_record_id = NULL;
+        telem_journal->prune_entry_callback = NULL;
+
+#ifdef DEBUG
+        printf("Records in db: %d\n", telem_journal->record_count);
+#endif
+
+        return telem_journal;
+}
+
+/* Exported function */
+void close_journal(TelemJournal *telem_journal)
+{
+        if (telem_journal) {
+                free(telem_journal->boot_id);
+                free(telem_journal->journal_file);
+                free(telem_journal->latest_record_id);
+                fclose(telem_journal->fptr);
+                free(telem_journal);
+        }
+}
+
+/**
+ * Checks if class is a prefix i.e. t/\* or t/t/\*
+ *
+ * @param class A pointer to string with classification
+ *              value to check.
+ * @return 1 if class is a prefix, 0 otherwise
+ */
+static int is_class_prefix(char *class)
+{
+        size_t class_len = strlen(class);
+        return (strcmp((char *)(class + class_len - 2), "/*") == 0) ? 1 : 0;
+}
+
+/**
+ * Print records content
+ *
+ * @param record_id Unique record identifier
+ *
+ */
+static void print_record(char *record_id)
+{
+        int rc = 0;
+        size_t read = 0;
+        char buff[MID_BUFF] = { '\0' };
+        char *filepath = NULL;
+        FILE *recordfp = NULL;
+
+        rc = asprintf(&filepath, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (rc == -1) {
+                // just bail out, there are worse problems
+                return;
+        }
+
+        recordfp = fopen(filepath, "r");
+        if (!recordfp) {
+                telem_perror("Error when opening a record to print");
+                return;
+        }
+
+        while ((read = fread(buff, 1, sizeof(buff), recordfp)) > 0) {
+                fwrite(buff, 1, read, stdout);
+        }
+
+        free(filepath);
+        fclose(recordfp);
+}
+
+/* Exported function */
+int print_journal(TelemJournal *telem_journal, char *classification,
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record)
+{
+        int n = 0;
+        int rc = 0;
+        int count = 0;
+        char str_time[80] = { '\0' };
+        char *line = NULL;
+        size_t len = 0;
+        FILE *journal_fileptr = NULL;
+        struct tm ts;
+        struct JournalEntry *entry = NULL;
+
+        if (telem_journal == NULL) {
+                return -1;
+        }
+
+        journal_fileptr = fopen(telem_journal->journal_file, "r");
+        if (!journal_fileptr) {
+                return -1;
+        }
+
+        // Advance file pointer to line n
+        n = telem_journal->record_count - telem_journal->record_count_limit;
+        if (n > 0) {
+                rc = skip_n_lines(n, journal_fileptr, NULL);
+                if (rc == -1) {
+                        return rc;
+                } else if (rc != 0) {
+                        fprintf(stderr, "An error occurred while advancing journal file: %s\n", strerror(rc));
+                }
+        }
+
+        while (getline(&line, &len, journal_fileptr) != -1) {
+                deserialize_journal_entry(line, &entry);
+                if (entry) {
+                        /* filter entry out if one is provided */
+                        if (record_id != NULL && strcmp(entry->record_id, record_id) != 0) {
+                                goto skip_print;
+                        }
+                        if (boot_id != NULL && strcmp(entry->boot_id, boot_id) != 0) {
+                                goto skip_print;
+                        }
+                        if (event_id != NULL && strcmp(entry->event_id, event_id) != 0) {
+                                goto skip_print;
+                        }
+                        // In the case of class checking prefixes is an option
+                        if (classification != NULL) {
+                                // Check prefixes when classification ends in /*, otherwise use strcomp
+                                if (is_class_prefix(classification)) {
+                                        if (strncmp(entry->classification, classification, strlen(classification) - 1) != 0) {
+                                                continue;
+                                        }
+                                } else if (strcmp(entry->classification, classification) != 0) {
+                                        continue;
+                                }
+                        }
+                        /* end filters section */
+                        ts = *localtime(&entry->timestamp);
+                        if (strftime(str_time, sizeof(str_time), "%a %Y-%m-%d %H:%M:%S %Z", &ts) == 0) {
+                                continue;
+                        }
+                        /* print record metadata */
+                        fprintf(stdout, "%-30s %s %s %s %s\n", entry->classification, str_time, entry->record_id, entry->event_id, entry->boot_id);
+                        /* print record content */
+                        if (include_record) {
+                                print_record(entry->record_id);
+                        }
+                        count++;
+                }
+skip_print:
+                free_journal_entry(entry);
+        }
+        free(line);
+        fclose(journal_fileptr);
+
+        return count;
+}
+
+/**
+ * Validation for event id. Thic check makes sure
+ * that an event_id is a 32 hexadecimal chars long.
+ *
+ * @param event_id A pointer to id value.
+ *
+ * @return 0 on success, 1 on failure
+ */
+static int validate_event_id(char *event_id)
+{
+        const char alphab[] = EVENT_ID_ALPHAB;
+
+        if (event_id == NULL) {
+                return 1;
+        }
+
+        if (strlen(event_id) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        if (strspn(event_id, alphab) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        return 0;
+}
+
+/* Exported function */
+int new_journal_entry(TelemJournal *telem_journal, char *classification,
+                      time_t timestamp, char *event_id)
+{
+        int rc = 1;
+        char boot_id[BOOTID_LEN] = { '\0' };
+        char *record_id = NULL;
+        struct JournalEntry *entry = NULL;
+
+        if (telem_journal == NULL) {
+#ifdef DEBUG
+                fprintf(stderr, "Error: telem_journal was not initialized\n");
+#endif
+                return rc;
+        }
+
+        if (validate_classification(classification) != 0) {
+                return rc;
+        }
+
+        if (validate_event_id(event_id) != 0) {
+                return rc;
+        }
+
+        entry = malloc(sizeof(struct JournalEntry));
+        if (!entry) {
+                fprintf(stderr, "Error: unable to allocate more memory\n");
+                return rc;
+        }
+        entry->classification = NULL;
+        entry->record_id = NULL;
+        entry->event_id = NULL;
+        entry->boot_id = NULL;
+
+        if (get_random_id(&record_id) != 0) {
+#ifdef DEBUG
+                fprintf(stderr, "Error: unable to generate random id\n");
+#endif
+                goto quit;
+        }
+
+        if (read_boot_id(boot_id) != 0) {
+#ifdef DEBUG
+                fprintf(stderr, "Error: unable to read boot_id\n");
+#endif
+                goto quit;
+        }
+
+        if ((entry->classification = strdup(classification)) == NULL) {
+                goto quit;
+        }
+        if ((entry->event_id = strdup(event_id)) == NULL) {
+                goto quit;
+        }
+
+        entry->record_id = record_id;
+        /* boot_id includes \n at the end, strip RC during duplication */
+        entry->boot_id = strndup(boot_id, BOOTID_LEN - 1);
+        entry->timestamp = timestamp;
+
+        if ((rc = save_entry(telem_journal->fptr, entry)) == 0) {
+                telem_journal->record_count = telem_journal->record_count + 1;
+#ifdef DEBUG
+                fprintf(stdout, "%d records in journal\n", telem_journal->record_count);
+#endif
+        }
+
+        free(telem_journal->latest_record_id);
+        telem_journal->latest_record_id = strdup(entry->record_id);
+
+quit:
+        free_journal_entry(entry);
+
+        return rc;
+}
+
+/* Exported function */
+int prune_journal(struct TelemJournal *telem_journal, char *tmp_dir)
+{
+
+        int rc = 1;
+        int count = 0;
+        int deviation = DEVIATION;
+        char *tmp_file_path = NULL;
+
+        if (telem_journal == NULL) {
+                return rc;
+        }
+
+        if (telem_journal->record_count > (deviation + telem_journal->record_count_limit)) {
+                count = telem_journal->record_count - telem_journal->record_count_limit;
+
+                // jump to line# count
+                if ((rc = skip_n_lines(count, telem_journal->fptr, telem_journal->prune_entry_callback)) != 0) {
+#ifdef DEBUG
+                        fprintf(stderr, "Error: skipping %d journal lines\n", count);
+#endif
+                        // if rc == -1 (n > #lines in file, change to errno style)
+                        return (rc == -1) ? EADDRNOTAVAIL : rc;
+                }
+                // Use default if not tmp_dir is specified
+                if (tmp_dir == NULL) {
+                        if ((asprintf(&tmp_file_path, "%s/.journal", JOURNAL_TMPDIR) == -1)) {
+                                return ENOMEM;
+                        }
+                } else {
+                        if ((asprintf(&tmp_file_path, "%s/.journal", tmp_dir) == -1)) {
+                                return ENOMEM;
+                        }
+                }
+                // create new file with rest of file
+                if ((rc = copy_to_tmp(telem_journal->fptr, tmp_file_path)) != 0) {
+#ifdef DEBUG
+                        fprintf(stderr, "Error: copying partial journal to temp journal file\n");
+#endif
+                        goto quit;
+                }
+                // close file handler
+                if ((rc = fclose(telem_journal->fptr)) != 0) {
+#ifdef DEBUG
+                        fprintf(stderr, "Error: closing journal file handler\n");
+#endif
+                        goto quit;
+                }
+                // overwrite file
+                if ((rc = rename(tmp_file_path, telem_journal->journal_file)) != 0) {
+#ifdef DEBUG
+                        fprintf(stderr, "Error: while overwriting journal file\n");
+#endif
+                        goto quit;
+                }
+                // reopen file handler
+                telem_journal->fptr = fopen(telem_journal->journal_file, "a+");
+                if (!telem_journal->fptr) {
+#ifdef DEBUG
+                        fprintf(stderr, "Error: re-opening journal file\n");
+#endif
+                        return rc;
+                }
+                // update record count
+                telem_journal->record_count = telem_journal->record_count - count;
+#ifdef DEBUG
+                fprintf(stdout, "record_count: %d\n", telem_journal->record_count);
+#endif
+        }
+        rc = 0;
+
+quit:
+        free(tmp_file_path);
+
+        return rc;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/journal/journal.h

@@ -0,0 +1,117 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+/* default record limit */
+#define RECORD_LIMIT 100
+#define DEVIATION 50
+
+#include <time.h>
+#include <stdio.h>
+#include <stdbool.h>
+
+/* Journal entry type */
+typedef struct JournalEntry {
+        time_t timestamp;
+        char *classification;
+        char *record_id;
+        char *event_id;
+        char *boot_id;
+} JournalEntry;
+
+/* Telemetry journal type */
+typedef struct TelemJournal {
+        FILE *fptr;
+        char *journal_file;
+        char *boot_id;
+        char *latest_record_id;
+        int record_count;
+        int record_count_limit;
+        int (*prune_entry_callback)(char *);
+} TelemJournal;
+
+/**
+ * Telemetry journal initialization.
+ *
+ * @param journal_file A pointer to a string containing
+ *        the full path to file used as journal storage.
+ *        If journal_file is set to NULL a default value
+ *        will be used.
+ *
+ * @returns a telemetry journal structure in success or
+ *          NULL in case of failure.
+ */
+TelemJournal *open_journal(const char *journal_file);
+
+/**
+ * Closes journal file and deallocates memory that was
+ * previously initialized by open_journal call.
+ *
+ * @param telem_journal A pointer to struct that has a
+ *         reference to journal file pointer and other
+ *         members.
+ */
+void close_journal(TelemJournal *telem_journal);
+
+/**
+ * Prints journal contents to stdout. Use function parameters
+ * to filter journal entries to print.
+ *
+ * @param telem_journal A pointer to struct initialized
+ *        by open_journal call.
+ * @param classification A pointer to string used as
+ *        classification filter.
+ * @param record_id A pointer to string used as record_id filter.
+ * @param event_id A pointer to string used as event_id filter.
+ * @param boot_id A pointer to string used as boor_id filter.
+ * @param include_record A flag to control record content print.
+ *
+ * @return the number of lines printed to stdout on success, -1
+ *         on failure.
+ */
+int print_journal(TelemJournal *telem_journal, char *classification,
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record);
+
+/**
+ * Creates a new entry in journal.
+ *
+ * @param telem_journal A pointer to telemetry journal.
+ * @param classification A pointer to a classification value.
+ * @param timestamp Record creation time stamp value.
+ * @param event_id A pointer to a unique id value for record
+ *        event.
+ *
+ * @return 0 on success 1 on failure.
+ */
+int new_journal_entry(TelemJournal *telem_journal, char *classification,
+                      time_t timestamp, char *event_id);
+
+/**
+ * Checks number of lines in log and prunes the oldest records
+ * if journal grows more than telem_journal->record_count_limit.
+ *
+ * @param telem_journal A pointer to telemetry journal struct
+ *        returned by open_journal call.
+ * @param tmp_dir A pointer to a string with the path to use for
+ *                temp file during pruning process.
+ *
+ * @return 0 on success, errno on failure
+ */
+int prune_journal(TelemJournal *telem_journal, char *tmp_dir);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/journal/local.mk

@@ -0,0 +1,9 @@
+bin_PROGRAMS += \
+	%D%/telem_journal
+
+%C%_telem_journal_SOURCES = %D%/cli.c \
+	%D%/journal.c \
+	src/util.c \
+	src/common.c
+
+# vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

src/local.mk

@@ -5,7 +5,11 @@ bin_PROGRAMS = %D%/telemd
 	%D%/telemdaemon.c \
 	%D%/telemdaemon.h \
 	%D%/spool.c \
-	%D%/spool.h
+	%D%/spool.h \
+	%D%/journal/journal.c \
+	%D%/journal/journal.h \
+	%D%/retention.c \
+	%D%/retention.h
 
 %C%_telemd_LDADD = $(CURL_LIBS) \
 	%D%/libtelem-shared.la
@@ -34,9 +38,9 @@ endif
 endif
 
 # set library version info
-SHAREDLIB_CURRENT=3
+SHAREDLIB_CURRENT=4
 SHAREDLIB_REVISION=0
-SHAREDLIB_AGE=0
+SHAREDLIB_AGE=1
 
 noinst_LTLIBRARIES = %D%/libtelem-shared.la
 
@@ -44,20 +48,19 @@ noinst_LTLIBRARIES = %D%/libtelem-shared.la
 	%D%/util.c \
 	%D%/util.h \
 	%D%/configuration.c \
+	%D%/nica/inifile.c \
+	%D%/nica/hashmap.c \
+	%D%/nica/b64enc.c \
 	%D%/configuration.h \
 	%D%/common.c \
 	%D%/common.h
 
 %C%_libtelem_shared_la_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 
 %C%_libtelem_shared_la_LDFLAGS = \
 	$(AM_LDFLAGS)
 
-%C%_libtelem_shared_la_LIBADD = \
-	$(GLIB_LIBS)
-
 lib_LTLIBRARIES = \
 	%D%/libtelemetry.la
 
@@ -69,12 +72,10 @@ include_HEADERS = %D%/telemetry.h
 noinst_HEADERS = %D%/log.h
 
 %C%_libtelemetry_la_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 
 %C%_libtelemetry_la_LDFLAGS = \
 	$(AM_LDFLAGS) \
-	$(GLIB_LIBS) \
 	-version-info $(SHAREDLIB_CURRENT):$(SHAREDLIB_REVISION):$(SHAREDLIB_AGE) \
 	-Wl,--version-script=$(top_srcdir)/src/telemetry.sym
 

src/log.h

@@ -81,7 +81,7 @@
 #ifdef TM_LOG_STDERR
 #define telem_log(priority, ...) do { \
                 if (priority <= MAX_LOG_LEVEL) { \
-                        fprintf(stderr, __VA_ARGS__ ); \
+                        fprintf(stderr, __VA_ARGS__); \
                 } \
 } while (0);
 #endif

src/main.c

@@ -49,6 +49,7 @@
 #include "log.h"
 #include "telemdaemon.h"
 #include "configuration.h"
+#include "retention.h"
 #include "spool.h"
 
 /*
@@ -78,6 +79,7 @@ int main(int argc, char **argv)
         int c;
         char *config_file = NULL;
         int opt_index = 0;
+        nfds_t initial_nfds = 0;
         sigset_t mask;
         //bool interrupted = false;
 
@@ -121,6 +123,11 @@ int main(int argc, char **argv)
         }
         initialize_daemon(&daemon);
 
+        /* Register record retention delete action as a callback to prune entry */
+        if (daemon.record_journal) {
+                daemon.record_journal->prune_entry_callback = &delete_record_by_id;
+        }
+
         sigemptyset(&mask);
 
         if (sigaddset(&mask, SIGHUP) != 0) {
@@ -232,7 +239,7 @@ int main(int argc, char **argv)
         time_t last_spool_run_time = time(NULL);
         time_t last_daemon_start_time = time(NULL);
 
-	bool daemon_recycling_enabled = daemon_recycling_enabled_config();
+        bool daemon_recycling_enabled = daemon_recycling_enabled_config();
 
         ret = update_machine_id();
         if (ret == -1) {
@@ -244,9 +251,18 @@ int main(int argc, char **argv)
 
         time_t last_refresh_time = time(NULL);
 
+        /* Save initial count for non connection fds */
+        initial_nfds = daemon.nfds;
+
         /* Loop to accept clients */
         while (1) {
                 malloc_trim(0);
+                if (initial_nfds == daemon.nfds) {
+                        ret = prune_journal(daemon.record_journal, JOURNAL_TMPDIR);
+                        if (ret != 0) {
+                                telem_log(LOG_WARNING, "Unable to prune journal\n");
+                        }
+                }
                 ret = poll(daemon.pollfds, daemon.nfds, spool_process_time * 1000);
                 if (ret == -1) {
                         telem_perror("Failed to poll daemon file descriptors");
@@ -338,7 +354,7 @@ int main(int argc, char **argv)
                 } else {
                         time_t now = time(NULL);
                         /* If spool is empty and time to recycle the daemon has elapsed*/
-			if (daemon_recycling_enabled && (daemon.current_spool_size == 0) &&
+                        if (daemon_recycling_enabled && (daemon.current_spool_size == 0) &&
                             difftime(now, last_daemon_start_time) >= TM_DAEMON_EXIT_TIME) {
                                 /* Exit */
                                 telem_log(LOG_INFO, "Daemon exiting for recycling\n");
@@ -362,6 +378,8 @@ int main(int argc, char **argv)
         }
 
 clean_exit:
+        close_journal(daemon.record_journal);
+
         /* Free memory before exiting */
         while ((cl = LIST_FIRST(&(daemon.client_head))) != NULL) {
                 remove_client(&(daemon.client_head), cl);

src/nica/b64enc.c

@@ -0,0 +1,154 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdint.h>
+#include "b64enc.h"
+
+#define B64_LINE_LEN 76
+
+static int padding[] = {0, 2, 1, 0};
+static char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+
+/*
+ * base 64 encoding of 3 bytes to 4 printable characters
+ * (https://tools.ietf.org/html/rfc4648).
+ *
+ * i.e. (Reference: https://en.wikipedia.org/wiki/Base64)
+ *
+ *      [   M  ] [   a  ] [  n  ]
+ * bin  01001101 01100001 01101110
+ *
+ * b64  01001101 01100001 01101110  >> 18 & 0x3F -> 010011 -> ( 19 dec) T
+ *      ------
+ *
+ * b64  01001101 01100001 01101110  >> 12 & 0x3F -> 010110 -> ( 22 dec) W
+ *            -------
+ *
+ * b64  01001101 01100001 01101110  >>  6 & 0x3F -> 000101 -> (  5 dec) F
+ *                   -------
+ *
+ * b64  01001101 01100001 01101110  >>  0 & 0x3F -> 101110 -> (117 dec) u
+ *                          ------
+ *
+ */
+static void b64_3b(char bin[3], char *out) {
+
+    uint32_t x = (((uint32_t) bin[0] << 16) & 0xFF0000) |
+                 (((uint32_t) bin[1] <<  8) & 0xFF00)   |
+                 (((uint32_t) bin[2])       & 0xFF);
+
+    out[3] = table[x       & 0x3F];
+    out[2] = table[x >> 6  & 0x3F];
+    out[1] = table[x >> 12 & 0x3F];
+    out[0] = table[x >> 18 & 0x3F];
+
+}
+
+
+/*
+ * Provide padding to already initialized b64 output
+ */
+static void b64_padding(char *out, int pad_size) {
+
+    if (pad_size > 0) {
+        *out++ = '=';
+    }
+
+    if (pad_size > 1) {
+        *out++ = '=';
+    }
+
+    *out = '\0';
+}
+
+
+/*
+ * Encode file contents as b64 string
+ */
+int nc_b64enc_file(FILE *fh, char *buff, size_t buff_size) {
+
+    int ret_val = 1;
+    int out_count = 0;
+    int pad_size = 0;
+    char *buff_ptr = buff;
+    char bin[3] = {0, 0, 0};
+    size_t len = 0;
+
+    while ((len = fread(bin, 1, 3, fh)) > 0) {
+        b64_3b(bin, buff_ptr);
+        /*
+         Reset 2nd and 3rd bytes in case only one byte was read. If these
+         bytes are not clean b64 output will be corrupted with old bits.
+         */
+        bin[1] = bin[2] = 0;
+        /*
+         Advance pointer to the next free position of the b64 encoded
+         buffer. This happens to be len + 1 (as longs as len >= 1 and len <= 3)
+
+            len = 1 byte  -> b64 -> 2 bytes (or len + 1)
+            len = 2 bytes -> b64 -> 3 bytes (or len + 1)
+            len = 3 bytes -> b64 -> 4 bytes (or len + 1)
+        */
+        buff_ptr += (len + 1);
+        /*
+         Line break every 76 characters
+        */
+        out_count += (int) (len + 1);
+        if (out_count % B64_LINE_LEN == 0) {
+            *buff_ptr++ = '\n';
+        }
+        pad_size = padding[len];
+        /*
+         Check not to overflow buffer
+
+            buffer size <= b64 characters + padding length + null termination
+        */
+        if (buff_size <= (size_t) (buff_ptr - buff) + (size_t) pad_size + 1) {
+           ret_val = 0;
+           goto end_b64_enc;
+        }
+    }
+
+    b64_padding(buff_ptr, pad_size);
+
+end_b64_enc:
+
+    return ret_val;
+}
+
+
+int nc_b64enc_filename(const char *filename, char *buff, size_t buff_size) {
+
+    int ret = 0;
+    FILE *fh = NULL;
+
+    fh = fopen(filename, "rb");
+
+    if (fh == NULL) {
+        goto nc_b64_clean;
+    }
+
+    ret = nc_b64enc_file(fh, buff, buff_size);
+
+nc_b64_clean:
+
+    if (fh) {
+        fclose(fh);
+    }
+
+    return ret;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/nica/b64enc.h

@@ -0,0 +1,42 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * This is a chained base64 encode implementation.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include "macros.h"
+
+/*
+ * Encodes contents of file handler in base64
+ *
+ * @param fh File handler to encode as base64
+ * @param buff Output buffer where the base64 output should be written
+ * @param buff_size Size of the output buffer
+ *
+ * @returns 1 in success and 0 in failure
+ */
+_nica_public_ int nc_b64enc_file(FILE *fh, char *buff, size_t buff_size);
+
+
+/*
+ * Encodes contents of file (filename) in base64
+ *
+ * @param filename File to encode as base64
+ * @param buff Output buffer where the base64 output should be written
+ * @param buff_size Size of the output buffer
+ *
+ * @returns 1 in success and 0 in failure
+ */
+_nica_public_ int nc_b64enc_filename(const char *filename, char *buff, size_t buff_size);
+

src/nica/hashmap.c

@@ -0,0 +1,469 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "hashmap.h"
+
+#define INITIAL_SIZE 61
+
+/* When we're "full", 70% */
+#define FULL_FACTOR 0.7
+
+/* Multiple to increase bucket size by */
+#define INCREASE_FACTOR 4
+
+/**
+ * An bucket/chain within the hashmap
+ */
+typedef struct NcHashmapEntry {
+        void *hash;                  /**<The key for this item */
+        void *value;                 /**<Value for this item */
+        struct NcHashmapEntry *next; /**<Next item in the chain */
+        bool occ;                    /**<Whether this bucket is occupied */
+} NcHashmapEntry;
+
+/**
+ * A NcHashmap
+ */
+struct NcHashmap {
+        int size;                /**<Current size of the hashmap */
+        int next_size;           /**<Next size at which we need to resize */
+        int n_buckets;           /**<Current number of buckets */
+        NcHashmapEntry *buckets; /**<Stores our bucket chains */
+
+        nc_hash_create_func hash;     /**<Hash generation function */
+        nc_hash_compare_func compare; /**<Key comparison function */
+        nc_hash_free_func key_free;   /**<Cleanup function for keys */
+        nc_hash_free_func value_free; /**<Cleanup function for values */
+};
+
+/**
+ * Iteration object
+ */
+typedef struct _NcHashmapIter {
+        int bucket;     /**<Current bucket position */
+        NcHashmap *map; /**<Associated NcHashmap */
+        void *item;     /**<Current item in this iteration */
+} _NcHashmapIter;
+
+static void nc_hashmap_update_next_size(NcHashmap *self);
+static bool nc_hashmap_resize(NcHashmap *self);
+
+static inline bool nc_hashmap_maybe_resize(NcHashmap *self)
+{
+        if (!self) {
+                return false;
+        }
+        if (self->size >= self->next_size) {
+                return true;
+        }
+        return false;
+}
+
+static NcHashmap *nc_hashmap_new_internal(nc_hash_create_func create, nc_hash_compare_func compare,
+                                          nc_hash_free_func key_free, nc_hash_free_func value_free)
+{
+        NcHashmap *map = NULL;
+        NcHashmapEntry *buckets = NULL;
+
+        map = calloc(1, sizeof(NcHashmap));
+        if (!map) {
+                return NULL;
+        }
+
+        buckets = calloc(INITIAL_SIZE, sizeof(NcHashmapEntry));
+        if (!buckets) {
+                free(map);
+                return NULL;
+        }
+        map->buckets = buckets;
+        map->n_buckets = INITIAL_SIZE;
+        map->hash = create ? create : nc_simple_hash;
+        map->compare = compare ? compare : nc_simple_compare;
+        map->key_free = key_free;
+        map->value_free = value_free;
+        map->size = 0;
+
+        nc_hashmap_update_next_size(map);
+
+        return map;
+}
+
+NcHashmap *nc_hashmap_new(nc_hash_create_func create, nc_hash_compare_func compare)
+{
+        return nc_hashmap_new_internal(create, compare, NULL, NULL);
+}
+
+NcHashmap *nc_hashmap_new_full(nc_hash_create_func create, nc_hash_compare_func compare,
+                               nc_hash_free_func key_free, nc_hash_free_func value_free)
+{
+        return nc_hashmap_new_internal(create, compare, key_free, value_free);
+}
+
+static inline unsigned nc_hashmap_get_hash(NcHashmap *self, const void *key)
+{
+        unsigned hash = self->hash(key);
+        return hash;
+}
+
+static int nc_hashmap_insert_bucket(NcHashmap *self, NcHashmapEntry *buckets, int n_buckets,
+                                    unsigned hash, const void *key, void *value)
+{
+        NcHashmapEntry *row = &(buckets[hash % (unsigned)n_buckets]);
+        NcHashmapEntry *head = NULL;
+        NcHashmapEntry *parent = head = row;
+        NcHashmapEntry *tomb = NULL;
+        int ret = 1;
+
+        while (row) {
+                if (!row->occ) {
+                        tomb = row;
+                }
+                parent = row;
+                if (row->occ && self->compare(row->hash, key)) {
+                        if (self->value_free) {
+                                self->value_free(row->value);
+                        }
+                        if (self->key_free) {
+                                self->key_free(row->hash);
+                        }
+                        row->hash = (void *)key;
+                        row->value = value;
+                        return 0;
+                }
+                row = row->next;
+        }
+
+        if (tomb) {
+                row = tomb;
+        }
+
+        if (!row) {
+                row = calloc(1, sizeof(NcHashmapEntry));
+                if (!row) {
+                        return -1;
+                }
+        }
+
+        row->hash = (void *)key;
+        row->value = value;
+        row->occ = true;
+        if (parent != row && parent) {
+                parent->next = row;
+        }
+
+        return ret;
+}
+
+bool nc_hashmap_put(NcHashmap *self, const void *key, void *value)
+{
+        if (!self) {
+                return false;
+        }
+        int inc;
+
+        if (nc_hashmap_maybe_resize(self)) {
+                if (!nc_hashmap_resize(self)) {
+                        return false;
+                }
+        }
+        unsigned hash = nc_hashmap_get_hash(self, key);
+        inc = nc_hashmap_insert_bucket(self, self->buckets, self->n_buckets, hash, key, value);
+        if (inc >= 0) {
+                self->size += inc;
+                return true;
+        } else {
+                return false;
+        }
+}
+
+static NcHashmapEntry *nc_hashmap_get_entry(NcHashmap *self, const void *key)
+{
+        if (!self) {
+                return NULL;
+        }
+
+        unsigned hash = nc_hashmap_get_hash(self, key);
+        NcHashmapEntry *row = &(self->buckets[hash % (unsigned)self->n_buckets]);
+
+        while (row) {
+                if (self->compare(row->hash, key)) {
+                        return row;
+                }
+                row = row->next;
+        }
+        return NULL;
+}
+
+void *nc_hashmap_get(NcHashmap *self, const void *key)
+{
+        if (!self) {
+                return NULL;
+        }
+
+        NcHashmapEntry *row = nc_hashmap_get_entry(self, key);
+        if (row) {
+                return row->value;
+        }
+        return NULL;
+}
+
+static bool nc_hashmap_remove_internal(NcHashmap *self, const void *key, bool remove)
+{
+        if (!self) {
+                return false;
+        }
+        NcHashmapEntry *row = nc_hashmap_get_entry(self, key);
+
+        if (!row) {
+                return false;
+        }
+
+        if (remove) {
+                if (self->key_free) {
+                        self->key_free(row->hash);
+                }
+                if (self->value_free) {
+                        self->value_free(row->value);
+                }
+        }
+        self->size -= 1;
+        row->hash = NULL;
+        row->value = NULL;
+        row->occ = false;
+
+        return true;
+}
+
+bool nc_hashmap_steal(NcHashmap *self, const void *key)
+{
+        return nc_hashmap_remove_internal(self, key, false);
+}
+
+bool nc_hashmap_remove(NcHashmap *self, const void *key)
+{
+        return nc_hashmap_remove_internal(self, key, true);
+}
+
+bool nc_hashmap_contains(NcHashmap *self, const void *key)
+{
+        return (nc_hashmap_get(self, key)) != NULL;
+}
+
+static inline void nc_hashmap_free_bucket(NcHashmap *self, NcHashmapEntry *bucket, bool nuke)
+{
+        if (!self) {
+                return;
+        }
+        NcHashmapEntry *tmp = bucket;
+        NcHashmapEntry *bk = bucket;
+        NcHashmapEntry *root = bucket;
+
+        while (tmp) {
+                bk = NULL;
+                if (tmp->next) {
+                        bk = tmp->next;
+                }
+
+                if (nuke && tmp->occ) {
+                        if (self->key_free) {
+                                self->key_free(tmp->hash);
+                        }
+                        if (self->value_free) {
+                                self->value_free(tmp->value);
+                        }
+                }
+                if (tmp != root) {
+                        free(tmp);
+                }
+                tmp = bk;
+        }
+}
+
+void nc_hashmap_free(NcHashmap *self)
+{
+        if (!self) {
+                return;
+        }
+        for (int i = 0; i < self->n_buckets; i++) {
+                NcHashmapEntry *row = &(self->buckets[i]);
+                nc_hashmap_free_bucket(self, row, true);
+        }
+        if (self->buckets) {
+                free(self->buckets);
+        }
+
+        free(self);
+}
+
+static void nc_hashmap_update_next_size(NcHashmap *self)
+{
+        if (!self) {
+                return;
+        }
+        self->next_size = (int)(self->n_buckets * FULL_FACTOR);
+}
+
+int nc_hashmap_size(NcHashmap *self)
+{
+        if (!self) {
+                return -1;
+        }
+        return self->size;
+}
+
+static bool nc_hashmap_resize(NcHashmap *self)
+{
+        if (!self || !self->buckets) {
+                return false;
+        }
+
+        NcHashmapEntry *old_buckets = self->buckets;
+        NcHashmapEntry *new_buckets = NULL;
+        NcHashmapEntry *entry = NULL;
+        int incr;
+
+        int old_size, new_size;
+        int items = 0;
+
+        new_size = old_size = self->n_buckets;
+        new_size *= INCREASE_FACTOR;
+
+        new_buckets = calloc((size_t)new_size, sizeof(NcHashmapEntry));
+        if (!new_buckets) {
+                return false;
+        }
+
+        for (int i = 0; i < old_size; i++) {
+                entry = &(old_buckets[i]);
+                while (entry) {
+                        if (entry->occ) {
+                                unsigned hash = nc_hashmap_get_hash(self, entry->hash);
+                                if ((incr = nc_hashmap_insert_bucket(self,
+                                                                     new_buckets,
+                                                                     new_size,
+                                                                     hash,
+                                                                     entry->hash,
+                                                                     entry->value)) > 0) {
+                                        items += incr;
+                                } else {
+                                        /* Likely a memory issue */
+                                        goto failure;
+                                }
+                        }
+                        entry = entry->next;
+                }
+        }
+        /* Successfully resized - do this separately because we need to
+         * guarantee old data is preserved */
+        for (int i = 0; i < old_size; i++) {
+                nc_hashmap_free_bucket(self, &(old_buckets[i]), false);
+        }
+
+        free(old_buckets);
+        self->n_buckets = new_size;
+        self->size = items;
+        self->buckets = new_buckets;
+
+        nc_hashmap_update_next_size(self);
+        return true;
+
+failure:
+        for (int i = 0; i < new_size; i++) {
+                nc_hashmap_free_bucket(self, &(new_buckets[i]), true);
+        }
+        free(new_buckets);
+        return false;
+}
+
+void nc_hashmap_iter_init(NcHashmap *map, NcHashmapIter *citer)
+{
+        _NcHashmapIter *iter = NULL;
+        if (!map || !citer) {
+                return;
+        }
+        iter = (_NcHashmapIter *)citer;
+        _NcHashmapIter it = {
+                .bucket = -1, .map = map, .item = NULL,
+        };
+        *iter = it;
+}
+
+bool nc_hashmap_iter_next(NcHashmapIter *citer, void **key, void **value)
+{
+        _NcHashmapIter *iter = NULL;
+        NcHashmapEntry *item = NULL;
+        NcHashmap *map = NULL;
+        int n_buckets = 0;
+
+        if (!citer) {
+                return false;
+        }
+
+        iter = (_NcHashmapIter *)citer;
+        map = iter->map;
+        if (!map) {
+                return false;
+        }
+        n_buckets = map->n_buckets;
+        item = iter->item;
+
+        for (;;) {
+                if (iter->bucket >= n_buckets) {
+                        if (item && !item->next) {
+                                return false;
+                        }
+                }
+                if (!item) {
+                        iter->bucket++;
+                        if (iter->bucket > n_buckets - 1) {
+                                return false;
+                        }
+                        item = &(map->buckets[iter->bucket]);
+                }
+                if (item) {
+                        if (item->occ) {
+                                goto success;
+                        }
+                        item = item->next;
+                }
+        }
+        return false;
+
+success:
+        iter->item = item->next;
+        if (key) {
+                *key = item->hash;
+        }
+        if (value) {
+                *value = item->value;
+        }
+
+        return true;
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/hashmap.h

@@ -0,0 +1,241 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * This is a chained Hashmap implementation. It focuses on being clean
+ * and efficient, and is comparable (at -O2) to open addressing hashmaps
+ * up until around 105,000 elements, where open addressing will begin
+ * to come out in front. At 1 million elements, open addressing is a clear
+ * winner, however currently we only need a maximum of 70k elements in
+ * our implementation.
+ *
+ * Given the memory required to even begin to deal with 1 million elements,
+ * it becomes very questionable whether a hashmap should have even been
+ * used in the first place (mmap'd db?)
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "macros.h"
+#include "util.h"
+
+/* Convert between uint and void* */
+#define NC_HASH_KEY(x) ((void *)((uintptr_t)(x)))
+#define NC_HASH_VALUE(x) NC_HASH_KEY(x)
+#define NC_UNHASH_KEY(x) ((unsigned int)((uintptr_t)(x)))
+#define NC_UNHASH_VALUE(x) NC_UNHASH_KEY(x)
+
+typedef struct NcHashmap NcHashmap;
+
+/**
+ * Iteration object
+ */
+typedef struct NcHashmapIter {
+        int n0;
+        void *n1;
+        void *n2;
+} NcHashmapIter;
+
+/**
+ * Hash comparison function definition
+ *
+ * @param l First value to compare
+ * @param r Second value to compare
+ *
+ * @return true if l and r both match, otherwise false
+ */
+typedef bool (*nc_hash_compare_func)(const void *l, const void *r);
+
+/**
+ * Hash creation function definition
+ *
+ * @param key Key to generate a hash for
+ *
+ * @return an unsigned integer hash result
+ */
+typedef unsigned (*nc_hash_create_func)(const void *key);
+
+/**
+ * Callback definition to free keys and values
+ *
+ * @param p Single-depth pointer to either a key or value that should be freed
+ */
+typedef void (*nc_hash_free_func)(void *p);
+
+/**
+ * Default hash/comparison functions
+ *
+ * @note These are only used for comparison of *keys*, not values. Unless
+ * explicitly using string keys, you should most likely stick with the default
+ * nc_simple_hash and nc_simple_compare functions
+ */
+
+/* Default string hash */
+static inline unsigned nc_string_hash(const void *key)
+{
+        unsigned hash = 5381;
+        const signed char *c;
+
+        /* DJB's hash function */
+        for (c = key; *c != '\0'; c++) {
+                hash = (hash << 5) + hash + (unsigned)*c;
+        }
+        return hash;
+}
+
+/**
+ * Trivial pointer->uint hash
+ */
+static inline unsigned nc_simple_hash(const void *source)
+{
+        return NC_UNHASH_KEY(source);
+}
+
+/**
+ * Comparison of string keys
+ */
+static inline bool nc_string_compare(const void *l, const void *r)
+{
+        if (!l || !r) {
+                return false;
+        }
+        return (strcmp(l, r) == 0);
+}
+
+/**
+ * Trivial pointer comparison
+ */
+static inline bool nc_simple_compare(const void *l, const void *r)
+{
+        return (l == r);
+}
+
+/**
+ * Create a new NcHashmap
+ *
+ * @param hash Hash creation function
+ * @param compare Key comparison function
+ *
+ * @return A newly allocated NcHashmap
+ */
+_nica_public_ NcHashmap *nc_hashmap_new(nc_hash_create_func hash, nc_hash_compare_func compare);
+
+/**
+ * Create a new NcHashmap with cleanup functions
+ *
+ * @param hash Hash creation function
+ * @param compare Key comparison function
+ * @param key_free Function to free keys when removed/destroyed
+ * @param value_free Function to free values when removed/destroyed
+ *
+ * @return A newly allocated NcHashmap
+ */
+_nica_public_ NcHashmap *nc_hashmap_new_full(nc_hash_create_func hash, nc_hash_compare_func compare,
+                                             nc_hash_free_func key_free,
+                                             nc_hash_free_func value_free);
+
+/**
+ * Store a key/value pair in the hashmap
+ *
+ * @note This will displace duplicate keys, and may free both the key
+ * and value if key_free and value_free are non null
+ *
+ * @param key Key to store in the hashmap
+ * @param value Value to be associated with the key
+ *
+ * @return true if the operation succeeded.
+ */
+_nica_public_ bool nc_hashmap_put(NcHashmap *map, const void *key, void *value);
+
+/**
+ * Get the value associated with the unique key
+ *
+ * @param key Unique key to obtain a value for
+ * @return The associated value if it exists, otherwise NULL
+ */
+_nica_public_ void *nc_hashmap_get(NcHashmap *map, const void *key);
+
+/**
+ * Determine if the key has an associated value in the NcHashmap
+ *
+ * @param key Unique key to check value for
+ * @return True if the key exists in the hashmap
+ */
+_nica_public_ bool nc_hashmap_contains(NcHashmap *self, const void *key);
+
+/**
+ * Free the given NcHashmap, and all keys/values if appropriate
+ */
+_nica_public_ void nc_hashmap_free(NcHashmap *map);
+
+/**
+ * Remove the value and key identified by key.
+ *
+ * @note If key_free or value_free are non-NULL, they will be invoked
+ * for both the key and value being removed
+ *
+ * @param key The unique key to remove
+ * @return true if the key/value pair were removed
+ */
+_nica_public_ bool nc_hashmap_remove(NcHashmap *map, const void *key);
+
+/**
+ * Remove the value and key identified by key without freeing them
+ *
+ * @return true if the key/value pair were stolen
+ */
+_nica_public_ bool nc_hashmap_steal(NcHashmap *map, const void *key);
+
+/**
+ * Return the current size of the hashmap
+ *
+ * @return size of the current hashmap (element count)
+ */
+_nica_public_ int nc_hashmap_size(NcHashmap *map);
+
+/**
+ * Initialise a NcHashmapIter for iteration purposes
+ *
+ * @note The iter *must* be re-inited to re-use, or if it becomes
+ * exhausted from a previous iteration run
+ *
+ * @param iter pointer to a NcHashmapIter
+ */
+_nica_public_ void nc_hashmap_iter_init(NcHashmap *map, NcHashmapIter *iter);
+
+/**
+ * Iterate every key/value pair in the hashmap
+ *
+ * @param iter A correctly initialised NcHashmapIter
+ * @param key Pointers to store the key in, may be NULL to skip
+ * @param value Pointer to store the value in, may be NULL to skip
+ *
+ * @return true if it's possible to iterate
+ */
+_nica_public_ bool nc_hashmap_iter_next(NcHashmapIter *iter, void **key, void **value);
+
+DEF_AUTOFREE(NcHashmap, nc_hashmap_free)
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/inifile.c

@@ -0,0 +1,386 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "hashmap.h"
+#include "inifile.h"
+
+/**
+ * Mapping of @NcIniError to static strings
+ */
+static const char *_errors[] =
+    {[NC_INI_ERROR_FILE] = "",
+     [NC_INI_ERROR_EMPTY_KEY] = "Encountered empty key",
+     [NC_INI_ERROR_NOT_CLOSED] = "Expected closing \']\' for section",
+     [NC_INI_ERROR_NO_SECTION] = "Encountered key=value mapping without a valid section",
+     [NC_INI_ERROR_INVALID_LINE] = "Expected key=value notation",
+     [NC_INI_ERROR_INTERNAL] = "A fatal internal error was encountered" };
+
+const char *nc_ini_error(enum NcIniError error)
+{
+        int j = abs(error);
+        if (j < NC_INI_ERROR_FILE || j >= NC_INI_ERROR_MAX) {
+                return "[Unknown Error]";
+        }
+        return _errors[j];
+}
+
+/**
+ * Strip leading whitespace from string (left)
+ */
+static char *lstrip(char *str, ssize_t len, ssize_t *out_len)
+{
+        ssize_t skip_len = 0;
+
+        for (ssize_t i = 0; i < len; i++) {
+                if (str[i] == ' ' || str[i] == '\t') {
+                        ++skip_len;
+                        continue;
+                }
+                break;
+        }
+        *out_len = len - skip_len;
+        if (skip_len > 0) {
+                char *c = strdup(str + skip_len);
+                free(str);
+                return c;
+        }
+        return str + skip_len;
+}
+
+/**
+ * Strip trailing whitespace from string (right)
+ */
+static char *rstrip(char *str, ssize_t len, ssize_t *out_len)
+{
+        ssize_t skip_len = 0;
+
+        for (ssize_t i = len; i > 0; i--) {
+                if (str[i] == ' ' || str[i] == '\t') {
+                        ++skip_len;
+                        continue;
+                } else if (str[i] == '\0') {
+                        continue;
+                }
+                break;
+        }
+        *out_len = len - skip_len;
+        if (len == skip_len) {
+                return str;
+        }
+        str[(len - skip_len)] = '\0';
+        return str;
+}
+
+/**
+ * Chew both ends of a null terminated string of whitespace
+ * If the left side is whitespace padded, this will always result
+ * in a new allocation due to fast-forwarding the pointer. Keep
+ * this in mind.
+ */
+static char *string_chew_terminated(char *inp)
+{
+        int skip_offset = 0;
+        int len = 0;
+        int end_len = 0;
+        bool count = true;
+
+        if (!inp) {
+                return NULL;
+        }
+
+        for (char *c = inp; *c; c++) {
+                if (count && (*c == ' ' || *c == '\t')) {
+                        ++skip_offset;
+                        continue;
+                } else {
+                        count = false;
+                }
+                ++len;
+        }
+        for (int i = len; i > skip_offset; i--) {
+                if (inp[i] == ' ' || inp[i] == '\t') {
+                        ++end_len;
+                        continue;
+                } else if (inp[i] == '\0') {
+                        continue;
+                }
+                break;
+        }
+
+        if (end_len > 0) {
+                inp[(len - end_len)] = '\0';
+        }
+        if (skip_offset > 0) {
+                char *c = strdup(inp + skip_offset);
+                free(inp);
+                return c;
+        }
+        return inp;
+}
+
+/**
+ * Munch both ends of the string
+ */
+static char *string_strip(char *str, ssize_t len, ssize_t *out_len)
+{
+        ssize_t mylen = len;
+
+        char *c = lstrip(str, len, &mylen);
+        c = rstrip(c, mylen, &mylen);
+        if (out_len) {
+                *out_len = mylen;
+        }
+        return c;
+}
+
+/* eat \s */
+static char *string_unescape(char *str)
+{
+	char *c, *c1, *c2, *c3;;
+
+	if (!str)
+		return NULL;
+
+	c = calloc(strlen(str) + 1, 1);
+	c1 = c;
+
+	c2 = str;
+	while (c2 && *c2 != 0) {
+		c3 = c2 + 1;
+		if (*c2 == '\\') {
+			if (*c3 == 's') {
+				*c = ' ';
+				c++;
+				c2 += 2;
+			} else {
+				c2++;
+			}
+		} else {
+			*c = *c2;
+			c++;
+			c2++;
+		}
+	}
+
+	free(str);	
+        return c1;
+}
+
+NcHashmap *nc_ini_file_parse(const char *path)
+{
+        NcHashmap *ret = NULL;
+        int error_line = 0;
+        int r = 0;
+
+        r = nc_ini_file_parse_full(path, &ret, &error_line);
+        if (r != 0) {
+                if (abs(r) == NC_INI_ERROR_FILE) {
+                        fprintf(stderr, "[inifile] %s: %s\n", strerror(errno), path);
+                } else {
+                        fprintf(stderr,
+                                "[inifile] %s [L%d]: %s\n",
+                                nc_ini_error(r),
+                                error_line,
+                                path);
+                }
+                return NULL;
+        }
+        return ret;
+}
+
+int nc_ini_file_parse_full(const char *path, NcHashmap **out_map, int *error_line_number)
+{
+        autofree(FILE) *file = NULL;
+        char *buf = NULL;
+        ssize_t r = 0;
+        size_t sn = 0;
+        int line_count = 1;
+        char *current_section = NULL;
+        NcHashmap *root_map = NULL;
+        NcHashmap *section_map = NULL;
+        bool failed = false;
+        int err_ret = 0;
+
+        file = fopen(path, "r");
+        if (!file) {
+                return -(NC_INI_ERROR_FILE);
+        }
+
+        if (!out_map) {
+                fprintf(stderr, "nc_ini_file_parse_full(): NcHashmap pointer invalid\n");
+                return -1;
+        }
+
+        root_map = nc_hashmap_new_full(nc_string_hash,
+                                       nc_string_compare,
+                                       free,
+                                       (nc_hash_free_func)nc_hashmap_free);
+
+        while ((r = getline(&buf, &sn, file)) != -1) {
+                char *ch = NULL;
+                ssize_t str_len = r;
+                char *key = NULL;
+                char *value = NULL;
+
+                /* Fix newline */
+                if (buf[r - 1] == '\n') {
+                        buf[r - 1] = '\0';
+                        --r;
+                }
+                str_len = r;
+
+                buf = string_strip(buf, r, &str_len);
+                /* Empty lines are fine */
+                if (streq(buf, "")) {
+                        goto next;
+                }
+
+                if (buf[0] == '[') {
+                        /* Validate section start */
+                        if (buf[str_len - 1] != ']') {
+                                /* Throw error */
+                                err_ret = NC_INI_ERROR_NOT_CLOSED;
+                                goto fail;
+                        }
+                        /* Grab the section name, and "close" last section */
+                        buf[str_len - 1] = '\0';
+                        if (current_section) {
+                                free(current_section);
+                        }
+                        current_section = strdup(buf + 1);
+                        section_map = nc_hashmap_get(root_map, current_section);
+                        if (!section_map) {
+                                /* Create a new section dynamically */
+                                section_map = nc_hashmap_new_full(nc_string_hash,
+                                                                  nc_string_compare,
+                                                                  free,
+                                                                  free);
+                                nc_hashmap_put(root_map, strdup(current_section), section_map);
+                        }
+                        goto next;
+                } else if (buf[0] == '#' || buf[0] == ';') {
+                        /* Skip comment */
+                        goto next;
+                }
+
+                /* Look for key = value */
+                ch = strchr(buf, '=');
+                if (!ch) {
+                        /* Throw error */
+                        err_ret = NC_INI_ERROR_INVALID_LINE;
+                        goto fail;
+                }
+
+                /* Can't have sectionless k->v */
+                if (!current_section) {
+                        err_ret = NC_INI_ERROR_NO_SECTION;
+                        goto fail;
+                }
+
+                long int offset = ch - buf;
+
+                /* Grab the key->value from this assignment line */
+                value = strdup((buf + offset) + 1);
+                buf[offset] = '\0';
+                key = strdup(buf);
+                key = string_chew_terminated(key);
+                value = string_chew_terminated(value);
+		value = string_unescape(value);
+
+                if (streq(key, "")) {
+                        err_ret = NC_INI_ERROR_EMPTY_KEY;
+                        free(key);
+                        key = NULL;
+                        free(value);
+                        value = NULL;
+                        goto fail;
+                }
+
+                /* Ensure a section mapping exists */
+                section_map = nc_hashmap_get(root_map, current_section);
+                if (!section_map) {
+                        err_ret = NC_INI_ERROR_INTERNAL;
+                        fprintf(stderr,
+                                "[inifile] Fatal! No section map for named "
+                                "section: %s\n",
+                                current_section);
+                        goto fail;
+                }
+
+                /* Insert these guys into the map */
+                if (!nc_hashmap_put(section_map, key, value)) {
+                        err_ret = NC_INI_ERROR_INTERNAL;
+                        fprintf(stderr, "[inifile] Fatal! Out of memory\n");
+                        goto fail;
+                }
+        /* Progression + cleanup */
+        next:
+                if (buf) {
+                        free(buf);
+                        buf = NULL;
+                }
+                ++line_count;
+                sn = 0;
+                continue;
+        /* Parsing error, bail */
+        fail:
+                failed = true;
+                if (error_line_number) {
+                        *error_line_number = line_count;
+                }
+                if (key) {
+                        free(key);
+                }
+                if (value) {
+                        free(value);
+                }
+                break;
+        }
+
+        if (buf) {
+                free(buf);
+                buf = NULL;
+        }
+
+        if (current_section) {
+                free(current_section);
+                current_section = NULL;
+        }
+
+        if (failed) {
+                if (root_map) {
+                        nc_hashmap_free(root_map);
+                }
+                return -(err_ret);
+        }
+        *out_map = root_map;
+        return 0;
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/inifile.h

@@ -0,0 +1,114 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#include "hashmap.h"
+#include "macros.h"
+#include "util.h"
+
+/**
+ * The Nica INI Parser focuses on simplicity and ease of use, therefore it
+ * currently only supports a read-only parse operation.
+ *
+ * The parse functions are used to return a _root_ @NcHashmap which contains
+ * name to section mappings. Each section is in turn an @NcHashmap with string
+ * to string mappings.
+ *
+ * Consider the following example:
+ *
+ *      [Person]
+ *      alive = true
+ *      ; This person is alive
+ *
+ * The returned @NcHashmap will contain the key "Person", which corresponds to
+ * another @NcHashmap. This @NcHashmap will contain the key "alive", which has
+ * the value "true". Therefore we can access this variable using the @NcHashmap
+ * API:
+ *
+ *      char *alive = nc_hashmap_get(nc_hashmap_get(ini, "Person"), "alive");
+ *
+ * Conversely, we can use the API to determine whether a section or key is set:
+ *
+ *      if (nc_hashmap_contains(ini, "Person"))
+ *
+ * As @NcHashmap returns NULL in nc_hashmap_get, you can happily pass NULL
+ * keys, allowing quick key fetches without checking if the section exists.
+ *
+ * In our INI file, lines beginning with ";" or "#" are considered as comments
+ * and are not processed. A line following the "key = value" notation is an
+ * _assignment_. "[Section]" is considered to be a section in the INI file, and
+ * all INI files must have at least one section.
+ *
+ * We do allow duplication section definitions, by following a merge policy. If
+ * a key is redefined then the original key->value mapping is freed and the new
+ * key->value mapping is inserted.
+ *
+ * The parser will return a correctly configured @NcHashmap, the only cleanup
+ * required is to call nc_hashmap_free on it. Alternatively, use the autofree
+ * helper, for a RAII approach:
+ *
+ *      autofree(NcHashmap) *map = nc_ini_file_parse("myfile.ini");
+ */
+
+/**
+ * Errors that are reported when parsing an ini file
+ */
+enum NcIniError {
+        NC_INI_ERROR_MIN = 0,
+        NC_INI_ERROR_FILE,         /**< File based error, check strerror(errno) */
+        NC_INI_ERROR_EMPTY_KEY,    /**< Empty key in an assignment line */
+        NC_INI_ERROR_NOT_CLOSED,   /**< Encountered section start that wasn't closed with a ']' */
+        NC_INI_ERROR_NO_SECTION,   /**< Key assignment with no defined sections */
+        NC_INI_ERROR_INVALID_LINE, /**< Encountered an invalid line (syntax) */
+        NC_INI_ERROR_INTERNAL,     /**< Fatal internal error encountered. Caller should exit */
+        NC_INI_ERROR_MAX
+};
+/**
+
+ * Convenience wrapper for nc_ini_file_parse_full, reports errors to stderrr
+ * and returns an NcHashmap if the parsing succeeded
+ */
+_nica_public_ NcHashmap *nc_ini_file_parse(const char *path);
+
+/**
+ * Parse an INI file into a hash of hashes.
+ *
+ * @param path Path on the filesystem to parse, must be in INI syntax.
+ * @param out_map Pointer to store the resulting root NcHashmap in
+ * @param error_line_number If not null, then the erronous line number is
+ * stored.
+ *
+ * @return 0 if the call was successful, or a negative integer. See nc_ini_error
+ */
+_nica_public_ int nc_ini_file_parse_full(const char *path, NcHashmap **out_map,
+                                         int *error_line_number);
+
+/**
+ * Return a string representation for a given @NcIniError
+ *
+ * @param error Valid NcIniError code
+ * @return a static string owned by the implementation
+ */
+_nica_public_ const char *nc_ini_error(enum NcIniError error);
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/local.mk

@@ -0,0 +1,6 @@
+noinst_HEADERS += \
+	%D%/hashmap.h \
+	%D%/inifile.h \
+	%D%/macros.h \
+	%D%/nc-string.h \
+	%D%/util.h

src/nica/macros.h

@@ -0,0 +1,29 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#define _nica_public_ __attribute__((visibility("default")))
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/nc-string.c

@@ -0,0 +1,144 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "nc-string.h"
+
+nc_string *nc_string_dup(const char *str)
+{
+        if (!str) {
+                return NULL;
+        }
+        struct nc_string_t *st = calloc(1, sizeof(struct nc_string_t));
+        if (!st) {
+                return NULL;
+        }
+        st->len = asprintf(&st->str, "%s", str);
+        if (st->len < 0 || !st->str) {
+                free(st);
+                return NULL;
+        }
+        return st;
+}
+
+nc_string *nc_string_dup_printf(const char *ptn, ...)
+{
+        if (!ptn) {
+                return NULL;
+        }
+
+        struct nc_string_t *st = calloc(1, sizeof(struct nc_string_t));
+        if (!st) {
+                return NULL;
+        }
+
+        va_list va;
+        va_start(va, ptn);
+
+        st->len = vasprintf(&st->str, ptn, va);
+        if (st->len < 0 || !st->str) {
+                free(st);
+                st = NULL;
+                goto end;
+        }
+end:
+        va_end(va);
+
+        return st;
+}
+
+nc_string *nc_string_append_printf(nc_string *st, const char *ptn, ...)
+{
+	char *newstr;
+	char *newstr2;
+	int ret;
+
+        if (!ptn) {
+                return NULL;
+        }
+
+        va_list va;
+        va_start(va, ptn);
+
+        ret = vasprintf(&newstr, ptn, va);
+	if (ret <= 0)
+		newstr = strdup("");
+
+	st->len = asprintf(&newstr2, "%s%s", st->str, newstr);
+	free(st->str);
+	st->str = newstr2;
+	free(newstr);
+        va_end(va);
+
+        return st;
+}
+
+
+bool nc_string_cat(nc_string *s, const char *append)
+{
+        char *p = NULL;
+        int len = 0;
+
+        if (!s || !append) {
+                return false;
+        }
+        if (!s->str) {
+                return false;
+        }
+        len = asprintf(&p, "%s%s", s->str, append);
+        if (!p || len < s->len) {
+                return false;
+        }
+        free(s->str);
+        s->str = p;
+        s->len = len;
+        return true;
+}
+
+bool nc_string_prepend(nc_string *s, const char *prepend)
+{
+        char *p = NULL;
+        int len = 0;
+
+        if (!s || !prepend) {
+                return false;
+        }
+        if (!s->str) {
+                return false;
+        }
+        len = asprintf(&p, "%s%s", prepend, s->str);
+        if (!p || len < s->len) {
+                return false;
+        }
+        free(s->str);
+        s->str = p;
+        s->len = len;
+        return true;
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/nc-string.h

@@ -0,0 +1,220 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "macros.h"
+
+/**
+ * Safely represent and store a buffer as a string
+ */
+typedef struct nc_string_t {
+        char *str; /**<Buffer holding a NUL-terminated string */
+        int len;   /**<Current length of the string */
+} nc_string;
+
+/**
+ * Construct a new string using printf style syntax
+ *
+ * @param format Printf-style format string
+ * @param ... Variable arguments
+ *
+ * @return a new nc_string if allocation succeeded, otherwise NULL
+ */
+_nica_public_ nc_string *nc_string_dup_printf(const char *format, ...)
+    __attribute__((format(printf, 1, 2)));
+
+
+_nica_public_ nc_string *nc_string_append_printf(nc_string *st, const char *ptn, ...)
+	__attribute__((format(printf, 2, 3)));
+
+/**
+ * Duplicate a string into a new NUL-terminated nc_string
+ *
+ * @param str Pointer to a string
+ * @return a new nc_string if allocation succeeded, otherwise NULL
+ */
+_nica_public_ nc_string *nc_string_dup(const char *str);
+
+/**
+ * Free an existing string and its contents
+ *
+ * @param str Pointer to an existing string
+ */
+static inline void nc_string_free(nc_string *str)
+{
+        if (!str) {
+                return;
+        }
+        if (str->str) {
+                free(str->str);
+        }
+        free(str);
+}
+
+/**
+ * Append the contents of 'append' into the given nc_string
+ *
+ * @param str Pointer to a valid nc_string
+ * @param append Text to append into the nc_string
+ *
+ * @return a boolean value indicating success
+ */
+_nica_public_ bool nc_string_cat(nc_string *str, const char *append);
+
+_nica_public_ bool nc_string_prepend(nc_string *s, const char *prepend);
+
+
+/**
+ * Determine if string A is equal to string B
+ *
+ * @note This function will not check beyond the length of string A
+ *
+ * @param a string to check
+ * @param b string to check against
+ * @return a boolean value, true if the strings match, otherwise false
+ */
+static inline bool nc_string_equal(nc_string *a, nc_string *b)
+{
+        if (!a || !b) {
+                return false;
+        }
+        if (!a->str || !b->str) {
+                return false;
+        }
+        return strncmp(a->str, b->str, (size_t)a->len) == 0;
+}
+
+/**
+ * Determine if string A is equal to string B
+ *
+ * @note This function will not check beyond the length of string A
+ *
+ * @param a string to check
+ * @param b const char* string to check against
+ * @return a boolean value, true if the strings match, otherwise false
+ */
+static inline bool nc_string_const_equal(nc_string *a, const char *b)
+{
+        if (!a || !b) {
+                return false;
+        }
+        if (!a->str) {
+                return false;
+        }
+        return strncmp(a->str, b, (size_t)a->len) == 0;
+}
+
+/**
+ * Convenience wrapper to obtain string length
+ *
+ * @note This will assert the string is not NULL, ensuring a termination.
+ *
+ * @return length of the given string
+ */
+static inline int ncstrlen(nc_string *str)
+{
+        assert(str != NULL);
+        return str->len;
+}
+
+/**
+ * Determine if a string has the given suffix
+ *
+ * @param str Valid nc_string
+ * @param suffix A suffix to check
+ * @param len Length of the suffix
+ *
+ * @return True if @str has the given suffix
+ */
+static inline bool nc_string_has_suffix_const(nc_string *str, char *suffix, ssize_t len)
+{
+        if (!str || !suffix) {
+                return false;
+        }
+        if (len > str->len || len <= 0) {
+                return false;
+        }
+        return (strncmp(str->str + (str->len - (len)), suffix, (size_t)len) == 0);
+}
+
+/**
+ * Determine if a string has the given prefix
+ *
+ * @param str Valid nc_string
+ * @param prefix A prefix to check
+ * @param len Length of the prefix
+ *
+ * @return True if @str has the given prefix
+ */
+static inline bool nc_string_has_prefix_const(nc_string *str, char *prefix, ssize_t len)
+{
+        if (!str || !prefix) {
+                return false;
+        }
+        if (len > str->len || len <= 0) {
+                return false;
+        }
+        return (strncmp(str->str, prefix, (size_t)len) == 0);
+}
+
+/**
+ * Perform inplace modification of a strip to return
+ * a stripped pointer, that is, without start and end whitespace.
+ *
+ * This does not modify the length of the string.
+ */
+static inline char *nc_string_strip(nc_string *str)
+{
+        int i;
+
+        if (!str || !str->str) {
+                return NULL;
+        }
+        for (i = 0; i < str->len; i++) {
+                if (str->str[i] != ' ') {
+                        break;
+                }
+        }
+        for (int j = str->len - 1; j > i; j--) {
+                if (str->str[j] != ' ') {
+                        str->str[j + 1] = '\0';
+                        break;
+                }
+        }
+
+        return str->str + i;
+}
+/**
+ * To be used only with compile time constants.
+ */
+#define nc_string_has_suffix(a, suff) nc_string_has_suffix_const(a, suff, (sizeof(suff) - 1))
+#define nc_string_has_prefix(a, pref) nc_string_has_prefix_const(a, pref, (sizeof(pref) - 1))
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/util.h

@@ -0,0 +1,69 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <nica/macros.h>
+
+#define DEF_AUTOFREE(N, C)                                                                         \
+        static inline void _autofree_func_##N(void *p)                                             \
+        {                                                                                          \
+                if (p && *(N **)p) {                                                               \
+                        C(*(N **)p);                                                               \
+                        (*(void **)p) = NULL;                                                      \
+                }                                                                                  \
+        }
+
+#define autofree(N) __attribute__((cleanup(_autofree_func_##N))) N
+
+/**
+ * Dump any leaked file descriptors
+ */
+_nica_public_ void nc_dump_file_descriptor_leaks(void);
+
+_nica_public_ void *greedy_realloc(void **p, size_t *allocated, size_t need);
+
+/**
+ * Test string equality of two null terminated strings
+ */
+__attribute__((always_inline)) static inline bool streq(const char *a, const char *b)
+{
+        if (a == b) {
+                return true;
+        }
+        if (!a || !b) {
+                return false;
+        }
+        return strcmp(a, b) == 0;
+}
+
+DEF_AUTOFREE(char, free)
+DEF_AUTOFREE(FILE, fclose)
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/probes/README.md

@@ -0,0 +1,53 @@
+# Probes
+
+The default probes provided along the telemetry client code are:
+
+* bertprobe: this probe reports on the Boot Error Region Table if such
+  entry is found in ```/sys/firmware/acpi/tables/BERT```.
+
+* crashprobe: This probe processes core dump files. It can be registered as
+  the kernel core file handler in /proc/sys/kernel/core_pattern.
+
+* hprobe: a simple probe that sends a keep alive message. This probe can also
+  be useful for testing
+
+* journalprobe: a probe that monitors the systemd journal for log messages
+  from failed services.
+
+* oopsprobe: a probe to collect 'oops messages' when the kernel detects a
+  problem.
+
+* pstoreprobe: probe to collect messages left on pstore filesystem.
+
+* telem-record-gen: this is a "general-purpose probe" for sending custom
+  records on-the-fly. This tool can be used stand alone or as a part of a
+  script to implement a probe.
+
+For example to generate an hprobe like message we can execute the following
+command:
+
+```
+$ telem-record-gen --class org.example/hello/world --payload hello
+```
+
+for more options use the ```--help``` switch i.e.:
+
+```
+$ telem-record-gen --help
+Usage:
+  telem-record-gen [OPTIONS] - create and send a custom telemetry record
+
+Help Options:
+  -h, --help            Show help options
+
+Application Options:
+  -f, --config-file     Path to configuration file (not implemented yet)
+  -V, --version         Print the program version
+  -s, --severity        Severity level (1-4) - (default 1)
+  -c, --class           Classification level_1/level_2/level_3
+  -p, --payload         Record body (max size = 8k)
+  -P, --payload-file    File to read payload from
+  -R, --record-version  Version number for format of payload (default 1)
+  -e, --event-id        Event id to use in the record
+
+```

src/probes/bert_probe.c

@@ -0,0 +1,142 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <getopt.h>
+#include "common.h"
+#include "telemetry.h"
+#include "nica/b64enc.h"
+
+#include "config.h"
+
+static const char bert_record_file[] = "/sys/firmware/acpi/tables/data/BERT";
+static const char telem_record_class[] = "org.clearlinux/bert/debug";
+static uint32_t severity = 2;
+static uint32_t payload_version = 1;
+
+void print_usage(char *prog)
+{
+        printf("%s: Usage\n", prog);
+        printf("  -f,  --config_file    Specify a configuration file other than default\n");
+        printf("  -h,  --help           Display this help message\n");
+        printf("  -V,  --version        Print the program version\n");
+}
+
+static int allocate_payload_buffer(char **payload)
+{
+        int ret = 0;
+
+        *payload = (char *)malloc(MAX_PAYLOAD_SIZE);
+
+        if (*payload == NULL) {
+                goto out;
+        }
+
+        *payload = memset(*payload, 0, MAX_PAYLOAD_SIZE);
+        ret = 1;
+
+out:
+        return ret;
+}
+
+int main(int argc, char **argv)
+{
+        struct telem_ref *tm_handle = NULL;
+        char *classification = (char *)telem_record_class;
+        char *payload = NULL;
+
+        // Following vars are for arg parsing.
+        int c;
+        char *cfile = NULL;
+        int opt_index = 0;
+        int ret = 0;
+
+        struct option opts[] = {
+                { "config_file", 1, NULL, 'f' },
+                { "help", 0, NULL, 'h' },
+                { "version", 0, NULL, 'V' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "f:hHV", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'f':
+                                if (asprintf(&cfile, "%s", (char *)optarg) < 0) {
+                                        exit(EXIT_FAILURE);
+                                }
+
+                                struct stat buf;
+                                ret = stat(cfile, &buf);
+
+                                /* check if the file exists and is a regular file */
+                                if (ret == -1 || !S_ISREG(buf.st_mode)) {
+                                        exit(EXIT_FAILURE);
+                                }
+
+                                tm_set_config_file(cfile);
+                                break;
+                        case 'h':
+                                print_usage(argv[0]);
+                                exit(EXIT_SUCCESS);
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        if (!(ret = allocate_payload_buffer(&payload))) {
+                printf("Unable to allocate more memory.\n");
+                goto fail;
+        }
+
+        if (!(ret = nc_b64enc_filename(bert_record_file, payload, MAX_PAYLOAD_SIZE))) {
+                printf("Failed to read payload from: %s\n", bert_record_file);
+                goto fail;
+        }
+
+        if ((ret = tm_create_record(&tm_handle, severity, classification,
+                                    payload_version)) < 0) {
+                printf("Failed to create record: %s\n", strerror(-ret));
+                goto fail;
+        }
+
+        if ((ret = tm_set_payload(tm_handle, payload)) < 0) {
+                printf("Failed to set record payload: %s\n", strerror(-ret));
+                goto fail;
+        }
+
+        free(payload);
+
+        if ((ret = tm_send_record(tm_handle)) < 0) {
+                printf("Failed to send record to daemon: %s\n", strerror(-ret));
+                goto fail;
+        }
+
+fail:
+        free(cfile);
+        tm_free_record(tm_handle);
+        tm_handle = NULL;
+
+        return ret;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/crash_probe.c

@@ -17,6 +17,7 @@
 #define _GNU_SOURCE
 #include <assert.h>
 #include <fcntl.h>
+#include <getopt.h>
 #include <grp.h>
 #include <inttypes.h>
 #include <limits.h>
@@ -31,13 +32,13 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
-#include <glib.h>
-
 #include <libelf.h>
 #include <elfutils/libdwfl.h>
 
+#include "nica/nc-string.h"
 #include "config.h"
 #include "log.h"
+#include "probe.h"
 #include "telemetry.h"
 
 static Dwfl *d_core = NULL;
@@ -53,10 +54,10 @@ static Dwfl *d_core = NULL;
 static char *debuginfo_path = "-/usr/lib/debug";
 
 static unsigned int frame_counter = 0;
-static gchar *proc_name = NULL;
+static char *proc_name = NULL;
 static pid_t core_for_pid = 0;
-static GString *header = NULL;
-static gchar *errorstr = NULL;
+static nc_string *header = NULL;
+static char *errorstr = NULL;
 
 static uint32_t unknown_severity = 2;
 static uint32_t default_severity = 3;
@@ -67,6 +68,9 @@ static char clr_build_class[40] = "org.clearlinux/crash/clr-build";
 static char error_class[30] = "org.clearlinux/crash/error";
 static char unknown_class[30] = "org.clearlinux/crash/unknown";
 
+static char temp_core[] = "/tmp/corefile-XXXXXX";
+static bool keep_core = false;
+
 static const Dwfl_Callbacks cb =
 {
         .find_elf = dwfl_build_id_find_elf,
@@ -84,9 +88,17 @@ static inline void tm_dwfl_err(const char *msg)
         telem_log(LOG_ERR, "%s: %s\n", msg, dwfl_errmsg(-1));
 }
 
-static gchar *replace_exclamations(gchar *str)
+static char *replace_exclamations(char *str)
 {
-        return g_strdelimit(str, "!", '/');
+        char *c;
+        while (1) {
+                c = strchr(str, '!');
+                if (!c) {
+                        break;
+                }
+                *c = '/';
+        }
+        return str;
 }
 
 static void drop_privs(void)
@@ -138,23 +150,14 @@ static int temp_core_file(void)
         int tmp;
         ssize_t ret;
 
-        char core[PATH_MAX] = "/tmp/corefile-XXXXXX";
-
         /* mkstemp() opens the file with O_EXCL and 0600 permissions, so no need
          * to change umask or manipulate the fd to meet those requirements.
          */
-        if ((tmp = mkstemp(core)) < 0) {
+        if ((tmp = mkstemp(temp_core)) < 0) {
                 telem_perror("Failed to create temp core file");
                 return -1;
         }
 
-#ifndef DEBUG
-        if (unlink(core) < 0) {
-                telem_perror("Failed to unlink temp core file");
-                return -1;
-        }
-#endif
-
         while (true) {
                 // Use Linux-specific splice(2) here;
                 // simplifies copying data from pipe->file
@@ -176,6 +179,55 @@ static int temp_core_file(void)
         return tmp;
 }
 
+/* There are a number of initialization routines required to initialize the Elf
+ * and Dwfl objects used by libdwfl to later process a core file. The argument
+ * e_core is the address of an Elf pointer declared by the caller, and core_fd
+ * is the open file descriptor for the core file.
+ */
+static int prepare_corefile(Elf **e_core, int core_fd)
+{
+        // Cleanup previous corefile processing if needed
+        if (d_core) {
+                dwfl_end(d_core);
+        }
+        if (*e_core) {
+                elf_end(*e_core);
+        }
+
+        // Boilerplate initialization for elfutils (libdwfl)
+        if (!(*e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
+                tm_elf_err("Failed to get file descriptor for ELF core file");
+                goto fail;
+        }
+
+        if (!(d_core = dwfl_begin(&cb))) {
+                tm_dwfl_err("Failed to start new libdwfl session");
+                goto fail;
+        }
+
+        int core_modules = 0;
+        core_modules = dwfl_core_file_report(d_core, *e_core, NULL);
+        if (core_modules == -1) {
+                tm_dwfl_err("Failed to report modules for ELF core file");
+                goto fail;
+        }
+
+        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
+                tm_dwfl_err("Failed to finish reporting modules");
+                goto fail;
+        }
+
+        if ((core_for_pid = dwfl_core_file_attach(d_core, *e_core)) < 0) {
+                tm_dwfl_err("Failed to prepare libdwfl session for thread"
+                            " iteration");
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
 /* This callback is invoked for every frame in a thread. From a Dwfl_Frame, we
  * are able to extract the program counter (PC), and from that, the procedure
  * name via a Dwfl_Module.
@@ -185,15 +237,20 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
         Dwarf_Addr pc;
         Dwarf_Addr pc_adjusted;
         Dwfl_Module *module;
+        Dwfl_Line *line;
         const char *procname;
         const char *modname;
         bool activation;
-        GString **bt = (GString **)userdata;
+        nc_string **bt = (nc_string **)userdata;
 
         if (!dwfl_frame_pc(frame, &pc, &activation)) {
-                errorstr = g_strdup_printf("Failed to find program counter for"
-                                           " current frame: %s\n",
-                                           dwfl_errmsg(-1));
+                int ret;
+                ret = asprintf(&errorstr, "Failed to find program counter for"
+                               " current frame: %s\n",
+                               dwfl_errmsg(-1));
+                if (ret < 0) {
+                        return DWARF_CB_ABORT;
+                }
                 return DWARF_CB_ABORT;
         }
 
@@ -210,8 +267,8 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
         module = dwfl_addrmodule(d_core, pc_adjusted);
 
         if (!module) {
-                errorstr = g_strdup("Failed to find module for current"
-                                    " frame\n");
+                errorstr = strdup("Failed to find module for current"
+                                  " frame\n");
                 return DWARF_CB_ABORT;
         }
 
@@ -219,41 +276,54 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
                                    NULL);
         procname = dwfl_module_addrname(module, pc_adjusted);
 
+        line = dwfl_module_getsrc(module, pc_adjusted);
+
         if (procname && modname) {
-                g_string_append_printf(*bt, "#%u %s() - [%s]\n",
-                                       frame_counter++, procname, modname);
+                nc_string_append_printf(*bt, "#%u %s() - [%s]",
+                                        frame_counter++, procname, modname);
         } else if (modname) {
-                g_string_append_printf(*bt, "#%u ??? - [%s]\n",
-                                       frame_counter++, modname);
+                nc_string_append_printf(*bt, "#%u ??? - [%s]",
+                                        frame_counter++, modname);
         } else {
                 // TODO: decide on "no symbol" representation
-                g_string_append_printf(*bt, "#%u (no symbols)\n",
-                                       frame_counter++);
+                nc_string_append_printf(*bt, "#%u (no symbols)",
+                                        frame_counter++);
         }
 
+        if (line) {
+                const char *src;
+                int lineno, linecol;
+                src =  dwfl_lineinfo(line, &pc_adjusted, &lineno, &linecol, NULL, NULL);
+                if (src) {
+                        nc_string_append_printf(*bt, " - %s:%i", src, lineno);
+                }
+        }
+        nc_string_append_printf(*bt, "\n");
         return DWARF_CB_OK;
 }
 
 static int thread_cb(Dwfl_Thread *thread, void *userdata)
 {
         int ret;
-        GString **bt = (GString **)userdata;
+        nc_string **bt = (nc_string **)userdata;
         pid_t tid;
 
         tid = dwfl_thread_tid(thread);
 
-        g_string_append_printf(*bt, "\nBacktrace (TID %u):\n",
-                               (unsigned int)tid);
+        nc_string_append_printf(*bt, "\nBacktrace (TID %u):\n",
+                                (unsigned int)tid);
 
         ret = dwfl_thread_getframes(thread, frame_cb, userdata);
 
         switch (ret) {
                 case -1:
-                        errorstr = g_strdup_printf("Error while iterating"
-                                                   " through frames for thread"
-                                                   " %u: %s\n",
-                                                   (unsigned int)tid,
-                                                   dwfl_errmsg(-1));
+                        if (asprintf(&errorstr, "Error while iterating"
+                                     " through frames for thread"
+                                     " %u: %s\n",
+                                     (unsigned int)tid,
+                                     dwfl_errmsg(-1)) < 0) {
+                                errorstr = NULL;
+                        }
                         return DWARF_CB_ABORT;
                 case DWARF_CB_ABORT:
                         /* already set the error string in frame_cb */
@@ -269,14 +339,14 @@ static int thread_cb(Dwfl_Thread *thread, void *userdata)
         frame_counter = 0;
 
 #if 0
-        g_string_append_printf(*bt, "\nRegisters (TID %u):\nTODO\n", current,
-                               (unsigned int)tid);
+        nc_string_append_printf(*bt, "\nRegisters (TID %u):\nTODO\n", current,
+                                (unsigned int)tid);
 #endif
 
         return DWARF_CB_OK;
 }
 
-static bool send_data(GString **backtrace, uint32_t severity, char *class)
+static bool send_data(nc_string **backtrace, uint32_t severity, char *class)
 {
         struct telem_ref *handle = NULL;
         int ret;
@@ -305,80 +375,137 @@ fail:
         return false;
 }
 
-static bool in_clr_build(gchar *fullpath)
+/* This is the entry point for libdwfl to unwind the backtrace from the core
+ * file. All data necessary for processing is referenced by d_core (a *Dwfl),
+ * which must be initialized prior to calling this function.  The callback
+ * function for processing each core thread is passed as the second argument to
+ * dwfl_getthreads(). The backtrace argument is the address of a pointer to a
+ * nc_string object declared by the caller which stores the backtrace data as a
+ * string.
+ */
+static int process_corefile(nc_string **backtrace)
 {
+        if (*backtrace) {
+                nc_string_free(*backtrace);
+        }
+        *backtrace = nc_string_dup("");
+
+        if (dwfl_getthreads(d_core, thread_cb, backtrace) != DWARF_CB_OK) {
+                /* When errors occur during the unwinding, we reach this point.
+                 * If an error string is set for the particular error, send an
+                 * "error" record to capture at least a partial backtrace if
+                 * some frames were processed.
+                 */
+                if (errorstr != NULL) {
+                        telem_log(LOG_ERR, "%s", errorstr);
+                        nc_string_append_printf(header, "Error: %s", errorstr);
+                        nc_string_prepend(*backtrace, header->str);
+                        send_data(backtrace, error_severity, error_class);
+                }
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
+static bool startswith(const char *full, const char *prefix)
+{
+        while (*prefix) {
+                if (*prefix != *full) {
+                        return false;
+                }
+                full++;
+                prefix++;
+        }
+        return true;
+}
+
+static bool in_clr_build(char *fullpath)
+{
+        // Global override for privacy filters
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                return false;
+        }
+
         /*
          * The build environment for Clear Linux packages is set up by 'mock',
          * and the chroot in which rpmbuild builds the packages has this prefix.
          */
-        if (g_regex_match_simple("^[!/]builddir[!/]build[!/]BUILD[!/]",
-                                 fullpath, 0, 0) == TRUE) {
+        if (startswith(fullpath, "/builddir/build/BUILD/") ||
+            startswith(fullpath, "!builddir!build!BUILD!")) {
                 return true;
         }
 
         return false;
 }
 
-static bool filter_binaries(gchar *fullpath)
+static bool is_banned_path(char *fullpath)
 {
-        // Anything outside of /usr/, or in /usr/local/, we consider third-party
-        if (g_regex_match_simple("^[!/]usr[!/]", fullpath, 0, 0) == FALSE) {
-                return false;
-        } else if (g_regex_match_simple("^[!/]usr[!/]local[!/]", fullpath,
-                                        0, 0) == TRUE) {
+        // Global override for privacy filters
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
                 return false;
         }
 
-        return true;
+        // Anything outside of /usr/, or in /usr/local/, we consider third-party
+
+        if (!startswith(fullpath, "/usr/") &&
+            !startswith(fullpath, "!usr!")) {
+                return true;
+        }
+
+        if (startswith(fullpath, "/usr/local/") ||
+            startswith(fullpath, "!usr!local!")) {
+                return true;
+        }
+
+        return false;
 }
 
-static gchar *config_file = NULL;
-static gchar *core_file = NULL;
-static gchar *proc_path = NULL;
-static gint signal_num = -1;
-static gboolean version_p = FALSE;
-static gboolean verbose = FALSE;
+static char *config_file = NULL;
+static char *core_file = NULL;
+static char *proc_path = NULL;
+static long int signal_num = -1;
+static bool verbose = false;
 
-static GOptionEntry options[] = {
-        { "config-file", 'f', 0, G_OPTION_ARG_FILENAME, &config_file,
-          "Path to configuration file (not implemented yet)", NULL },
-        { "core-file", 'c', 0, G_OPTION_ARG_FILENAME, &core_file,
-          "Path to core file to process", NULL },
-        { "process-name", 'p', 0, G_OPTION_ARG_STRING, &proc_name,
-          "Name of process for crash report (required)", NULL },
-        { "process-path", 'E', 0, G_OPTION_ARG_STRING, &proc_path,
-          "Absolute path of crashed process, with ! or / delimiters", NULL },
-        { "signal", 's', 0, G_OPTION_ARG_INT, &signal_num,
-          "Signal number that crashed the process", NULL },
-        { "version", 'V', 0, G_OPTION_ARG_NONE, &version_p,
-          "Print the program version", NULL },
-        { "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose,
-          "Print the crash probe payload to stdout", NULL },
-        { NULL }
+static const struct option prog_opts[] = {
+        { "help", no_argument, 0, 'h' },
+        { "config-file", required_argument, 0, 'f' },
+        { "core-file", required_argument, 0, 'c' },
+        { "process-name", required_argument, 0, 'p' },
+        { "process-path", required_argument, 0, 'E' },
+        { "signal", required_argument, 0, 's' },
+        { "version", no_argument, 0, 'V' },
+        { "verbose", no_argument, 0, 'v' },
+        { 0, 0, 0, 0 }
 };
 
-static void free_glib_strings(void)
+static void print_help(void)
 {
-        if (core_file) {
-                g_free(core_file);
-        }
-        if (proc_name) {
-                g_free(proc_name);
-        }
-        if (proc_path) {
-                g_free(proc_path);
-        }
+        printf("Usage:\n");
+        printf("  crashprobe [OPTIONS] - collect data from core files\n");
+        printf("\n");
+        printf("Help Options:\n");
+        printf("  -h, --help            Show help options\n");
+        printf("\n");
+        printf("Application Options:\n");
+        printf("  -f, --config-file     Path to configuration file (not implemented yet)\n");
+        printf("  -c, --core-file       Path to core file to process\n");
+        printf("  -p, --process-name    Name of process for crash report (required)\n");
+        printf("  -E, --process-path    Absolute path of crashed process, with ! or / delimiters\n");
+        printf("  -s, --signal          Signal number that crashed the process\n");
+        printf("  -V, --version         Print the program version\n");
+        printf("  -v, --verbose         Print the crash payload to stdout\n");
+        printf("\n");
 }
 
 int main(int argc, char **argv)
 {
         Elf *e_core = NULL;
-        int core_modules = 0;
         int ret = EXIT_FAILURE;
         int core_fd = STDIN_FILENO;
-        GString *backtrace = NULL;
-        GError *error = NULL;
-        GOptionContext *context;
+        nc_string *backtrace = NULL;
 
         if (fcntl(STDERR_FILENO, F_GETFL) < 0) {
                 // redirect stderr to avoid bad things to happen with
@@ -394,46 +521,54 @@ int main(int argc, char **argv)
 
         drop_privs();
 
-        context = g_option_context_new("- collect data from core files");
-        g_option_context_add_main_entries(context, options, NULL);
-        g_option_context_set_translate_func(context, NULL, NULL, NULL);
-        if (!g_option_context_parse(context, &argc, &argv, &error)) {
-                g_print("Failed to parse options: %s\n", error->message);
-                exit(EXIT_FAILURE);
-        }
+        int opt;
 
-        if (version_p) {
-                g_print(PACKAGE_VERSION "\n");
-                exit(EXIT_SUCCESS);
+        while ((opt = getopt_long(argc, argv, "hf:c:p:E:s:Vv", prog_opts, NULL)) != -1) {
+                switch (opt) {
+                        case 'h':
+                                print_help();
+                                goto success;
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                goto success;
+                        case 'f':
+                                config_file = strdup(optarg);
+                                break;
+                        case 'c':
+                                core_file = strdup(optarg);
+                                break;
+                        case 'p':
+                                proc_name = strdup(optarg);
+                                break;
+                        case 'E':
+                                proc_path = strdup(optarg);
+                                break;
+                        case 's':
+                                errno = 0;
+                                char *endptr = NULL;
+
+                                signal_num = strtol(optarg, &endptr, 10);
+
+                                if (errno != 0) {
+                                        telem_perror("Failed to convert signal number");
+                                        goto fail;
+                                }
+                                if (endptr && *endptr != '\0') {
+                                        telem_log(LOG_ERR, "Invalid signal number. Must be an integer\n");
+                                        goto fail;
+                                }
+                                break;
+                        case 'v':
+                                verbose = true;
+                                break;
+                }
         }
 
         if (!proc_name) {
-                g_print("Missing required -p option. See --help output\n");
+                printf("Missing required -p option. See --help output\n");
                 exit(EXIT_FAILURE);
         }
 
-        if (proc_path && in_clr_build(proc_path)) {
-                telem_log(LOG_NOTICE, "Ignoring core (from mock build)\n");
-
-                backtrace = g_string_new("Crash from Clear package build\n");
-
-                if (!send_data(&backtrace, unknown_severity, clr_build_class)) {
-                        goto fail;
-                }
-                goto success;
-        }
-
-        if (proc_path && !filter_binaries(proc_path)) {
-                telem_log(LOG_NOTICE, "Ignoring core (third-party binary)\n");
-
-                backtrace = g_string_new("Crash from third party\n");
-
-                if (!send_data(&backtrace, unknown_severity, unknown_class)) {
-                        goto fail;
-                }
-                goto success;
-        }
-
         if (core_file) {
                 core_fd = open(core_file, O_RDONLY);
                 if (core_fd == -1) {
@@ -460,62 +595,74 @@ int main(int argc, char **argv)
                                 goto fail;
                         }
                 } else {
-                        g_print("Cannot process core file. Use the -c option,"
-                                " or pass the core file on stdin.\n");
+                        printf("Cannot process core file. Use the -c option,"
+                               " or pass the core file on stdin.\n");
                         goto fail;
                 }
         }
 
+        if (proc_path && in_clr_build(proc_path)) {
+                telem_log(LOG_NOTICE, "Ignoring core (from mock build)\n");
+
+                backtrace = nc_string_dup("Crash from Clear package build\n");
+
+                keep_core = true;
+
+                if (!send_data(&backtrace, unknown_severity, clr_build_class)) {
+                        goto fail;
+                }
+                goto success;
+        }
+
+        if (proc_path && is_banned_path(proc_path)) {
+                telem_log(LOG_NOTICE, "Ignoring core (third-party binary)\n");
+
+                backtrace = nc_string_dup("Crash from third party\n");
+
+                keep_core = true;
+
+                if (!send_data(&backtrace, unknown_severity, unknown_class)) {
+                        goto fail;
+                }
+                goto success;
+        }
+
         elf_version(EV_CURRENT);
 
-        if (!(e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
-                tm_elf_err("Failed to get file descriptor for ELF core file");
+        if (prepare_corefile(&e_core, core_fd) < 0) {
                 goto fail;
         }
 
-        if (!(d_core = dwfl_begin(&cb))) {
-                tm_dwfl_err("Failed to start new libdwfl session");
-                goto fail;
-        }
-
-        core_modules = dwfl_core_file_report(d_core, e_core, NULL);
-        if (core_modules == -1) {
-                tm_dwfl_err("Failed to report modules for ELF core file");
-                goto fail;
-        }
-
-        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
-                tm_dwfl_err("Failed to finish reporting modules");
-                goto fail;
-        }
-
-        if ((core_for_pid = dwfl_core_file_attach(d_core, e_core)) < 0) {
-                tm_dwfl_err("Failed to prepare libdwfl session for thread"
-                            " iteration");
-                goto fail;
-        }
-
-        header = g_string_new(NULL);
-        g_string_printf(header, "Process: %s\nPID: %u\n",
-                        proc_path ? replace_exclamations(proc_path) : proc_name,
-                        (unsigned int)core_for_pid);
+        header = nc_string_dup_printf("Process: %s\nPID: %u\n",
+                                      proc_path ? replace_exclamations(proc_path) : proc_name,
+                                      (unsigned int)core_for_pid);
 
         if (signal_num >= 0) {
-                g_string_append_printf(header, "Signal: %d\n", signal_num);
+                nc_string_append_printf(header, "Signal: %ld\n", signal_num);
         }
 
-        backtrace = g_string_new(NULL);
-        if (dwfl_getthreads(d_core, thread_cb, &backtrace) != DWARF_CB_OK) {
-                if (errorstr != NULL) {
-                        telem_log(LOG_ERR, "%s", errorstr);
-                        g_string_append_printf(header, "Error: %s", errorstr);
-                        g_string_prepend(backtrace, header->str);
-                        send_data(&backtrace, error_severity, error_class);
-                }
+        if (process_corefile(&backtrace) < 0) {
                 goto fail;
         }
 
-        g_string_prepend(backtrace, header->str);
+        /* On Clear Linux OS, missing symbols may appear if automatic debuginfo
+         * downloads are still in flight. So if any missing symbols appear on
+         * the first run (indicated by the presence of "??? - ["), wait 10
+         * seconds and try again.
+         */
+        if (strstr(backtrace->str, "??? - [")) {
+                sleep(10);
+
+                if (prepare_corefile(&e_core, core_fd) < 0) {
+                        goto fail;
+                }
+
+                if (process_corefile(&backtrace) < 0) {
+                        goto fail;
+                }
+        }
+
+        nc_string_prepend(backtrace, header->str);
 
         if (!send_data(&backtrace, default_severity, clr_class)) {
                 goto fail;
@@ -530,22 +677,25 @@ success:
 
         ret = EXIT_SUCCESS;
 fail:
-        free_glib_strings();
-
-        if (context) {
-                g_option_context_free(context);
+        // Do not remove the core file if any errors occur
+        if (ret == EXIT_FAILURE) {
+                keep_core = true;
         }
 
+        free(core_file);
+        free(proc_name);
+        free(proc_path);
+
         if (header) {
-                g_string_free(header, TRUE);
+                nc_string_free(header);
         }
 
         if (backtrace) {
-                g_string_free(backtrace, TRUE);
+                nc_string_free(backtrace);
         }
 
         if (errorstr) {
-                g_free(errorstr);
+                free(errorstr);
         }
 
         if (d_core) {
@@ -560,6 +710,12 @@ fail:
                 close(core_fd);
         }
 
+        // Remove the core file by default, except when the --core-file option
+        // is specified, or when keep_core is overridden to true.
+        if (!core_file && !keep_core) {
+                unlink(temp_core);
+        }
+
         return ret;
 }
 

src/probes/hello.c

@@ -49,7 +49,7 @@ int main(int argc, char **argv)
         int ret = 0;
 
         struct option opts[] = {
-                { "cfile", 1, NULL, 'f' },
+                { "config_file", 1, NULL, 'f' },
                 { "heartbeat", 0, NULL, 'H' },
                 { "help", 0, NULL, 'h' },
                 { "version", 0, NULL, 'V' },

src/probes/journal.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2015-2017 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -15,6 +15,7 @@
  */
 
 #define _GNU_SOURCE
+#include <getopt.h>
 #include <poll.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -22,8 +23,6 @@
 #include <string.h>
 #include <unistd.h>
 
-#include <glib.h>
-
 /* Certain static analysis tools do not understand GCC's __INCLUDE_LEVEL__
  * macro; the conditional definition below is used to fix the build with
  * systemd's _sd-common.h, included by the public systemd headers, which rely on
@@ -39,10 +38,11 @@
 #include "config.h"
 #include "log.h"
 #include "telemetry.h"
-
+#include "probe.h"
+#include "nica/nc-string.h"
 #define BOOT_ID_LEN 33
 
-static GString *payload = NULL;
+static nc_string *payload = NULL;
 static uint32_t severity = 2;
 static uint32_t payload_version = 1;
 static char error_class[30] = "org.clearlinux/journal/error";
@@ -61,12 +61,11 @@ static inline void tm_journal_match_err(int ret)
 static void add_to_payload(const void *data, size_t length)
 {
         if (payload != NULL) {
-                g_string_append_printf(payload, "%.*s", (int)length,
-                                       (char *)data);
+                nc_string_append_printf(payload, "%.*s\n", (int)length,
+                                        (char *)data);
         } else {
-                payload = g_string_new(NULL);
-                g_string_printf(payload, "%.*s", (int)length,
-                                (char *)data);
+                payload = nc_string_dup_printf("%.*s\n", (int)length,
+                                               (char *)data);
         }
 }
 
@@ -77,16 +76,17 @@ static bool send_data(char *class)
 
         if ((ret = tm_create_record(&handle, severity, class,
                                     payload_version)) < 0) {
-                telem_log(LOG_ERR, "Failed to create record: %s",
+                telem_log(LOG_ERR, "Failed to create record: %s\n",
                           strerror(-ret));
                 goto fail;
         }
 
-        gchar *payload_str = g_string_free(payload, FALSE);
+        char *payload_str = strdup(payload->str);
+        nc_string_free(payload);
         payload = NULL;
 
         if ((ret = tm_set_payload(handle, (char *)payload_str)) < 0) {
-                telem_log(LOG_ERR, "Failed to set payload: %s", strerror(-ret));
+                telem_log(LOG_ERR, "Failed to set payload: %s\n", strerror(-ret));
                 free(payload_str);
                 tm_free_record(handle);
                 goto fail;
@@ -95,7 +95,7 @@ static bool send_data(char *class)
         free(payload_str);
 
         if ((ret = tm_send_record(handle)) < 0) {
-                telem_log(LOG_ERR, "Failed to send record: %s", strerror(-ret));
+                telem_log(LOG_ERR, "Failed to send record: %s\n", strerror(-ret));
                 tm_free_record(handle);
                 goto fail;
         }
@@ -122,6 +122,16 @@ static int read_new_entries(sd_journal *journal)
 
                 add_to_payload(data, length);
 
+                // For now, we send one record per log message, in case the
+                // there is a large backlog of messages and we exceed the
+                // payload size limit (8KB). And ignore errors, hoping that it's
+                // a transient problem.
+
+                if (!send_data(error_class)) {
+                        telem_log(LOG_ERR, "Failed to send data. Ignoring.\n");
+                        return num_entries;
+                }
+
                 num_entries++;
         }
 
@@ -148,19 +158,13 @@ static bool process_existing_entries(sd_journal *journal)
                 return false;
         }
 
-        if (!read_new_entries(journal)) {
+        ret = read_new_entries(journal);
+        if (ret < 0) {
                 return false;
-        }
-
-        if (!payload) {
-                telem_log(LOG_DEBUG, "No existing entries found\n");
+        } else if (ret == 0) {
                 return true;
         }
 
-        if (!send_data(error_class)) {
-                return false;
-        }
-
         return true;
 }
 
@@ -185,6 +189,33 @@ static bool get_boot_id(char **data)
         return true;
 }
 
+#define JOURNAL_MATCH(data) \
+        do { \
+                r = sd_journal_add_match(journal, data, 0); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
+#define JOURNAL_AND \
+        do { \
+                r = sd_journal_add_conjunction(journal); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
+#define JOURNAL_OR \
+        do { \
+                r = sd_journal_add_disjunction(journal); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
 static bool add_filters(sd_journal *journal)
 {
         char *data = NULL;
@@ -194,24 +225,42 @@ static bool add_filters(sd_journal *journal)
                 return false;
         }
 
-        r = sd_journal_add_match(journal, data, 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
-        }
+        /* The semantics of how journal entry matching works is described in
+         * detail in sd_journal_add_match(3).
+         *
+         * When the privacy filter override is enabled, the matches declared
+         * here correspond to this logical expression:
+         *
+         *   BOOTID && ((P0 || P1 || P2 || P3) || EXITED)
+         *
+         * Otherwise, the expression is:
+         *
+         *   BOOTID && EXITED
+         *
+         * BOOTID is short for _BOOT_ID=VAL, where VAL is the boot ID for the
+         * current boot. P0, P1, etc stand for PRIORITY=0, etc. And EXITED is
+         * short for EXIT_CODE=exited.
+         */
+
+        JOURNAL_MATCH(data);
         free(data);
+        JOURNAL_AND;
 
-        r = sd_journal_add_match(journal, "SYSLOG_IDENTIFIER=crashprobe", 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
+        // Filter messages with the four highest log levels, all indicating
+        // errors, but only when the privacy filters override is in effect;
+        // because the log messages contain arbitrary strings, and this probe
+        // does not yet keep a whitelist of allowed patterns or a blacklist of
+        // banned patterns.
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                JOURNAL_MATCH("PRIORITY=0");
+                JOURNAL_MATCH("PRIORITY=1");
+                JOURNAL_MATCH("PRIORITY=2");
+                JOURNAL_MATCH("PRIORITY=3");
+                JOURNAL_OR;
         }
 
-        r = sd_journal_add_match(journal, "PRIORITY=3", 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
-        }
+        // Only set for service-level error conditions
+        JOURNAL_MATCH("EXIT_CODE=exited");
 
         return true;
 }
@@ -258,50 +307,56 @@ static bool process_journal(void)
                                 } else if (r < 0) {
                                         return false;
                                 }
-
-                                if (!send_data(error_class)) {
-                                        return false;
-                                }
                         }
                 }
         }
 }
 
-static gchar *config_file = NULL;
-static gboolean version_p = FALSE;
+static char *config_file = NULL;
 
-static GOptionEntry options[] = {
-        { "config-file", 'f', 0, G_OPTION_ARG_FILENAME, &config_file,
-          "Path to configuration file (not implemented yet)", NULL },
-        { "version", 'V', 0, G_OPTION_ARG_NONE, &version_p,
-          "Print the program version", NULL },
-        { NULL }
+static const struct option prog_opts[] = {
+        { "help", no_argument, 0, 'h' },
+        { "config-file", required_argument, 0, 'f' },
+        { "version", no_argument, 0, 'V' },
+        { 0, 0, 0, 0 }
 };
 
-static void free_glib_strings(void)
+static void print_help(void)
 {
-        if (config_file) {
-                g_free(config_file);
-        }
+        printf("Usage:\n");
+        printf("  journalprobe [OPTIONS] - collect data from systemd journal\n");
+        printf("\n");
+        printf("Help Options:\n");
+        printf("  -h, --help            Show help options\n");
+        printf("\n");
+        printf("Application Options:\n");
+        printf("  -f, --config-file     Path to configuration file (not implemented yet)\n");
+        printf("  -V, --version         Print the program version\n");
+        printf("\n");
+}
+
+static void free_strings(void)
+{
+        free(config_file);
 }
 
 int main(int argc, char **argv)
 {
         int ret = EXIT_FAILURE;
-        GError *error = NULL;
-        GOptionContext *context;
+        int opt;
 
-        context = g_option_context_new("- collect data from systemd journal");
-        g_option_context_add_main_entries(context, options, NULL);
-        g_option_context_set_translate_func(context, NULL, NULL, NULL);
-        if (!g_option_context_parse(context, &argc, &argv, &error)) {
-                g_print("Failed to parse options: %s\n", error->message);
-                goto fail;
-        }
-
-        if (version_p) {
-                g_print(PACKAGE_VERSION "\n");
-                goto success;
+        while ((opt = getopt_long(argc, argv, "hf:V", prog_opts, NULL)) != -1) {
+                switch (opt) {
+                        case 'h':
+                                print_help();
+                                goto success;
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                goto success;
+                        case 'f':
+                                config_file = strdup(optarg);
+                                break;
+                }
         }
 
         if (!process_journal()) {
@@ -311,18 +366,14 @@ int main(int argc, char **argv)
 success:
         ret = EXIT_SUCCESS;
 fail:
-        free_glib_strings();
+        free_strings();
 
         if (journal) {
                 sd_journal_close(journal);
         }
 
-        if (context) {
-                g_option_context_free(context);
-        }
-
         if (payload) {
-                g_string_free(payload, TRUE);
+                nc_string_free(payload);
         }
 
         return ret;

src/probes/klog_main.c

@@ -7,6 +7,7 @@
 #include <errno.h>
 #include <unistd.h>
 #include <malloc.h>
+#include <signal.h>
 
 #include "log.h"
 #include "oops_parser.h"
@@ -22,7 +23,6 @@ int main(void)
         char *bufp = NULL;
         size_t buflen = 0;
         int bytes = 0;
-        int loopcount = 0;
 
         oops_parser_init(write_oops_to_file);
 
@@ -44,20 +44,21 @@ int main(void)
                 // Gets the size of the kernel ring buffer
                 log_size = klogctl(SYSLOG_ACTION_SIZE_BUFFER, NULL, 0);
                 if (log_size < 0) {
-                        telem_log(LOG_ERR, "Cannot read size of kernel ring buffer\n");
+                        telem_log(LOG_ERR, "Cannot read size of kernel ring buffer: %s\n", strerror(errno));
                         return 1;
                 }
 
                 buflen = (size_t)log_size;
 
-                if (loopcount++ > 0 && buflen > MAX_BUF)
+                if (buflen > MAX_BUF) {
                         buflen = MAX_BUF;
+                }
 
                 // Gets the contents of the kernel ring buffer
                 bufp = (char *)calloc(buflen, sizeof(char));
 
                 malloc_trim(0);
-                bytes = klogctl(SYSLOG_ACTION_READ, bufp, log_size);
+                bytes = klogctl(SYSLOG_ACTION_READ, bufp, (int)buflen);
                 if (bytes < 0) {
                         telem_log(LOG_ERR, "Cannot read contents of kernel ring buffer: %s\n", strerror(errno));
                         return 1;

src/probes/klog_scanner.c

@@ -22,7 +22,6 @@
 #include <fcntl.h>
 #include <errno.h>
 #include <unistd.h>
-#include <glib.h>
 #include <sys/stat.h>
 
 #include "log.h"

src/probes/local.mk

@@ -5,7 +5,8 @@ bin_PROGRAMS += \
 	%D%/klogscanner \
 	%D%/pstoreprobe \
 	%D%/oopsprobe \
-	%D%/pstoreclean
+	%D%/pstoreclean \
+	%D%/bertprobe
 
 %C%_hprobe_SOURCES = %D%/hello.c
 %C%_hprobe_LDADD = $(top_builddir)/src/libtelemetry.la
@@ -13,12 +14,17 @@ bin_PROGRAMS += \
 	$(AM_LDFLAGS) \
 	-pie
 
+%C%_bertprobe_SOURCES = %D%/bert_probe.c \
+	src/nica/b64enc.c
+%C%_bertprobe_LDADD = $(top_builddir)/src/libtelemetry.la
+%C%_bertprobe_LDFLAGS = \
+        $(AM_LDFLAGS) \
+        -pie
+
 %C%_telem_record_gen_SOURCES = %D%/telem_record_gen.c
 %C%_telem_record_gen_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
-%C%_telem_record_gen_LDADD = $(top_builddir)/src/libtelemetry.la \
-	$(GLIB_LIBS)
+	$(AM_CFLAGS)
+%C%_telem_record_gen_LDADD = $(top_builddir)/src/libtelemetry.la
 %C%_telem_record_gen_LDFLAGS = \
 	$(AM_LDFLAGS) \
 	-pie
@@ -44,15 +50,15 @@ endif
 
 
 %C%_crashprobe_SOURCES = \
-	%D%/crash_probe.c
+	%D%/crash_probe.c \
+	src/nica/nc-string.c \
+	%D%/probe.h
 %C%_crashprobe_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 %C%_crashprobe_LDADD = \
 	$(top_builddir)/src/libtelemetry.la \
 	$(top_builddir)/src/libtelem-shared.la \
-	@ELFUTILS_LIBS@ \
-	$(GLIB_LIBS)
+	@ELFUTILS_LIBS@
 %C%_crashprobe_LDFLAGS = \
        $(AM_LDFLAGS) \
        -pie
@@ -68,14 +74,13 @@ endif
 
 %C%_pstoreprobe_SOURCES = \
 	%D%/pstore_probe.c \
+	src/nica/nc-string.c \
 	%D%/oops_parser.c
 %C%_pstoreprobe_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 %C%_pstoreprobe_LDADD = \
 	$(top_builddir)/src/libtelemetry.la \
-	$(top_builddir)/src/libtelem-shared.la \
-	$(GLIB_LIBS)
+	$(top_builddir)/src/libtelem-shared.la
 
 if LOG_SYSTEMD
 if HAVE_SYSTEMD_JOURNAL
@@ -91,13 +96,12 @@ endif
         %D%/klog_scanner.c \
 	%D%/klog_scanner.h \
 	%D%/oops_parser.h \
+	src/nica/nc-string.c \
 	%D%/oops_parser.c
 %C%_klogscanner_CFLAGS = \
-        $(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+        $(AM_CFLAGS)
 %C%_klogscanner_LDADD = \
-        $(top_builddir)/src/libtelemetry.la \
-	$(GLIB_LIBS)
+        $(top_builddir)/src/libtelemetry.la
 %C%_klogscanner_LDFLAGS = \
         $(AM_LDFLAGS) \
         -pie
@@ -120,14 +124,14 @@ endif
 
 %C%_oopsprobe_SOURCES = \
         %D%/oops_probe.c \
-	%D%/oops_parser.c
+	%D%/oops_parser.c \
+	src/nica/nc-string.c \
+	%D%/probe.h
 %C%_oopsprobe_CFLAGS = \
-        $(AM_CFLAGS) \
-        $(GLIB_CFLAGS)
+        $(AM_CFLAGS)
 %C%_oopsprobe_LDADD = \
         $(top_builddir)/src/libtelemetry.la \
-        $(top_builddir)/src/libtelem-shared.la \
-        $(GLIB_LIBS)
+        $(top_builddir)/src/libtelem-shared.la
 
 if LOG_SYSTEMD
 if HAVE_SYSTEMD_JOURNAL
@@ -144,16 +148,15 @@ bin_PROGRAMS += \
 	%D%/journalprobe
 
 %C%_journalprobe_SOURCES = \
+	src/nica/nc-string.c \
 	%D%/journal.c
 %C%_journalprobe_CFLAGS = \
 	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS) \
 	$(SYSTEMD_ID128_CFLAGS) \
 	$(SYSTEMD_JOURNAL_CFLAGS)
 %C%_journalprobe_LDADD = \
 	$(top_builddir)/src/libtelemetry.la \
 	$(top_builddir)/src/libtelem-shared.la \
-	$(GLIB_LIBS) \
 	$(SYSTEMD_ID128_LIBS) \
 	$(SYSTEMD_JOURNAL_LIBS)
 %C%_journalprobe_LDFLAGS = \

src/probes/oops_parser.c

@@ -21,9 +21,11 @@
 #include <string.h>
 #include <ctype.h>
 #include <inttypes.h>
+#include <unistd.h>
 
 #include "oops_parser.h"
 #include "log.h"
+#include "probe.h"
 
 #define NUM_TAINTED_FLAGS 16
 
@@ -182,7 +184,7 @@ char *skip_timestamp(char *line)
         return start;
 }
 
-char *skip_spaces(char *line)
+char *skip_space(char *line)
 {
         char *start = line;
 
@@ -192,6 +194,16 @@ char *skip_spaces(char *line)
         return start;
 }
 
+char *skip_spaces(char *line)
+{
+        char *start = line;
+
+        while (*start && isspace(*start)) {
+                start++;
+        }
+        return start;
+}
+
 bool starts_with(const char *line, const char *line_end, const char *substr)
 {
         size_t substrlen = strlen(substr);
@@ -344,7 +356,7 @@ void parse_single_line(char *line, size_t size)
         line_end = line + size;
         start = skip_log_level(line);
         start = skip_timestamp(start);
-        start = skip_spaces(start);
+        start = skip_space(start);
 
         struct oops_pattern *pattern;
         if (oops_msg.length == 0) {
@@ -376,13 +388,16 @@ void parse_single_line(char *line, size_t size)
                 } else if (strstr(start, "[ end trace")) {
                         end_found = true;
                 } else if (!in_stack_dump) {
-                        if (starts_with(start, line_end, " [<") ||
-                            starts_with(start, line_end, " <") ||      //case : <EOI> [<fff73..]
-                            starts_with(start, line_end, "Call Trace:")) {
+                        // This line indicates the beginning of a stack trace;
+                        // the next line is most likely the topmost frame.
+                        if (starts_with(start, line_end, "Call Trace:")) {
                                 in_stack_dump = true;
                         }
                 } else if (in_stack_dump) {
-                        if (!(starts_with(start, line_end, " [") || starts_with(start, line_end, " <")) ||
+                        // In Linux 4.10+, the oops format changed, so we look
+                        // for a single leading space character to indicate a
+                        // stack frame line.
+                        if (!(starts_with(start, line_end, " ")) ||
                             (strlen(start) < 8) ||
                             (strstr(start, "Code:") != NULL) ||
                             (strstr(start, "Instruction Dump::") != NULL)) {
@@ -425,9 +440,20 @@ struct stack_frame {
 void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, char *start)
 {
         /*
-         * Format is as :
+         * Format is:
+         *
          *  [<ffffffffa1002128>] do_one_initcall+0xb8/0x1e0
          *  [<ffffffffa118784a>] ? __vunmap+0x9a/0x100
+         *
+         * OR
+         *
+         *  <IRQ>  [<ffffffff816606a8>] dump_stack+0x19/0x1b
+         *
+         * OR (in 4.10+)
+         *
+         *  do_one_initcall+0xb8/0x1e0
+         *  ? __vunmap+0x9a/0x100
+         *
          */
 
         struct stack_frame *frame = NULL;
@@ -436,18 +462,22 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
         if (!start) {
                 return;
         }
-        start = start + 2;
+
+        // Skip leading spaces
+        start = skip_spaces(start);
 
         //case: <IRQ>  [<ffffffff816606a8>] dump_stack+0x19/0x1b
-        if (!strncmp(start, "IRQ>", 4) || !strncmp(start, "NMI>", 4) ||
-            !strncmp(start, "EOI>", 4) || !strncmp(start, "<EOE>>", 6)) {
-                while (*start && *start != '[') {
+        if (!strncmp(start, "<IRQ>", 5) || !strncmp(start, "<NMI>", 5) ||
+            !strncmp(start, "<EOI>", 5) || !strncmp(start, "<<EOE>>", 7)) {
+                while (*start && !isspace(*start)) {
                         start++;
                 }
-                if (*start == '\0') {
+                if (start && *start == '\0') {
                         return;
                 }
-                start++;
+
+                // Skip intervening space
+                start = skip_spaces(start);
         }
 
         frame = malloc(sizeof(struct stack_frame));
@@ -462,26 +492,29 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
                 *tail = frame;
         }
 
-        if (*start == '<') {
+        // Skip memory address parsing if the address is absent (as in Linux 4.10+).
+        if (!strncmp(start, "[", 1)) {
                 start++;
-                frame->addr = (uint64_t)strtoll(start, (char **)&start, 16);
-        } else {
-                frame->addr = 0;
-        }
 
-        /* Skip up to the space after the closing bracket */
-        while (*start && !isspace(*start)) {
-                start++;
-        }
-        if (*start) {
-                start++;
-        }
+                if (*start == '<') {
+                        start++;
+                        frame->addr = (uint64_t)strtoll(start, (char **)&start, 16);
+                } else {
+                        frame->addr = 0;
+                }
 
-        if (*start && *start == '?') {
-                start++;
-                if (*start && isspace(*start)) {
+                // Skip up to the space after the closing bracket
+                while (*start && !isspace(*start)) {
                         start++;
                 }
+                if (start && *start == '\0') {
+                        return;
+                }
+
+                // Skip intervening space
+                start = skip_spaces(start);
+        } else {
+                frame->addr = 0;
         }
 
         offset_ptr = strchr(start, '+');
@@ -706,18 +739,19 @@ void parse_registers(char *line)
 
 }
 
-void append_registers_to_bt(GString **backtrace)
+void append_registers_to_bt(nc_string **backtrace)
 {
         struct reg_s *reg_entry = reg_head;
 
         while (reg_entry != NULL) {
-                #ifdef DEBUG
-                g_string_append_printf(*backtrace, "Register %s: %" PRIx64 "\n", reg_entry->reg_name,
-                                       reg_entry->reg_value);
-                #else
-                g_string_append_printf(*backtrace, "Register %s: %s\n", reg_entry->reg_name,
-                                       reg_entry->reg_value ? "Non-zero" : "Zero");
-                #endif
+                // Global override for privacy filters
+                if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                        nc_string_append_printf(*backtrace, "Register %s: %" PRIx64 "\n", reg_entry->reg_name,
+                                                reg_entry->reg_value);
+                } else {
+                        nc_string_append_printf(*backtrace, "Register %s: %s\n", reg_entry->reg_name,
+                                                reg_entry->reg_value ? "Non-zero" : "Zero");
+                }
                 reg_entry = reg_entry->next;
         }
 
@@ -728,25 +762,36 @@ void append_registers_to_bt(GString **backtrace)
         }
 }
 
-GString *parse_backtrace(struct oops_log_msg *msg)
+nc_string *parse_backtrace(struct oops_log_msg *msg)
 {
         struct stack_frame *head = NULL, *tail = NULL, *elem = NULL;
         //int in_stack_dump = 0;
         char *line = NULL;
-        GString *backtrace = NULL;
+        nc_string *backtrace = NULL;
         int frame_counter = 1;
         char *modules = NULL, *kernel_version = NULL, *tainted = NULL;
+        // Since lines are processed from last to first, the stack trace lines
+        // will appear first.
+        bool in_trace = true;
 
         for (int i = msg->length - 1; i > 0; i--) {
                 /* Check if this line is part of a stack trace */
                 line = msg->lines[i];
 
-                if (starts_with(line, line + strlen(line), " [") ||
-                    starts_with(line, line + strlen(line), " <")) {
+                if (in_trace && starts_with(line, line + strlen(line), " ")) {
                         stack_frame_append(&head, &tail, line);
                         continue;
                 }
 
+                // We've reached the end of the stack trace
+                if (starts_with(line, line + strlen(line), "Call Trace:")) {
+                        in_trace = false;
+                }
+
+                // Stack trace lines must be consecutive, so if this point is
+                // reached, we're outside the trace boundary.
+                in_trace = false;
+
                 if (str_starts_with_casei(line, "Modules linked in: ")) {
                         modules = line + strlen("Modules linked in: ");
                         continue;
@@ -760,23 +805,23 @@ GString *parse_backtrace(struct oops_log_msg *msg)
                 parse_registers(line);
         }
 
-        backtrace = g_string_new(NULL);
+        backtrace = nc_string_dup("");
         if (kernel_version) {
-                g_string_append_printf(backtrace, "Kernel Version : %s\n", kernel_version);
+                nc_string_append_printf(backtrace, "Kernel Version : %s\n", kernel_version);
                 free(kernel_version);
         }
 
         if (tainted) {
-                g_string_append_printf(backtrace, "Tainted : %s\n", tainted);
+                nc_string_append_printf(backtrace, "Tainted : %s\n", tainted);
                 free(tainted);
         }
 
         if (modules) {
-                g_string_append_printf(backtrace, "Modules : %s\n", modules);
+                nc_string_append_printf(backtrace, "Modules : %s\n", modules);
         }
 
         if (head) {
-                g_string_append_printf(backtrace, "Backtrace :\n");
+                nc_string_append_printf(backtrace, "Backtrace :\n");
         }
 
         if (reg_head) {
@@ -784,24 +829,24 @@ GString *parse_backtrace(struct oops_log_msg *msg)
         }
 
         for (elem = head; elem != NULL; elem = elem->next, frame_counter++) {
-                g_string_append_printf(backtrace, "#%d %s - [%s]\n", frame_counter,
-                                       elem->function ? elem->function : "???",
-                                       elem->module);
+                nc_string_append_printf(backtrace, "#%d %s - [%s]\n", frame_counter,
+                                        elem->function ? elem->function : "???",
+                                        elem->module);
         }
 
         stack_frame_free(&head);
         return backtrace;
 }
 
-GString *parse_payload(struct oops_log_msg *msg)
+nc_string *parse_payload(struct oops_log_msg *msg)
 {
-        GString *payload, *backtrace;
+        nc_string *payload, *backtrace;
 
-        payload = g_string_new("Crash Report:\n");
-        g_string_append_printf(payload, "Reason: %s\n", msg->lines[0]);
+        payload = nc_string_dup("Crash Report:\n");
+        nc_string_append_printf(payload, "Reason: %s\n", msg->lines[0]);
         backtrace = parse_backtrace(msg);
-        g_string_append (payload, backtrace->str);
-        g_string_free(backtrace, true);
+        nc_string_cat(payload, backtrace->str);
+        nc_string_free(backtrace);
         return payload;
 
 }
@@ -814,4 +859,3 @@ GString *parse_payload(struct oops_log_msg *msg)
  */
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */
-

src/probes/oops_parser.h

@@ -17,7 +17,7 @@
 
 #define MAX_LINES 100
 
-#include <glib.h>
+#include "nica/nc-string.h"
 #include <stdbool.h>
 #include <regex.h>
 
@@ -63,7 +63,7 @@ void oops_parser_init(oops_handler_t handler);
 void parse_single_line(char *line, size_t size);
 
 /* Parses a payload from an oops msg*/
-GString *parse_payload(struct oops_log_msg *msg);
+nc_string *parse_payload(struct oops_log_msg *msg);
 
 /* Given an entire oops log, verifies that the log is an oops log and
  * extracts the lines into an oops struct.
@@ -74,4 +74,3 @@ bool handle_entire_oops(char *buf, long size, struct oops_log_msg *msg);
 void oops_msg_cleanup(struct oops_log_msg *msg);
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */
-

src/probes/oops_probe.c

@@ -17,11 +17,9 @@
 #define _GNU_SOURCE
 #include <stdio.h>
 #include <sys/types.h>
-#include <glib.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <errno.h>
-#include <glib.h>
 #include <sys/inotify.h>
 #include <poll.h>
 
@@ -29,6 +27,8 @@
 #include "log.h"
 #include "telemetry.h"
 
+#include "nica/nc-string.h"
+
 char *oops_dir_path = KERNELOOPSDIR;
 
 static uint32_t version = 1;
@@ -68,7 +68,7 @@ void handle_oops_file(const char *fname)
         char *filename = NULL, *contents = NULL;
         long sz;
         size_t size, bytes_read;
-        GString *payload;
+        nc_string *payload;
 
         ret = asprintf(&filename, "%s/%s", oops_dir_path, fname);
         if (ret == -1) {
@@ -138,7 +138,7 @@ void handle_oops_file(const char *fname)
 
                 oops_msg_cleanup(&oops_msg);
                 send_data(payload->str, (char *)oops_msg.pattern->classification, (uint32_t)oops_msg.pattern->severity);
-                g_string_free(payload, true);
+                nc_string_free(payload);
         } else {
                 /* File does no contain an oops! */
                 telem_log(LOG_ERR, "Did not find an oops in the oops directort\n");

src/probes/probe.h

@@ -0,0 +1,25 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2017 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#pragma once
+
+/**
+ * Existence of the opt-in-no-privacy-filters file disables certain privacy
+ * filters enforced in telemetry probes.
+ */
+#define TM_PRIVACY_FILTERS_OVERRIDE "/etc/telemetrics/opt-in-no-privacy-filters"
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/pstore_probe.c

@@ -12,11 +12,10 @@
  * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
  * details.
  */
-
+#define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
 #include <dirent.h>
-#include <glib.h>
 #include <string.h>
 #include <errno.h>
 #include <assert.h>
@@ -26,6 +25,7 @@
 #include "log.h"
 #include "telemetry.h"
 #include "oops_parser.h"
+#include "nica/hashmap.h"
 
 char *pstore_dump_path = PSTOREDIR;
 
@@ -61,7 +61,7 @@ static bool send_data(char *backtrace, char *class, uint32_t severity)
 
 void handle_complete_oops_message(struct oops_log_msg *msg)
 {
-        GString *payload;
+        nc_string *payload;
 
 #ifdef DEBUG
         for (int i = 0; i < msg->length; i++) {
@@ -74,7 +74,7 @@ void handle_complete_oops_message(struct oops_log_msg *msg)
         printf("Payload Parsed :%s\n", payload->str);
 #endif
         send_data(payload->str, (char *)msg->pattern->classification, (uint32_t)msg->pattern->severity);
-        g_string_free(payload, true);
+        nc_string_free(payload);
 }
 
 void handle_crash_dump(char *dump, size_t size)
@@ -100,7 +100,7 @@ void handle_crash_dump(char *dump, size_t size)
 
 char *read_contents(char *filename, size_t *bytes)
 {
-        gchar *dump_file;
+        char *dump_file = NULL;
         FILE *fp = NULL;
         char *contents = NULL;
         long sz = 0;
@@ -108,7 +108,10 @@ char *read_contents(char *filename, size_t *bytes)
         char *hdr_end;
         size_t hdr_len;
 
-        dump_file = g_strconcat(pstore_dump_path, "/", filename, NULL);
+        if (asprintf(&dump_file, "%s/%s", pstore_dump_path, filename) < 0) {
+                goto end;
+        }
+
         fp = fopen(dump_file, "r");
         if (fp == NULL) {
                 telem_log(LOG_ERR, "Failed to open pstore dump file %s:%s\n", dump_file, strerror(errno));
@@ -170,7 +173,7 @@ end:
                 fclose(fp);
         }
 
-        g_free(dump_file);
+        free(dump_file);
         return contents;
 }
 
@@ -199,8 +202,8 @@ int main(int argc, char **argv)
         DIR *pstore_dir;
         struct dirent *entry;
         struct chunk_list  *elem, *head;
-        GHashTableIter iter;
-        gpointer key, value;
+        NcHashmapIter iter;
+        void *key, *value;
         int max_part;
         size_t totalsize = 0;
         char *crash_dump = NULL;
@@ -209,7 +212,7 @@ int main(int argc, char **argv)
          * Hash table used to store chunks belonging to a oops counter
          *  key -> count, value->head of linked list of chunks
          */
-        GHashTable *hash = g_hash_table_new(g_int_hash, g_int_equal);
+        NcHashmap *hash = nc_hashmap_new(nc_simple_hash, nc_simple_compare);
 
         pstore_dir = opendir(pstore_dump_path);
         if (pstore_dir == NULL) {
@@ -247,23 +250,20 @@ int main(int argc, char **argv)
                 elem->contents = read_contents(entry->d_name, &(elem->size));
                 elem->next = NULL;
 
-                gpointer head = g_hash_table_lookup(hash, (void *)(&elem->count));
+                void *head = nc_hashmap_get(hash, NC_HASH_KEY(elem->count));
                 if (head != NULL) {
                         elem->next = (struct chunk_list *)head;
                 }
 
-                gint *key = g_new(gint, 1);
-                *key = elem->count;
-                g_hash_table_insert(hash, key, elem);
-
+                nc_hashmap_put(hash, NC_HASH_KEY(elem->count), elem);
         }
 
         closedir(pstore_dir);
 
-        g_hash_table_iter_init (&iter, hash);
-        while (g_hash_table_iter_next (&iter, &key, &value)) {
+        nc_hashmap_iter_init(hash, &iter);
+        while (nc_hashmap_iter_next(&iter, (void **)&key, (void **)&value)) {
                 #ifdef DEBUG
-                printf("Count in the hash: %d\n", *((int *)key));
+                printf("Count in the hash: %d\n", NC_UNHASH_KEY(key));
                 #endif
 
                 head = (struct chunk_list *)value;

src/probes/telem_record_gen.c

@@ -15,6 +15,8 @@
  */
 
 #define _GNU_SOURCE
+#include <ctype.h>
+#include <getopt.h>
 #include <poll.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -22,40 +24,55 @@
 #include <string.h>
 #include <unistd.h>
 
-#include <glib.h>
 #include <string.h>
+#include <stdint.h>
 
 #include "config.h"
 #include "log.h"
 #include "common.h"
 #include "telemetry.h"
 
-static gchar *config_file = NULL;
-static gboolean version_p = FALSE;
-static guint severity = 1;
-static gchar *opt_class = NULL;
-static gchar *opt_payload = NULL;
-static guint payload_version = 1;
-static gchar *opt_payload_file = NULL;
+static char *config_file = NULL;
+static uint32_t severity = 1;
+static char *opt_class = NULL;
+static char *opt_payload = NULL;
+static uint32_t payload_version = 1;
+static char *opt_payload_file = NULL;
+static char *opt_event_id = NULL;
 
-static GOptionEntry options[] = {
-        { "config-file", 'f', 0, G_OPTION_ARG_FILENAME, &config_file,
-          "Path to configuration file (not implemented yet)", NULL },
-        { "version", 'V', 0, G_OPTION_ARG_NONE, &version_p,
-          "Print the program version", NULL },
-        { "severity", 's', 0, G_OPTION_ARG_INT, &severity,
-          "Severity level (1-4) - (default 1)", NULL },
-        { "class", 'c', 0, G_OPTION_ARG_STRING, &opt_class,
-          "Classification level_1/level_2/level_3", NULL },
-        { "payload", 'p', 0, G_OPTION_ARG_STRING, &opt_payload,
-          "Record body (max size = 8k)", NULL },
-        { "payload-file", 'P', 0, G_OPTION_ARG_STRING, &opt_payload_file,
-          "File to read payload from", NULL },
-        { "record-version", 'R', 0, G_OPTION_ARG_INT, &payload_version,
-          "Version number for format of payload (default 1)", NULL },
-        { NULL }
+static const struct option prog_opts[] = {
+        { "help", no_argument, 0, 'h' },
+        { "config-file", required_argument, 0, 'f' },
+        { "version", no_argument, 0, 'V' },
+        { "severity", required_argument, 0, 's' },
+        { "class", required_argument, 0, 'c' },
+        { "payload", required_argument, 0, 'p' },
+        { "payload-file", required_argument, 0, 'P' },
+        { "record-version", required_argument, 0, 'R' },
+        { "event-id", required_argument, 0, 'e' },
+        { 0, 0, 0, 0 }
 };
 
+static void print_help(void)
+{
+        printf("Usage:\n");
+        printf("  telem-record-gen [OPTIONS] - create and send a custom telemetry record\n");
+        printf("\n");
+        printf("Help Options:\n");
+        printf("  -h, --help            Show help options\n");
+        printf("\n");
+        printf("Application Options:\n");
+        printf("  -f, --config-file     Path to configuration file (not implemented yet)\n");
+        printf("  -V, --version         Print the program version\n");
+        printf("  -s, --severity        Severity level (1-4) - (default 1)\n");
+        printf("  -c, --class           Classification level_1/level_2/level_3\n");
+        printf("  -p, --payload         Record body (max size = 8k)\n");
+        printf("  -P, --payload-file    File to read payload from\n");
+        printf("  -R, --record-version  Version number for format of payload (default 1)\n");
+        printf("  -e, --event-id        Event id to use in the record\n");
+        printf("\n");
+}
+
 const unsigned int count_chars(const char *check, const char character)
 {
         unsigned int count = 0U;
@@ -70,51 +87,81 @@ const unsigned int count_chars(const char *check, const char character)
         return count;
 }
 
-static void free_glib_strings(void)
-{
-        if (config_file) {
-                g_free(config_file);
-        }
-
-        if (opt_class) {
-                g_free(opt_class);
-        }
-
-        if (opt_payload) {
-                g_free(opt_payload);
-        }
-}
-
 int parse_options(int argc, char **argv)
 {
         int ret = 0;
-        GError *error = NULL;
-        GOptionContext *context;
+        char *endptr = NULL;
+        long unsigned int tmp = 0;
 
-        context = g_option_context_new(
-                "- create and send a custom telemetry record\n");
-        g_option_context_add_main_entries(context, options, NULL);
-        g_option_context_set_translate_func(context, NULL, NULL, NULL);
+        int opt;
+        while ((opt = getopt_long(argc, argv, "hc:Vs:c:p:P:R:e:", prog_opts, NULL)) != -1) {
+                switch (opt) {
+                        case 'h':
+                                print_help();
+                                exit(EXIT_SUCCESS);
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case 's':
+                                errno = 0;
 
-        if (!g_option_context_parse(context, &argc, &argv, &error)) {
-                g_print("Failed to parse options: %s\n", error->message);
-                goto fail;
+                                tmp = strtoul(optarg, &endptr, 10);
+
+                                if (errno != 0) {
+                                        telem_perror("Failed to convert severity number");
+                                        goto fail;
+                                }
+                                if (endptr && *endptr != '\0') {
+                                        telem_log(LOG_ERR, "Invalid severity number. Must be an integer\n");
+                                        goto fail;
+                                }
+
+                                severity = (uint32_t)tmp;
+                                break;
+                        case 'c':
+                                opt_class = strdup(optarg);
+                                break;
+                        case 'p':
+                                opt_payload = strdup(optarg);
+                                break;
+                        case 'P':
+                                opt_payload_file = strdup(optarg);
+                                break;
+                        case 'R':
+                                errno = 0;
+
+                                tmp = strtoul(optarg, &endptr, 10);
+
+                                if (errno != 0) {
+                                        telem_perror("Failed to convert record-version number");
+                                        goto fail;
+                                }
+                                if (endptr && *endptr != '\0') {
+                                        telem_log(LOG_ERR, "Invalid record-version number. Must be an integer\n");
+                                        goto fail;
+                                }
+
+                                payload_version = (uint32_t)tmp;
+                                break;
+                        case 'e':
+                                opt_event_id = strdup(optarg);
+                                if (opt_event_id == NULL) {
+                                        goto fail;
+                                }
+                                break;
+                }
         }
 
         ret = 1;
-
 fail:
-        if (context) {
-                g_option_context_free(context);
-        }
-
         return ret;
 }
 
 int validate_opts(void)
 {
-        size_t i;
+        size_t len;
         int ret = 0;
+        const char alphab[] = EVENT_ID_ALPHAB;
 
         /* classification */
         if (opt_class == NULL) {
@@ -122,18 +169,20 @@ int validate_opts(void)
                 return ret;
 
         }
-        i = strlen(opt_class);
+        len = strlen(opt_class);
 
-        if ((i == 0) || (i > 120)) {
+        if ((len == 0) || (len > 120)) {
                 fprintf(stderr, "Error: Valid size for classification "
                         "is 1-120 chars\n");
                 return ret;
         }
 
-        if (!g_str_is_ascii(opt_class)) {
-                fprintf(stderr, "Error: Non-ascii characters detected "
-                        "in classification - aborting\n");
-                return ret;
+        for (int c = 0; c < len; c++) {
+                if (isascii(opt_class[c]) == 0) {
+                        fprintf(stderr, "Error: Non-ascii characters detected "
+                                "in classification - aborting\n");
+                        return ret;
+                }
         }
 
         if (count_chars(opt_class, '/') != 2) {
@@ -148,6 +197,20 @@ int validate_opts(void)
                 return ret;
         }
 
+        /* Event id  */
+        if (opt_event_id) {
+                size_t result = 0;
+                if ((result = strlen(opt_event_id)) != EVENT_ID_LEN) {
+                        fprintf(stderr, "Error: event_id length %zu it should have %d characters\n", result, EVENT_ID_LEN);
+                        return ret;
+                }
+                if ((result = strspn(opt_event_id, alphab)) != EVENT_ID_LEN) {
+                        fprintf(stderr, "Error: event_id contains invalid character '%c' at position %zu, valid characters are %s\n",
+                                opt_event_id[result], result, alphab);
+                        return ret;
+                }
+        }
+
         return 1;
 }
 
@@ -155,7 +218,7 @@ int allocate_payload_buffer(char **payload)
 {
         int ret = 0;
 
-        *payload = (char *)malloc(sizeof(gchar) * MAX_PAYLOAD_SIZE);
+        *payload = (char *)malloc(MAX_PAYLOAD_SIZE);
 
         if (*payload == NULL) {
                 goto out1;
@@ -182,7 +245,7 @@ int get_payload_from_file(char **payload)
                 goto out;
         }
 
-        bytes_in = fread(*payload, sizeof(gchar),
+        bytes_in = fread(*payload, 1,
                          MAX_PAYLOAD_SIZE - 1, fp);
 
         /* if fread fails */
@@ -215,7 +278,7 @@ void get_payload_from_stdin(char **payload)
         size_t bytes_in = 0;
         int c;
 
-        bytes_in = fread(*payload, sizeof(gchar), MAX_PAYLOAD_SIZE - 1, stdin);
+        bytes_in = fread(*payload, 1, MAX_PAYLOAD_SIZE - 1, stdin);
 
         if (bytes_in == MAX_PAYLOAD_SIZE - 1) {
                 /* Throw away the rest of stdin */
@@ -269,6 +332,10 @@ int send_record(char *payload)
                 goto out1;
         }
 
+        if (opt_event_id && tm_set_event_id(t_ref, opt_event_id) < 0) {
+                goto out2;
+        }
+
         if (tm_set_payload(t_ref, payload) < 0) {
                 goto out2;
         }
@@ -292,11 +359,6 @@ int main(int argc, char **argv)
                 goto fail;
         }
 
-        if (version_p) {
-                g_print(PACKAGE_VERSION "\n");
-                goto success;
-        }
-
         if (!validate_opts()) {
                 goto fail;
         }
@@ -309,13 +371,14 @@ int main(int argc, char **argv)
                 goto fail;
         }
 
-success:
         ret = EXIT_SUCCESS;
 fail:
-        free_glib_strings();
+        free(config_file);
+        free(opt_class);
+        free(opt_payload);
+        free(opt_event_id);
 
         return ret;
 }
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */
-

src/retention.c

@@ -0,0 +1,46 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+
+#include "log.h"
+#include "common.h"
+
+int delete_record_by_id(char *record_id)
+{
+        int ret = 0;
+        char *record_path = NULL;
+
+        ret = asprintf(&record_path, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (ret == -1) {
+                return 1;
+        }
+        ret = unlink(record_path);
+        if (ret == -1) {
+                telem_perror("Error deleting saved record");
+        }
+        free(record_path);
+
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/retention.h

@@ -0,0 +1,27 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+/**
+ * Delete record identified by record unique id
+ *
+ * @param record_id Unique identifier of entry
+ *
+ */
+int delete_record_by_id(char *record_id);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/telemdaemon.c

@@ -27,12 +27,14 @@
 #include <fcntl.h>
 #include <time.h>
 #include <malloc.h>
+#include <sys/uio.h>
 
 #include "telemdaemon.h"
 #include "common.h"
 #include "util.h"
 #include "log.h"
 #include "configuration.h"
+#include "retention.h"
 
 void initialize_daemon(TelemDaemon *daemon)
 {
@@ -43,7 +45,9 @@ void initialize_daemon(TelemDaemon *daemon)
         daemon->client_head = head;
         daemon->bypass_http_post_ts = 0;
         daemon->is_spool_valid = false;
+        daemon->record_journal = open_journal(JOURNAL_PATH);
         initialize_rate_limit(daemon);
+        initialize_record_delivery(daemon);
         daemon->current_spool_size = 0;
 }
 
@@ -61,6 +65,12 @@ void initialize_rate_limit(TelemDaemon *daemon)
         daemon->rate_limit_strategy = rate_limit_strategy_config();
 }
 
+void initialize_record_delivery(TelemDaemon *daemon)
+{
+        daemon->record_retention_enabled = record_retention_enabled_config();
+        daemon->record_server_delivery_enabled = record_server_delivery_enabled_config();
+}
+
 client *add_client(client_list_head *client_head, int fd)
 {
         client *cl;
@@ -262,6 +272,21 @@ void machine_id_replace(char **machine_header, char *machine_id_override)
         free(old_header);
 }
 
+void save_entry_to_journal(TelemDaemon *daemon, time_t t_stamp, char *headers[])
+{
+        char *classification_value;
+        char *event_id_value;
+
+        if (get_header_value(headers[TM_CLASSIFICATION], &classification_value) &&
+            get_header_value(headers[TM_EVENT_ID], &event_id_value)) {
+                if (new_journal_entry(daemon->record_journal, classification_value, t_stamp, event_id_value) != 0) {
+                        telem_log(LOG_INFO, "new_journal_entry in process_record: failed saving record entry\n");
+                }   
+        }   
+        free(classification_value);
+        free(event_id_value);
+}
+
 void process_record(TelemDaemon *daemon, client *cl)
 {
         int i = 0;
@@ -279,7 +304,6 @@ void process_record(TelemDaemon *daemon, client *cl)
         bool byte_check_passed = true;
         bool record_burst_enabled = true;
         bool byte_burst_enabled = true;
-
         /* Gets the current minute of time */
         time_t temp = time(NULL);
         struct tm *tm_s = localtime(&temp);
@@ -311,6 +335,22 @@ void process_record(TelemDaemon *daemon, client *cl)
         /* TODO : check if the body is within the limits. */
         body = msg + header_size;
 
+        /* Save record in journal */
+        save_entry_to_journal(daemon, temp, headers);
+
+        /* Save local copy */
+        if (daemon->record_retention_enabled) {
+                save_local_copy(daemon, body);
+        }
+
+        /* Bail out if server delivery is not enabled */
+        if (!daemon->record_server_delivery_enabled) {
+#ifdef DEBUG
+                fprintf(stdout, "process_record: record server delivery disabled\n");
+#endif
+                goto end;
+        }
+
         if (inside_direct_spool_window(daemon, time(NULL))) {
                 telem_log(LOG_INFO, "process_record: delivering directly to spool\n");
                 spool_record(daemon, headers, body);
@@ -342,12 +382,12 @@ void process_record(TelemDaemon *daemon, client *cl)
                         daemon->rate_limit_enabled = false;
                 }
         }
+
         /* Sends record if rate limiting is disabled, or all checks passed */
         if (!daemon->rate_limit_enabled || (record_check_passed && byte_check_passed)) {
                 /* Send the record as https post */
                 record_sent = post_record_ptr(headers, body, true);
         }
-
         // Get rate-limit strategy
         do_spool = spool_strategy_selected(daemon);
 
@@ -462,6 +502,11 @@ bool post_record_http(char *headers[], char *body, bool spool)
         const char *cert_file = get_cainfo_config();
         const char *tid_header = get_tidheader_config();
 
+        // Initialize the libcurl global environment once per POST. This lets us
+        // clean up the environment after each POST so that when the daemon is
+        // sitting idle, it will be consuming as little memory as possible.
+        curl_global_init(CURL_GLOBAL_ALL);
+
         curl = curl_easy_init();
         if (!curl) {
                 telem_log(LOG_ERR, "curl_easy_init(): Unable to start libcurl"
@@ -511,11 +556,11 @@ bool post_record_http(char *headers[], char *body, bool spool)
         if (res) {
                 size_t len = strlen(errorbuf);
                 if (len) {
-                    telem_log(LOG_ERR, "Failed sending record: %s%s", errorbuf,
-                              ((errorbuf[len - 1] != '\n') ? "\n" : ""));
+                        telem_log(LOG_ERR, "Failed sending record: %s%s", errorbuf,
+                                  ((errorbuf[len - 1] != '\n') ? "\n" : ""));
                 } else {
-                    telem_log(LOG_ERR, "Failed sending record: %s\n",
-                              curl_easy_strerror(res));
+                        telem_log(LOG_ERR, "Failed sending record: %s\n",
+                                  curl_easy_strerror(res));
                 }
         } else if (http_response != 201 && http_response != 200) {
                 /*  201 means the record was  successfully created
@@ -532,9 +577,45 @@ bool post_record_http(char *headers[], char *body, bool spool)
 
         curl_slist_free_all(custom_headers);
         curl_easy_cleanup(curl);
+
+        curl_global_cleanup();
+
         return res ? false : true;
 }
 
+void save_local_copy(TelemDaemon *daemon, char *body)
+{
+        int ret = 0;
+        char *tmpbuf = NULL;
+        FILE *tmpfile = NULL;
+
+        if (daemon == NULL || daemon->record_journal == NULL || 
+            daemon->record_journal->latest_record_id == NULL) {
+                return;
+        }
+
+        ret = asprintf(&tmpbuf, "%s/%s", RECORD_RETENTION_DIR,
+                       daemon->record_journal->latest_record_id);
+        if (ret == -1) {
+                telem_log(LOG_ERR, "Failed to allocate memory for record full path, aborting\n");
+                return;
+        }
+
+        tmpfile = fopen(tmpbuf, "w");
+        if (!tmpfile) {
+                telem_perror("Error opening local record copy temp file");
+                goto save_err;
+        }
+
+        // Save body
+        fprintf(tmpfile, "%s\n", body);
+        fclose(tmpfile);
+
+save_err:
+        free(tmpbuf);
+        return;
+}
+
 void spool_record(TelemDaemon *daemon, char *headers[], char *body)
 {
         int ret = 0;
@@ -576,7 +657,6 @@ void spool_record(TelemDaemon *daemon, char *headers[], char *body)
                 goto spool_err;
         }
 
-        //fp = fopen(spool_dir_path, const char *mode);
         tmpfile = fdopen(tmpfd, "a");
         if (!tmpfile) {
                 telem_perror("Error opening temp file");
@@ -671,7 +751,7 @@ bool get_machine_id(char *machine_id)
         return true;
 }
 
-int machine_id_write(uint64_t upper, uint64_t lower)
+int machine_id_write(char *new_id)
 {
         FILE *fp;
         int ret = 0;
@@ -683,7 +763,7 @@ int machine_id_write(uint64_t upper, uint64_t lower)
                 goto file_error;
         }
 
-        ret = fprintf(fp, "%.16" PRIx64 "%.16" PRIx64, upper, lower);
+        ret = fprintf(fp, "%s", new_id);
         if (ret < 0) {
                 telem_perror("Unable to write to machine id file");
                 goto file_error;
@@ -700,49 +780,17 @@ file_error:
 
 int generate_machine_id(void)
 {
-        int frandom = -1;
-        struct stat stat_buf;
-        int ret = 0;
-        struct iovec random_id[2];
-        uint64_t random_id_upper;
-        uint64_t random_id_lower;
-        int result = -1;
+        int result = 0;
+        char *new_id = NULL;
 
-        ret = stat("/dev/urandom", &stat_buf);
-        if (ret == -1) {
-                telem_log(LOG_ERR, "Unable to stat urandom device\n");
+        if ((result = get_random_id(&new_id)) != 0) {
                 goto rand_err;
         }
 
-        if (!S_ISCHR(stat_buf.st_mode)) {
-                telem_log(LOG_ERR, "/dev/urandom is not a character device file\n");
-                goto rand_err;
-        }
-
-        /* TODO : check for major and minor numbers to be extra sure ??
-           ((stat_buffer.st_rdev == makedev(1, 8)) || (stat_buffer.st_rdev == makedev(1, 9))*/
-
-        frandom = open("/dev/urandom", O_RDONLY);
-        if (frandom < 0) {
-                telem_perror("Error opening /dev/urandom");
-                goto rand_err;
-        }
-
-        random_id[0].iov_base = &random_id_upper;
-        random_id[0].iov_len = 8;
-        random_id[1].iov_base = &random_id_lower;
-        random_id[1].iov_len = 8;
-
-        if (readv(frandom, random_id, 2) < 0) {
-                telem_perror("Error while reading /dev/urandom");
-                goto rand_err;
-        }
-
-        result = machine_id_write(random_id_upper, random_id_lower);
+        result = machine_id_write(new_id);
 rand_err:
-        if (frandom >= 0) {
-                close(frandom);
-        }
+        free(new_id);
+
         return result;
 }
 

src/telemdaemon.h

@@ -22,6 +22,7 @@
 #include <sys/queue.h>
 #include <stdbool.h>
 #include <inttypes.h>
+#include "journal/journal.h"
 
 #define TM_MACHINE_ID_EXPIRY (3 /*d*/ * 24 /*h*/ * 60 /*m*/ * 60 /*s*/)
 
@@ -56,7 +57,8 @@ typedef struct TelemDaemon {
         client_list_head client_head;
         /* Time of last failed post */
         time_t bypass_http_post_ts;
-
+        /* Telemetry Journal*/
+        TelemJournal *record_journal;
         /* Rate limit record and byte arrays */
         size_t record_burst_array[TM_RATE_LIMIT_SLOTS];
         size_t byte_burst_array[TM_RATE_LIMIT_SLOTS];
@@ -67,10 +69,12 @@ typedef struct TelemDaemon {
         int64_t byte_burst_limit;
         int byte_window_length;
         const char *rate_limit_strategy;
-
+        /* Spool configuration */
         bool is_spool_valid;
         long current_spool_size;
-
+        /* Record local copy and delivery  */
+        bool record_retention_enabled;
+        bool record_server_delivery_enabled;
         char *machine_id_override;
 } TelemDaemon;
 
@@ -88,6 +92,14 @@ void initialize_daemon(TelemDaemon *daemon);
  */
 void initialize_rate_limit(TelemDaemon *daemon);
 
+/**
+ * Initialize record delivery to remote and locally
+ * in the daemon
+ *
+ * @param daemon A pointer to the daemon structure
+ */
+void initialize_record_delivery(TelemDaemon *daemon);
+
 /**
  * Add poll fd struct to the array of pollfds.
  *
@@ -267,7 +279,15 @@ bool post_record_http(char *header_values[], char *body, bool spool);
  * @param body The payload of the record
  *
  */
-void spool_record(TelemDaemon *daemon, char *headers[], char *body);
+void spool_record(TelemDaemon * daemon, char *headers[], char *body);
+
+/**
+ * Save a copy of the record payload locally
+ *
+ * @param body The payload of the record
+ *
+ */
+void save_local_copy(TelemDaemon *daemon, char *body);
 
 /**
  * Get random machine id stored in file

src/telemetry.c

@@ -15,7 +15,7 @@
  */
 
 #define _GNU_SOURCE
-#include <glib.h>
+
 #include <string.h>
 #include <stdio.h>
 #include <sys/stat.h>
@@ -29,7 +29,9 @@
 #include <sys/un.h>
 #include <limits.h>
 #include <inttypes.h>
+#include <ctype.h>
 
+#include "util.h"
 #include "common.h"
 #include "configuration.h"
 #include "telemetry.h"
@@ -137,38 +139,14 @@ static int set_severity_header(struct telem_ref *t_ref, uint32_t severity)
  */
 static int set_classification_header(struct telem_ref *t_ref, char *classification)
 {
-        size_t i, j;
-        int slashes = 0;
-        int x = 0;
 
-        j = strlen(classification);
-
-        if (j > MAX_CLASS_LENGTH) {
-                return -EINVAL;
-        }
-
-        for (i = 0, x = 0; i <= (j - 1); i++, x++) {
-                if (classification[i] == '/') {
-                        slashes++;
-                        x = 0;
-                } else {
-                        if (x > MAX_SUBCAT_LENGTH) {
-                                return -EINVAL;
-                        }
-                }
-        }
-
-        if (slashes != 2) {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Classification string should have two /s.\n");
-#endif
+        if (validate_classification(classification) == 1) {
                 return -EINVAL;
         }
 
         return set_header(&(t_ref->record->headers[TM_CLASSIFICATION]),
                           TM_CLASSIFICATION_STR, classification,
                           &(t_ref->record->header_size));
-
 }
 
 /**
@@ -400,6 +378,62 @@ static int set_timestamp_header(struct telem_ref *t_ref)
         return status;
 }
 
+/**
+ * Sets cpu model for telemetry record. The information from cpu is extracted
+ * from /proc/cpuinfo, specifically "model name" attribute.
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_cpu_model_header(struct telem_ref *t_ref)
+{
+        FILE *fs = NULL;
+        char buf[SMALL_LINE_BUF] = { 0 };
+        char *model_name = NULL;
+        const char *attr_name = "model name";
+        int status = 0;
+        size_t attr_len = strlen(attr_name);
+        size_t model_str_len = 0;
+
+        fs = fopen("/proc/cpuinfo", "r");
+        if (fs != NULL) {
+                while (fgets(buf, SMALL_LINE_BUF, fs)) {
+                        if (strncmp(attr_name, buf, attr_len) == 0) {
+                                model_name = strchr(buf, ':');
+                                break;
+                        }
+                }
+                fclose(fs);
+                if (model_name != NULL) {
+                        model_str_len = strlen(model_name);
+                        if (model_str_len > 2) {
+                                model_name = (char *)model_name + 2 * sizeof(char);
+                                model_name[model_str_len - 2] = '\0';
+                        } else {
+                                model_name = "blank";
+                        }
+                } else {
+                        model_name = "blank";
+                        fprintf(stderr, "NOTICE: Unable to find attribute:%s\n", attr_name);
+                }
+
+                status = set_header(
+                        &(t_ref->record->headers[TM_CPU_MODEL]),
+                        TM_CPU_MODEL_STR, model_name,
+                        &(t_ref->record->header_size));
+
+        } else {
+#ifdef DEBUG
+                fprintf(stderr, "NOTICE: Unable to open /proc/cpuinfo\n");
+#endif
+                status = -1;
+        }
+
+        return status;
+}
+
 /**
  * A healper function for set_host_type_header that reads the first line out of
  * a file and chomps the newline if necessary. If the file does not exist the
@@ -497,6 +531,118 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
         return ret;
 }
 
+/**
+ * Sets the board name for telemetry record, this record is a combination of
+ * board name and board vendor. This information is read from dmi filesystem
+ * Board Name (board_name) and Board Vendor (board_vendor).
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_board_name_header(struct telem_ref *t_ref)
+{
+        int status = 0;
+        int rc;
+        char *buf = NULL;
+        char *bn = NULL;
+        char *bv = NULL;
+
+        rc  = get_dmi_value("/sys/class/dmi/id/board_name", "bn", &bn);
+        if (rc < 0) {
+                status = rc;
+                goto cleanup;
+        }
+
+        rc  = get_dmi_value("/sys/class/dmi/id/board_vendor", "bv", &bv);
+        if (rc < 0) {
+                status = rc;
+                goto cleanup;
+        }
+
+        rc = asprintf(&buf, "%s|%s", bn, bv);
+
+        if (rc < 0) {
+                status = -ENOMEM;
+                goto cleanup;
+        } else {
+                status = set_header(
+                        &(t_ref->record->headers[TM_BOARD_NAME]),
+                        TM_BOARD_NAME_STR, buf,
+                        &(t_ref->record->header_size));
+                free(buf);
+        }
+
+cleanup:
+        if (bn != NULL) {
+                free(bn);
+        }
+
+        if (bv != NULL) {
+                free(bv);
+        }
+
+        return status;
+}
+
+/**
+ * Sets BIOS version header for telemetry record, this information is
+ * read from dmi filesystem BIOS Version (bios_version).
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_bios_version_header(struct telem_ref *t_ref)
+{
+        int status = 0;
+        int rc = 0;
+        char *bios_version  = NULL;
+
+        rc  = get_dmi_value("/sys/class/dmi/id/bios_version", "bv", &bios_version);
+        if (rc < 0) {
+                status = rc;
+        } else {
+                status = set_header(
+                        &(t_ref->record->headers[TM_BIOS_VERSION]),
+                        TM_BIOS_VERSION_STR, bios_version,
+                        &(t_ref->record->header_size));
+                free(bios_version);
+        }
+
+        return status;
+}
+
+/**
+ * Sets the event_id header, this id is an identifier that multiple
+ * records can share. This means that one event can lead to multiple
+ * records.
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, -1 on failure
+ *
+ */
+static int set_event_id_header(struct telem_ref *t_ref)
+{
+        int rc = 0;
+        char *buff = NULL;
+
+        rc = get_random_id(&buff);
+
+        if (rc == 0) {
+                rc = set_header(
+                        &(t_ref->record->headers[TM_EVENT_ID]),
+                        TM_EVENT_ID_STR, buff,
+                        &(t_ref->record->header_size));
+        }
+        free(buff);
+
+        return rc;
+}
+
 /**
  * Sets the hosttype header, which is a tuple of three values looked for
  * in the dmi filesystem.  System Vendor (sys_vendor), Product Name
@@ -730,6 +876,30 @@ int allocate_header(struct telem_ref *t_ref, uint32_t severity,
                 goto free_and_fail;
         }
 
+        i++;
+
+        if ((ret = set_board_name_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_cpu_model_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_bios_version_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_event_id_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
         i++; /* Not necessary, but including for future expansion */
 
         return ret;
@@ -806,8 +976,8 @@ int payload_is_ascii(char *payload, size_t len)
         int ret = 0;
 
         for (i = 0; i < len; i++) {
-                if (!g_ascii_isprint((gchar)payload[i]) &&
-                    !g_ascii_isspace((gchar)payload[i])) {
+                if (!isprint(payload[i]) &&
+                    !isspace(payload[i])) {
                         ret = -EINVAL;
                         break;
                 }
@@ -858,6 +1028,58 @@ int tm_set_payload(struct telem_ref *t_ref, char *payload)
         return ret;
 }
 
+/**
+ * Checks for id to have length = 32 characters
+ * and hexadecimal characters only.
+ *
+ * @param id A pointer to string to be checked
+ *
+ * @return 0 on success, 1 on failure
+ *
+ */
+static int validate_event_id(char *id)
+{
+        int rc = 0;
+        const char alphab[] = EVENT_ID_ALPHAB;
+
+        // make sure is not null
+        if (!id) {
+                return 1;
+        }
+
+        // 32 - 32 = 0
+        if ((rc = (int)(strlen(id) - EVENT_ID_LEN)) != 0) {
+                return 1;
+        }
+
+        // verify alphabet
+        // strspn checks for characters in alphab
+        if (strspn(id, alphab) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        return rc;
+}
+
+int tm_set_event_id(struct telem_ref *t_ref, char *event_id)
+{
+        int rc = -1;
+
+        if (!validate_event_id(event_id)) {
+                if (t_ref && t_ref->record && t_ref->record->headers) {
+                        // free default id before overriding
+                        free(t_ref->record->headers[TM_EVENT_ID]);
+                        // set new event_id
+                        if (asprintf(&(t_ref->record->headers[TM_EVENT_ID]),
+                                     "%s: %s\n", TM_EVENT_ID_STR, event_id) > 0) {
+                                rc = 0;
+                        }
+                }
+        }
+
+        return rc;
+}
+
 /**
  * Write nbytes from buf to fd. Used to send records to telemd.
  *

src/telemetry.h

@@ -85,6 +85,19 @@ void tm_set_config_file(char *c_file);
 int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
                      char *classification, uint32_t payload_version);
 
+/**
+ * Sets the event_id to a user defined id
+ *
+ * @param t_ref A pointer to a telem_ref struct that was previously initialized
+ *        this struct will have an event_id already initialized and it will be
+ *        overriden by a user provided event_id
+ * @param event_id A pointer to a string that contains the 32 character length
+ *        event_id
+ *
+ * @return 0 on success, or a negative errno-style value on error
+ */
+int tm_set_event_id(struct telem_ref *t_ref, char *event_id);
+
 /**
  * Set the payload field of a telemetrics record
  *

src/telemetry.sym

@@ -16,3 +16,8 @@ TM_3_0_0 {
     tm_set_payload;
     tm_send_record;
 } TM_2_0_0;
+
+TM_4_0_0 {
+  global:
+    tm_set_event_id;
+} TM_3_0_0;

src/util.c

@@ -16,16 +16,22 @@
 
 #define _GNU_SOURCE
 
+#define RANDOM_ID_LEN 32
+
 #include <stdio.h>
 #include <string.h>
 #include <stdbool.h>
 #include <assert.h>
 #include <stdlib.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <dirent.h>
 #include <errno.h>
 
+#include "common.h"
 #include "util.h"
 
 bool get_header(const char *haystack, const char *needle, char **line, size_t len)
@@ -37,6 +43,28 @@ bool get_header(const char *haystack, const char *needle, char **line, size_t le
         return false;
 }
 
+bool get_header_value(const char *header, char **value)
+{
+        char *sep = NULL;
+
+        *value = NULL;
+
+        if (header == NULL) {
+                return false;
+        }
+
+        if ((sep = strchr(header, ':')) != NULL) {
+                sep++;
+                // skip space after colon if there's one
+                if (*sep == ' ') {
+                        sep++;
+                }
+                *value = strdup(sep);
+        }
+
+        return (bool)(value != NULL);
+}
+
 void *reallocate(void **addr, size_t *allocated, size_t requested)
 {
         void *newaddr;
@@ -107,4 +135,80 @@ long get_directory_size(const char *dir_path)
         return total_size;
 }
 
+/**
+ * Generates a 32 characters random id
+ *
+ * @param buffer pointer to allocate and copy data
+ *
+ */
+int get_random_id(char **buff)
+{
+        int result = -1;
+        int frandom = -1;
+        uint64_t random_id[2] = { '\0' };
+
+        frandom = open("/dev/urandom", O_RDONLY);
+        if (frandom < 0) {
+                return -1;
+        }
+
+        if (read(frandom, &random_id, sizeof(random_id)) == sizeof(random_id)) {
+                if (asprintf(buff, "%.16" PRIx64 "%.16" PRIx64, random_id[0], random_id[1]) == RANDOM_ID_LEN) {
+                        result = 0;
+                }
+        }
+
+        close(frandom);
+
+        return result;
+}
+
+/**
+ * Validate classification value. A valid classification
+ * is a string with 2 slashes, with max length of
+ * MAX_CLASS_LENGTH and strings betweeb slashes should
+ * have a max length of MAX_SUBCAT_LENGTH.
+ *
+ * @param classification A pointer to classification value
+ *        to be validated.
+ *
+ * @return 0 on sucess and 1 on failure
+ */
+int validate_classification(char *classification)
+{
+        size_t i, j;
+        int slashes = 0;
+        int x = 0;
+
+        if (classification == NULL) {
+                return 1;
+        }
+
+        j = strlen(classification);
+
+        if (j > MAX_CLASS_LENGTH) {
+                return 1;
+        }
+
+        for (i = 0, x = 0; i <= (j - 1); i++, x++) {
+                if (classification[i] == '/') {
+                        slashes++;
+                        x = 0;
+                } else {
+                        if (x > MAX_SUBCAT_LENGTH) {
+                                return 1;
+                        }
+                }
+        }
+
+        if (slashes != 2) {
+#ifdef DEBUG
+                fprintf(stderr, "ERR: Classification string should have two /s.\n");
+#endif
+                return 1;
+        }
+
+        return 0;
+}
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/util.h

@@ -24,7 +24,16 @@ void *reallocate(void **addr, size_t *allocated, size_t requested);
 /* Check if haystacks begins with needle and return a copy of haystack */
 bool get_header(const char *haystack, const char *needle, char **line, size_t len);
 
+/* Get the value of a header */
+bool get_header_value(const char *header, char **value);
+
 /* Get the size of the directory */
 long get_directory_size(const char *sdir);
 
+/* Initialize buff and copy generated id */
+int get_random_id(char **buff);
+
+/* Validates classification value */
+int validate_classification(char *classification);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

tests/check_config.c

@@ -50,11 +50,39 @@ START_TEST(check_read_valid_config)
         ck_assert_str_eq(config.strValues[CONF_CAINFO], "/tmp/cacert.crt");
         ck_assert_str_eq(config.strValues[CONF_TIDHEADER],
                          "X-Telemetry-TID: 6907c830-eed9-4ce9-81ae-76daf8d88f0f");
-	ck_assert(config.boolValues[CONF_DAEMON_RECYCLING_ENABLED] == true);
+        ck_assert(config.boolValues[CONF_DAEMON_RECYCLING_ENABLED] == true);
 
 }
 END_TEST
 
+START_TEST(check_read_valid_config_defaults)
+{
+        char *config_file = TOPSRCDIR "/src/data/example.conf";
+        configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
+
+        int ret = read_config_from_file(config_file, &config);
+        ck_assert(ret == true);
+
+        // RECORD_RETENTION_ENABLED_DEFAULT = false
+        ck_assert(config.boolValues[CONF_RECORD_RETENTION_ENABLED] == RECORD_RETENTION_ENABLED_DEFAULT);
+        // RECORD_SERVER_DELIVERY_ENABLED_DEFAULT = true
+        ck_assert(config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED] == RECORD_SERVER_DELIVERY_ENABLED_DEFAULT);
+}
+END_TEST
+
+START_TEST(check_read_valid_config_record_retention_delivery)
+{
+        char *config_file = TOPSRCDIR "/src/data/example.1.conf";
+        configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
+
+        int ret = read_config_from_file(config_file, &config);
+        ck_assert(ret == true);
+
+        ck_assert(config.boolValues[CONF_RECORD_RETENTION_ENABLED] == true);
+        ck_assert(config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED] == false);
+}
+END_TEST
+
 START_TEST(check_config_initialised)
 {
         char *config_file = ABSTOPSRCDIR "/src/data/example.conf";
@@ -76,7 +104,7 @@ START_TEST(check_config_initialised)
         ck_assert_str_eq(get_cainfo_config(), "/tmp/cacert.crt");
         ck_assert_str_eq(get_tidheader_config(),
                          "X-Telemetry-TID: 6907c830-eed9-4ce9-81ae-76daf8d88f0f");
-	ck_assert(daemon_recycling_enabled_config() == true);
+        ck_assert(daemon_recycling_enabled_config() == true);
 }
 END_TEST
 
@@ -89,6 +117,8 @@ Suite *config_suite(void)
         TCase *t = tcase_create("config");
         tcase_add_test(t, check_read_config_for_invalid_file);
         tcase_add_test(t, check_read_valid_config);
+        tcase_add_test(t, check_read_valid_config_defaults);
+        tcase_add_test(t, check_read_valid_config_record_retention_delivery);
         tcase_add_test(t, check_config_initialised);
 
         // add more TCases here

tests/check_daemon.c

@@ -295,7 +295,11 @@ START_TEST(check_process_record_with_correct_size_and_data)
                         "machine_id: 1234\ncreation_timestamp: 1418672344\narch:x86_64\n"
                         "host_type: macbookpro\nbuild: 200\nkernel_version: 3.15\n"
                         "payload_format_version: 1\n"
-                        "system_name: clear-linux-os\n";
+                        "system_name: clear-linux-os\n"
+                        "board_name: Qemu|Intel\n"
+                        "cpu_model: Intel(R) Core(TM) i7-5650U CPU @ 2.20GHz\n"
+                        "bios_version: Qemu\n"
+                        "event_id: 3a2d799826edc6266d72824d2aac6763\n";
         char *post_body = "test message";
 
         set_up_socket_pair(&client_fd, &server_fd);

tests/check_journal.c

@@ -0,0 +1,260 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#include <check.h>
+#include <errno.h>
+#include <stdio.h>
+#include "common.h"
+#include "journal/journal.h"
+
+static char *journal_file = "journal.txt";
+static char *journal_print_file = "journal.print.txt";
+static struct TelemJournal *journal = NULL;
+static char *eid = "00007766547776eb7fc478eb0eb43e43";
+static int K = 20;
+
+void teardown(void) {
+        remove(journal_file);
+}
+
+START_TEST(check_open_journal)
+{
+        struct TelemJournal *j = open_journal(journal_file);
+        ck_assert_ptr_nonnull(j);
+        ck_assert_ptr_nonnull(j->fptr);
+        ck_assert_ptr_nonnull(j->boot_id);
+        ck_assert_int_eq(j->record_count, 0);
+        close_journal(j);
+}
+END_TEST
+
+START_TEST(check_unsuccessful_open_journal)
+{
+        char *bad_journal_file = "/sys/firmware/acpi/tables/journal";
+        struct TelemJournal *j = open_journal(bad_journal_file);
+        ck_assert_ptr_null(j);
+}
+END_TEST
+
+void new_entry_setup(void)
+{
+        if (journal) {
+                return;
+        }
+
+        journal = open_journal(journal_file);
+}
+
+START_TEST(check_new_entry)
+{
+        int ret;
+        ret = new_journal_entry(journal, "t/t/t", 1520054957,
+                                "3bc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 0);
+        close_journal(journal);
+}
+END_TEST
+
+START_TEST(check_new_entry_bad_class)
+{
+        int ret = 0;
+        /* has an invalid class */
+        ret = new_journal_entry(journal, "t/t", 1520054957,
+                                "3bc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+START_TEST(check_new_entry_bad_id)
+{
+        int ret = 0;
+        /* event_id is invalid */
+        ret = new_journal_entry(journal, "t/t/t", 1520054957,
+                                "Xbc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+START_TEST(check_new_entry_null_class)
+{
+        int ret = 0;
+        /* param is NULL */
+        ret = new_journal_entry(journal, NULL, 1520054957,
+                                "3bc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+START_TEST(check_new_entry_null_id)
+{
+        int ret = 0;
+        /* null id */
+        ret = new_journal_entry(journal, "t/t/t", 1520054957, NULL);
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+void new_entry_teardown(void)
+{
+        if (journal) {
+                close_journal(journal);
+                journal = NULL;
+        }
+        remove(journal_file);
+}
+
+void insert_n_records(int n, struct TelemJournal *j)
+{
+        for (int i = 0; i < n; i++) {
+                new_journal_entry(j, "t/t/t", 1520054957 + i,
+                                  "3bc17766547776eb7fc478eb0eb43e43");
+        }
+}
+
+/*
+   failing in travis-ci
+ */
+START_TEST(check_journal_file_prune)
+{
+        int rc = 0;
+        struct TelemJournal *j = open_journal(journal_file);
+
+        insert_n_records(j->record_count_limit * 2, j);
+        ck_assert_int_gt(j->record_count, j->record_count_limit);
+        rc = prune_journal(j, "./");
+        ck_assert(rc == 0);
+        // Record count should always be below the limit + hysteresis
+        ck_assert_int_lt(j->record_count, j->record_count_limit + DEVIATION);
+        close_journal(j);
+}
+END_TEST
+
+void journal_entry_setup(void)
+{
+        int result = 0;
+
+        if (journal) {
+                close_journal(journal);
+        }
+
+        journal = open_journal(journal_print_file);
+
+        insert_n_records(K, journal);
+        /* Insert recors with specific values */
+        result = new_journal_entry(journal, "a/b/c", 1520054957, eid);
+        ck_assert(result == 0);
+        result = new_journal_entry(journal, "a/b/d", 1520054957, eid);
+        ck_assert(result == 0);
+
+}
+
+START_TEST(check_journal_print)
+{
+        int count = 0;
+        count = print_journal(journal, NULL, NULL, NULL, NULL, 0);
+        ck_assert_int_eq(count, K + 2);
+}
+END_TEST
+
+START_TEST(check_journal_filter_by_class)
+{
+        int result = print_journal(journal, "a/b/c", NULL, NULL, NULL, 0);
+        ck_assert_int_eq(result, 1);
+}
+END_TEST
+
+START_TEST(check_journal_filter_by_class_prefix)
+{
+        int result = print_journal(journal, "a/b/*", NULL, NULL, NULL, 0);
+        ck_assert_int_eq(result, 2);
+}
+END_TEST
+
+START_TEST(check_journal_filter_by_event_id)
+{
+        int result = print_journal(journal, NULL, NULL, eid, NULL, 0);
+        ck_assert(result == 2);
+}
+END_TEST
+
+void journal_entry_teardown(void)
+{
+        close_journal(journal);
+        remove(journal_print_file);
+}
+
+Suite *config_suite(void)
+{
+        // A suite is comprised of test cases, defined below
+        Suite *s = suite_create("journal_feature");
+
+        // Individual unit tests are added to "test cases"
+        TCase *t = tcase_create("journal");
+        tcase_add_unchecked_fixture(t, NULL, teardown);
+        tcase_add_test(t, check_open_journal);
+        tcase_add_test(t, check_unsuccessful_open_journal);
+        suite_add_tcase(s, t);
+
+        t = tcase_create("new entry");
+        tcase_add_unchecked_fixture(t, new_entry_setup, new_entry_teardown);
+        tcase_add_test(t, check_new_entry);
+        tcase_add_test(t, check_new_entry_bad_class);
+        tcase_add_test(t, check_new_entry_bad_id);
+        tcase_add_test(t, check_new_entry_null_class);
+        tcase_add_test(t, check_new_entry_null_id);
+        suite_add_tcase(s, t);
+
+        t = tcase_create("prunning journal");
+        tcase_add_unchecked_fixture(t, NULL, teardown);
+        tcase_add_test(t, check_journal_file_prune);
+        suite_add_tcase(s, t);
+
+        t = tcase_create("print journal");
+        tcase_add_unchecked_fixture(t, journal_entry_setup,
+                                    journal_entry_teardown);
+        tcase_add_test(t, check_journal_print);
+        tcase_add_test(t, check_journal_filter_by_class);
+        tcase_add_test(t, check_journal_filter_by_class_prefix);
+        tcase_add_test(t, check_journal_filter_by_event_id);
+        suite_add_tcase(s, t);
+
+        return s;
+}
+
+int main(void)
+{
+        Suite *s;
+        SRunner *sr;
+
+        s = config_suite();
+        sr = srunner_create(s);
+
+        // Use the TAP driver for now, so that each
+        // unit test will PASS/FAIL in the log output.
+        srunner_set_log(sr, NULL);
+        srunner_set_tap(sr, "-");
+
+        srunner_run_all(sr, CK_SILENT);
+        // failed = srunner_ntests_failed(sr);
+        srunner_free(sr);
+
+        // if you want the TAP driver to report a hard error based
+        // on certain conditions (e.g. number of failed tests, etc.),
+        // return non-zero here instead.
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

tests/check_libtelemetry.c

@@ -24,6 +24,7 @@
 #include "telemetry.h"
 
 static struct telem_ref *ref = NULL;
+static char *original_event_id = NULL;
 
 void create_setup(void)
 {
@@ -157,6 +158,87 @@ START_TEST(record_create_severity_overflow)
 }
 END_TEST
 
+void event_id_setup(void)
+{
+        int ret;
+
+        if (ref) {
+                return;
+        }
+
+        ret = tm_create_record(&ref, 1, "t/t/t", 2000);
+        original_event_id = strdup(ref->record->headers[TM_EVENT_ID]);
+        if (!original_event_id) {
+                return;
+        }
+        ck_assert_msg(ret != -ECONNREFUSED,
+                      "Opt-out enabled. Opt in to run this test");
+}
+
+START_TEST(record_set_event_id)
+{
+        int ret;
+        char *event_id = "aaaaaa00000033333344444466666622";
+        char *result;
+        if (asprintf(&result, "%s: %s\n", TM_EVENT_ID_STR, event_id) < 0) {
+                return;
+        }
+        ck_assert_str_ne(ref->record->headers[TM_EVENT_ID], result);
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, 0);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], result);
+        free(result);
+}
+END_TEST
+
+START_TEST(record_set_event_id_invalid_chars)
+{
+        int ret;
+        char *event_id = "aaaaaa000000333333444444666666ZZ";
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+START_TEST(record_set_event_id_null)
+{
+        int ret;
+        char *event_id = NULL;
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+START_TEST(record_set_event_id_short)
+{
+        int ret;
+        char *event_id = "aaaa";
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+START_TEST(record_set_event_id_long)
+{
+        int ret;
+        char *event_id = "0000000000000000000000000000000000000000000";
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+void event_id_teardown(void)
+{
+        if (ref) {
+                free(ref);
+        }
+        free(original_event_id);
+}
+
 Suite *lib_suite(void)
 {
         Suite *s = suite_create("libtelemetry");
@@ -180,6 +262,15 @@ Suite *lib_suite(void)
         tcase_add_test(t, record_create_severity_overflow);
         suite_add_tcase(s, t);
 
+        t = tcase_create("event id");
+        tcase_add_unchecked_fixture(t, event_id_setup, event_id_teardown);
+        tcase_add_test(t, record_set_event_id);
+        tcase_add_test(t, record_set_event_id_invalid_chars);
+        tcase_add_test(t, record_set_event_id_null);
+        tcase_add_test(t, record_set_event_id_short);
+        tcase_add_test(t, record_set_event_id_long);
+        suite_add_tcase(s, t);
+
         return s;
 }
 

tests/check_ncb64.c

@@ -0,0 +1,193 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2017 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define OUT_MAX_LEN 100
+
+#include <stdio.h>
+#include <check.h>
+#include "nica/b64enc.h"
+
+START_TEST(check_nc_b64_no_overflow)
+{
+        size_t n = 2;
+        char out[OUT_MAX_LEN];
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foobar";
+
+        ck_assert_msg(!nc_b64enc_filename(filename, &out[0], n), "Should quit since buffer smaller than content");
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_empty)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "\0";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/empty";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_fo)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm8=";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/fo";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_foob)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm9vYg==";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foob";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_foobar)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm9vYmFyZm9vYmFy";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foobar";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_long_text)
+{
+        size_t out_len = 8000;
+        char out[out_len];
+        const char *expected_out = \
+                "TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIG5vbnVtbXkgdml0YWUsIGluIHZpdmFtdXMgc3Vz\n"
+                "cGVuZGlzc2UgYWMuIFNlbXBlciBzZWQgcGhhcmV0cmEgc2NlbGVyaXNxdWUuIEVnZXQgZWxlaWZl\n"
+                "bmQgYW1ldCB2ZWwgbnVuYyB2b2x1dHBhdCBjdXJzdXMsIGF1Y3RvciBwbGF0ZWEgcHJldGl1bSwg\n"
+                "bWF0dGlzIHJpc3VzIGZhY2lsaXNpcyBmYXVjaWJ1cywgc3VzcGVuZGlzc2UgZGlhbSwgbGVjdHVz\n"
+                "IG1ldHVzIG51bGxhLiBOdW5jIGp1c3RvIGZhY2lsaXNpIG5hbSBmZWxpcyB2ZWwgbGFvcmVldCwg\n"
+                "bmliaCBsZW8gbWFzc2Egc3VzcGVuZGlzc2UgYWNjdW1zYW4gY29udmFsbGlzLCBzZWQgcXVpcyBw\n"
+                "ZWxsZW50ZXNxdWUgZWdlc3RhcywgbG9yZW0gYW50ZSBtb3JiaSBtYXR0aXMgdml0YWUuIFByYWVz\n"
+                "ZW50IGluIHJ1dHJ1bSBlZ2V0IGFjLCB2aXZhbXVzIHBlZGUgc3VzcGVuZGlzc2UgbGVjdHVzIG51\n"
+                "bGxhbSwgZXQgcGVkZSBldSBvZGlvIHBlZGUsIGV0aWFtIGxlbyBlZ2VzdGFzIGluLCBub251bW15\n"
+                "IHNlbXBlci4gQSBhY2N1bXNhbiBkdWksIHZlc3RpYnVsdW0gcmlkaWN1bHVzIGV0IGluLCBqdXN0\n"
+                "byBldCBwbGFjZXJhdCBkdWlzIHV0IHZpdmFtdXMgbGliZXJvLiBWZXN0aWJ1bHVtIGVnZXQgdXQg\n"
+                "bW9sbGlzLCBudW5jIGEgc29sbGljaXR1ZGluLCBmcmluZ2lsbGEgZXJvcyBwb3N1ZXJlIG1vbGxp\n"
+                "cyBhYyBuYXRvcXVlIHBlZGUsIHJpc3VzIG51bGxhIHJ1dHJ1bSB0dXJwaXMgYSB2aXRhZSBpZC4g\n"
+                "SW5jZXB0b3MgdGluY2lkdW50IHF1aWRlbSBhcmN1IHR1cnBpcyBudW5jIHNvbGxpY2l0dWRpbi4g\n"
+                "UGhhc2VsbHVzIG1ldHVzIGVyYXQgcGxhY2VyYXQsIGluIGVyYXQgbG9yZW0gbWF1cmlzIG51bGxh\n"
+                "LCBhdCBsaWJlcm8gZXJhdCBzdXNwZW5kaXNzZSBzYXBpZW4gbW9udGVzIHJob25jdXMsIG1hZ25h\n"
+                "IHF1YW0uIEp1c3RvIHByYWVzZW50IGxhY3VzIG1hc3NhLCByZXByZWhlbmRlcml0IG51bmMsIHJp\n"
+                "c3VzIGRpZ25pc3NpbSBmZWxpcyBub24sIGRpZ25pc3NpbSBvZGlvIHRlbGx1cyBhbGlxdWFtLCBl\n"
+                "bmltIHNvZGFsZXMgdGVtcG9yIHF1aXNxdWUgc29kYWxlcyBtYWduaXMgcG9ydHRpdG9yLiBMaWd1\n"
+                "bGEgc2VkIHJpc3VzLCBldSBlc3QgcXVpc3F1ZSwgY2xhc3Mgbm9uIGF1dGUgZXUsIHBsYXRlYSBz\n"
+                "Y2VsZXJpc3F1ZSBpZCBpbnRlZ2VyIGRpZ25pc3NpbW9zIGludGVnZXIgc3VzY2lwaXQuClZlaGlj\n"
+                "dWxhIGVpdXMgbmVjLCBjb252YWxsaXMgbWFlY2VuYXMgbGVjdHVzIHB1cnVzIHF1aXNxdWUgYWxp\n"
+                "cXVhbSwgZXQgZXQsIGxhY2luaWEgZGlzIGRpZ25pc3NpbSBsb3JlbS4gVm9sdXB0YXRlbSB1dCwg\n"
+                "dml2ZXJyYSBhZW5lYW4gcGhhc2VsbHVzIGZlbGlzLCBmZXJtZW50dW0gbGlndWxhIGVnZXQgdWxs\n"
+                "YW1jb3JwZXIgYW1ldCwgbWF1cmlzIGxlbyBsdWN0dXMuIEV0IGV0aWFtIHNlbXBlciBwaGFyZXRy\n"
+                "YSBuaWJoIG1pIG1hdXJpcy4gSGVuZHJlcml0IHNlZCBhbGlxdWFtLCBlZ2V0IGlkIHRlbGx1cyBt\n"
+                "YWduYSwgc2VkIGxhb3JlZXQgdWx0cmljZXMgZW5pbSwgbW9sbGlzIHN1c3BlbmRpc3NlIGluLiBF\n"
+                "dSBudWxsYW0gbG9yZW0gc2l0IGV0aWFtLCBvcmNpIGluIGxpYmVyby4gTnVuYyBpcHN1bSBtYXVy\n"
+                "aXMgZXQgc2VtIGhhYy4gRGljdHVtIGZhdWNpYnVzIGRpcyB2aXRhZSBpbiB2b2x1dHBhdCBtb3Ji\n"
+                "aSwgYSBkaWN0dW0sIHF1aXNxdWUgbWF1cmlzLCBhdWN0b3IgbmVjIHZlbCBwZWxsZW50ZXNxdWUg\n"
+                "dWxsYW1jb3JwZXIgZXRpYW0uIEN1bSBsaWJlcm8gd2lzaSBhY2N1bXNhbiBhbGlxdWFtIGNvbnNl\n"
+                "Y3RldHVlciB0ZWxsdXMsIHBoYXNlbGx1cyBjdXJhZSwgYXQgaWxsdW0gYW50ZSBwcmV0aXVtIG5p\n"
+                "YmggbW9yYmkuIFNlZCBibGFuZGl0IHB1bHZpbmFyIHB1bHZpbmFyLiBEaWN0dW0gZXN0IG51bmMg\n"
+                "ZWxlaWZlbmQsIHZlbGl0IHZlbGl0IHRlbXBvciBhY2N1bXNhbiBsb2JvcnRpcyBsYW9yZWV0IGNv\n"
+                "bmd1ZS4gU2l0IGFtZXQsIHNlZCBuaWJoIHBvcnJvIG5lcXVlIGF1Y3RvciBoeW1lbmFlb3MgcG9z\n"
+                "dWVyZSwgb2RpbyBudWxsYSBibGFuZGl0IGNvbmd1ZSBlbGl0LiBJZCBmYXVjaWJ1cyBldCB0ZW1w\n"
+                "dXMgbWFsZXN1YWRhIHBsYXRlYS4=";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/long_text";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], out_len), "Error opening file");
+        ck_assert_str_eq(out, expected_out);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_filehandler_foobar)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm9vYmFyZm9vYmFy";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foobar";
+        FILE *fh = NULL;
+
+        fh = fopen(filename, "rb");
+
+        if (fh == NULL) {
+                ck_abort_msg("Unable to open foobar file");
+        }
+
+        ck_assert_msg(nc_b64enc_file(fh, &out[0], n), "Result does not match");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+Suite *config_suite(void)
+{
+        // A suite is comprised of test cases, defined below
+        Suite *s = suite_create("nc_b64enc");
+
+        // Individual unit tests are added to "test cases"
+        TCase *t = tcase_create("nc_b64enc");
+        tcase_add_test(t, check_nc_b64_no_overflow);
+        tcase_add_test(t, check_nc_b64_enc_n_file_empty);
+        tcase_add_test(t, check_nc_b64_enc_n_file_fo);
+        tcase_add_test(t, check_nc_b64_enc_n_file_foob);
+        tcase_add_test(t, check_nc_b64_enc_n_file_foobar);
+        tcase_add_test(t, check_nc_b64_enc_n_file_long_text);
+        tcase_add_test(t, check_nc_b64_enc_n_filehandler_foobar);
+
+        suite_add_tcase(s, t);
+
+        return s;
+}
+
+int main(void)
+{
+        Suite *s;
+        SRunner *sr;
+
+        s = config_suite();
+        sr = srunner_create(s);
+
+        // Use the TAP driver for now, so that each
+        // unit test will PASS/FAIL in the log output.
+        srunner_set_log(sr, NULL);
+        srunner_set_tap(sr, "-");
+
+        srunner_run_all(sr, CK_SILENT);
+        // failed = srunner_ntests_failed(sr);
+        srunner_free(sr);
+
+        // if you want the TAP driver to report a hard error based
+        // on certain conditions (e.g. number of failed tests, etc.),
+        // return non-zero here instead.
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */
+

tests/check_probes.c

@@ -22,16 +22,16 @@
 #include <errno.h>
 #include <stdbool.h>
 #include <string.h>
-#include <glib.h>
 
+#include "nica/nc-string.h"
 #include "log.h"
 #include "read_oopsfile.h"
 #include "src/probes/klog_scanner.h"
 #include "src/probes/oops_parser.h"
 
 char reason[1024];
-GString *bt;
-GString *pl;
+nc_string *bt;
+nc_string *pl;
 
 void callback_func(struct oops_log_msg *msg)
 {
@@ -100,26 +100,26 @@ START_TEST(watchdog_payload)
         ck_assert(strstr(pl->str, "#1 dump_stack"));
         ck_assert(strstr(pl->str, "#2 warn_slowpath_common"));
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
-        ck_assert(strstr(pl->str, "#4 __queue_work"));
+        ck_assert(strstr(pl->str, "#4 ? __queue_work"));
         ck_assert(strstr(pl->str, "#5 dev_watchdog"));
-        ck_assert(strstr(pl->str, "#6 dev_deactivate_queue.constprop.30"));
+        ck_assert(strstr(pl->str, "#6 ? dev_deactivate_queue.constprop.30"));
         ck_assert(strstr(pl->str, "#7 call_timer_fn"));
-        ck_assert(strstr(pl->str, "#8 dev_deactivate_queue.constprop.30"));
+        ck_assert(strstr(pl->str, "#8 ? dev_deactivate_queue.constprop.30"));
         ck_assert(strstr(pl->str, "#9 run_timer_softirq"));
         ck_assert(strstr(pl->str, "#10 __do_softirq"));
         ck_assert(strstr(pl->str, "#11 irq_exit"));
         ck_assert(strstr(pl->str, "#12 smp_apic_timer_interrupt"));
         ck_assert(strstr(pl->str, "#13 apic_timer_interrupt"));
-        ck_assert(strstr(pl->str, "#14 __hrtimer_start_range_ns"));
-        ck_assert(strstr(pl->str, "#15 cpuidle_enter_state"));
-        ck_assert(strstr(pl->str, "#16 cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#14 ? __hrtimer_start_range_ns"));
+        ck_assert(strstr(pl->str, "#15 ? cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#16 ? cpuidle_enter_state"));
         ck_assert(strstr(pl->str, "#17 cpuidle_idle_call"));
         ck_assert(strstr(pl->str, "#18 arch_cpu_idle"));
         ck_assert(strstr(pl->str, "#19 cpu_startup_entry"));
-        ck_assert(strstr(pl->str, "#20 clockevents_register_device"));
+        ck_assert(strstr(pl->str, "#20 ? clockevents_register_device"));
         ck_assert(strstr(pl->str, "#21 start_secondary"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 
 }
 END_TEST
@@ -141,10 +141,10 @@ START_TEST(alsa_bug_payload)
         ck_assert(strstr(pl->str, "Tainted : Not tainted"));
 
         ck_assert(strstr(pl->str, "#1 warn_on_slowpath"));
-        ck_assert(strstr(pl->str, "#2 vsnprintf"));
-        ck_assert(strstr(pl->str, "#3 _spin_unlock_irqrestore"));
+        ck_assert(strstr(pl->str, "#2 ? vsnprintf"));
+        ck_assert(strstr(pl->str, "#3 ? _spin_unlock_irqrestore"));
         ck_assert(strstr(pl->str, "#10 start_kernel"));
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -173,16 +173,16 @@ START_TEST(warning_payload)
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
         ck_assert(strstr(pl->str, "#4 preempt_notifier_register"));
         ck_assert(strstr(pl->str, "#5 VBoxHost_RTThreadCtxHooksRegister"));
-        ck_assert(strstr(pl->str, "#6 update_curr"));
-        ck_assert(strstr(pl->str, "#7 supdrvIOCtlFast"));
-        ck_assert(strstr(pl->str, "#8 VBoxDrvLinuxIOCtl_4_3_28"));
-        ck_assert(strstr(pl->str, "#9 put_prev_entity"));
-        ck_assert(strstr(pl->str, "#10 do_vfs_ioctl"));
-        ck_assert(strstr(pl->str, "#11 pick_next_task_rt"));
-        ck_assert(strstr(pl->str, "#12 SyS_ioctl"));
-        ck_assert(strstr(pl->str, "#13 entry_SYSCALL_64_fastpath"));
+        ck_assert(strstr(pl->str, "#6 ? update_curr"));
+        ck_assert(strstr(pl->str, "#7 ? supdrvIOCtlFast"));
+        ck_assert(strstr(pl->str, "#8 ? VBoxDrvLinuxIOCtl_4_3_28"));
+        ck_assert(strstr(pl->str, "#9 ? put_prev_entity"));
+        ck_assert(strstr(pl->str, "#10 ? do_vfs_ioctl"));
+        ck_assert(strstr(pl->str, "#11 ? pick_next_task_rt"));
+        ck_assert(strstr(pl->str, "#12 ? SyS_ioctl"));
+        ck_assert(strstr(pl->str, "#13 ? entry_SYSCALL_64_fastpath"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -208,21 +208,21 @@ START_TEST(warn_on_payload)
         ck_assert(strstr(pl->str, "#2 warn_slowpath_common"));
         ck_assert(strstr(pl->str, "#3 warn_slowpath_fmt"));
         ck_assert(strstr(pl->str, "#4 dev_watchdog"));
-        ck_assert(strstr(pl->str, "#5 qdisc_rcu_free"));
+        ck_assert(strstr(pl->str, "#5 ? qdisc_rcu_free"));
         ck_assert(strstr(pl->str, "#6 call_timer_fn"));
-        ck_assert(strstr(pl->str, "#7 qdisc_rcu_free"));
+        ck_assert(strstr(pl->str, "#7 ? qdisc_rcu_free"));
         ck_assert(strstr(pl->str, "#8 run_timer_softirq"));
         ck_assert(strstr(pl->str, "#9 __do_softirq"));
         ck_assert(strstr(pl->str, "#10 irq_exit"));
         ck_assert(strstr(pl->str, "#11 smp_apic_timer_interrupt"));
         ck_assert(strstr(pl->str, "#12 apic_timer_interrupt"));
-        ck_assert(strstr(pl->str, "#13 cpuidle_enter_state"));
-        ck_assert(strstr(pl->str, "#14 cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#13 ? cpuidle_enter_state"));
+        ck_assert(strstr(pl->str, "#14 ? cpuidle_enter_state"));
         ck_assert(strstr(pl->str, "#15 cpuidle_enter"));
         ck_assert(strstr(pl->str, "#16 cpu_startup_entry"));
         ck_assert(strstr(pl->str, "#17 start_secondary"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -285,22 +285,22 @@ START_TEST(sysctl2_payload)
         ck_assert(strstr(pl->str, "Kernel Version : 2.6.32.53-bigmem-5-openvz #28"));
         ck_assert(strstr(pl->str, "Tainted : P"));
 
-        ck_assert(strstr(pl->str, "#1 set_fail"));
-        ck_assert(strstr(pl->str, "#2 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#3 sysctl_check_lookup"));
-        ck_assert(strstr(pl->str, "#4 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#5 sysctl_check_lookup"));
-        ck_assert(strstr(pl->str, "#6 sysctl_check_table"));
-        ck_assert(strstr(pl->str, "#7 __kmalloc"));
-        ck_assert(strstr(pl->str, "#8 __register_sysctl_paths"));
-        ck_assert(strstr(pl->str, "#9 proc_register"));
-        ck_assert(strstr(pl->str, "#10 register_net_sysctl_table"));
-        ck_assert(strstr(pl->str, "#11 nf_conntrack_net_init"));
-        ck_assert(strstr(pl->str, "#12 setup_net"));
-        ck_assert(strstr(pl->str, "#13 copy_net_ns"));
-        ck_assert(strstr(pl->str, "#14 create_new_namespaces"));
+        ck_assert(strstr(pl->str, "#1 ? set_fail"));
+        ck_assert(strstr(pl->str, "#2 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#3 ? sysctl_check_lookup"));
+        ck_assert(strstr(pl->str, "#4 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#5 ? sysctl_check_lookup"));
+        ck_assert(strstr(pl->str, "#6 ? sysctl_check_table"));
+        ck_assert(strstr(pl->str, "#7 ? __kmalloc"));
+        ck_assert(strstr(pl->str, "#8 ? __register_sysctl_paths"));
+        ck_assert(strstr(pl->str, "#9 ? proc_register"));
+        ck_assert(strstr(pl->str, "#10 ? register_net_sysctl_table"));
+        ck_assert(strstr(pl->str, "#11 ? nf_conntrack_net_init"));
+        ck_assert(strstr(pl->str, "#12 ? setup_net"));
+        ck_assert(strstr(pl->str, "#13 ? copy_net_ns"));
+        ck_assert(strstr(pl->str, "#14 ? create_new_namespaces"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -327,7 +327,7 @@ START_TEST(softlockup_payload)
         ck_assert(strstr(pl->str, "#2 btrfs_tree_read_lock"));
         ck_assert(strstr(pl->str, "#3 btrfs_read_lock_root_node"));
         ck_assert(strstr(pl->str, "#4 btrfs_search_slot"));
-        ck_assert(strstr(pl->str, "#5 crypto_shash_update"));
+        ck_assert(strstr(pl->str, "#5 ? crypto_shash_update"));
         ck_assert(strstr(pl->str, "#6 btrfs_lookup_xattr"));
         ck_assert(strstr(pl->str, "#7 __btrfs_getxattr"));
         ck_assert(strstr(pl->str, "#8 btrfs_getxattr"));
@@ -337,7 +337,7 @@ START_TEST(softlockup_payload)
         ck_assert(strstr(pl->str, "#12 SyS_fgetxattr"));
         ck_assert(strstr(pl->str, "#13 system_call_fastpath"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -356,21 +356,21 @@ START_TEST(rtnl_payload)
 
         ck_assert(strstr(pl->str, "Tainted : Not tainted"));
 
-        ck_assert(strstr(pl->str, "#1 dump_stack"));
-        ck_assert(strstr(pl->str, "#2 bond_set_rx_mode"));
-        ck_assert(strstr(pl->str, "#3 __dev_mc_add"));
-        ck_assert(strstr(pl->str, "#4 igmp6_group_added"));
-        ck_assert(strstr(pl->str, "#5 sock_def_readable"));
-        ck_assert(strstr(pl->str, "#6 ipv6_dev_mc_inc"));
-        ck_assert(strstr(pl->str, "#7 ipv6_sock_mc_join"));
-        ck_assert(strstr(pl->str, "#8 do_ipv6_setsockopt.isra.6"));
-        ck_assert(strstr(pl->str, "#9 pipe_write"));
-        ck_assert(strstr(pl->str, "#10 SYSC_sendto"));
-        ck_assert(strstr(pl->str, "#11 ipv6_setsockopt"));
-        ck_assert(strstr(pl->str, "#12 SyS_setsockopt"));
-        ck_assert(strstr(pl->str, "#13 system_call_fastpath"));
+        ck_assert(strstr(pl->str, "#1 ? dump_stack"));
+        ck_assert(strstr(pl->str, "#2 ? bond_set_rx_mode"));
+        ck_assert(strstr(pl->str, "#3 ? __dev_mc_add"));
+        ck_assert(strstr(pl->str, "#4 ? igmp6_group_added"));
+        ck_assert(strstr(pl->str, "#5 ? sock_def_readable"));
+        ck_assert(strstr(pl->str, "#6 ? ipv6_dev_mc_inc"));
+        ck_assert(strstr(pl->str, "#7 ? ipv6_sock_mc_join"));
+        ck_assert(strstr(pl->str, "#8 ? do_ipv6_setsockopt.isra.6"));
+        ck_assert(strstr(pl->str, "#9 ? pipe_write"));
+        ck_assert(strstr(pl->str, "#10 ? SYSC_sendto"));
+        ck_assert(strstr(pl->str, "#11 ? ipv6_setsockopt"));
+        ck_assert(strstr(pl->str, "#12 ? SyS_setsockopt"));
+        ck_assert(strstr(pl->str, "#13 ? system_call_fastpath"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -400,7 +400,7 @@ START_TEST(kernel_null_pointer_payload)
         ck_assert(strstr(pl->str, "#6 sys_inotify_rm_watch"));
         ck_assert(strstr(pl->str, "#7 system_call_fastpath"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -432,7 +432,7 @@ START_TEST(kernel_bug_payload)
         ck_assert(strstr(pl->str, "#8 ptregscall_common"));
         ck_assert(strstr(pl->str, "#9 sysret_signal"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -454,51 +454,51 @@ START_TEST(irq_payload)
         ck_assert(strstr(pl->str, "#1 dump_stack"));
         ck_assert(strstr(pl->str, "#2 __report_bad_irq"));
         ck_assert(strstr(pl->str, "#3 note_interrupt"));
-        ck_assert(strstr(pl->str, "#4 __do_softirq"));
+        ck_assert(strstr(pl->str, "#4 ? __do_softirq"));
         ck_assert(strstr(pl->str, "#5 handle_irq_event_percpu"));
         ck_assert(strstr(pl->str, "#6 handle_irq_event"));
-        ck_assert(strstr(pl->str, "#7 handle_nested_irq"));
+        ck_assert(strstr(pl->str, "#7 ? handle_nested_irq"));
         ck_assert(strstr(pl->str, "#8 handle_level_irq"));
-        ck_assert(strstr(pl->str, "#9 do_IRQ"));
-        ck_assert(strstr(pl->str, "#10 common_interrupt"));
-        ck_assert(strstr(pl->str, "#11 __do_softirq"));
-        ck_assert(strstr(pl->str, "#12 handle_irq"));
-        ck_assert(strstr(pl->str, "#13 irq_exit"));
-        ck_assert(strstr(pl->str, "#14 do_IRQ"));
-        ck_assert(strstr(pl->str, "#15 pci_write"));
-        ck_assert(strstr(pl->str, "#16 common_interrupt"));
-        ck_assert(strstr(pl->str, "#17 audit_make_tree"));
-        ck_assert(strstr(pl->str, "#18 usbdev_ioctl"));
-        ck_assert(strstr(pl->str, "#19 _raw_spin_unlock_irqrestore"));
-        ck_assert(strstr(pl->str, "#20 __setup_irq"));
-        ck_assert(strstr(pl->str, "#21 kmem_cache_alloc_trace"));
-        ck_assert(strstr(pl->str, "#22 vsnprintf"));
-        ck_assert(strstr(pl->str, "#23 request_threaded_irq"));
-        ck_assert(strstr(pl->str, "#24 usb_hcd_platform_shutdown"));
-        ck_assert(strstr(pl->str, "#25 request_threaded_irq"));
-        ck_assert(strstr(pl->str, "#26 usb_add_hcd"));
-        ck_assert(strstr(pl->str, "#27 usb_hcd_pci_probe"));
-        ck_assert(strstr(pl->str, "#28 rpm_resume"));
-        ck_assert(strstr(pl->str, "#29 pci_device_probe"));
-        ck_assert(strstr(pl->str, "#30 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#31 pci_match_device"));
-        ck_assert(strstr(pl->str, "#32 __driver_attach"));
-        ck_assert(strstr(pl->str, "#33 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#34 bus_for_each_dev"));
-        ck_assert(strstr(pl->str, "#35 driver_attach"));
-        ck_assert(strstr(pl->str, "#36 driver_probe_device"));
-        ck_assert(strstr(pl->str, "#37 bus_add_driver"));
-        ck_assert(strstr(pl->str, "#38 pci_match_id"));
-        ck_assert(strstr(pl->str, "#39 driver_register"));
-        ck_assert(strstr(pl->str, "#40 ohci_hcd_mod_init"));
-        ck_assert(strstr(pl->str, "#41 ohci_hcd_mod_init"));
-        ck_assert(strstr(pl->str, "#42 __pci_register_driver"));
-        ck_assert(strstr(pl->str, "#43 uhci_hcd_init"));
-        ck_assert(strstr(pl->str, "#44 do_one_initcall"));
-        ck_assert(strstr(pl->str, "#45 kernel_init_freeable"));
-        ck_assert(strstr(pl->str, "#46 do_early_param"));
+        ck_assert(strstr(pl->str, "#9 ? do_IRQ"));
+        ck_assert(strstr(pl->str, "#10 ? common_interrupt"));
+        ck_assert(strstr(pl->str, "#11 ? __do_softirq"));
+        ck_assert(strstr(pl->str, "#12 ? handle_irq"));
+        ck_assert(strstr(pl->str, "#13 ? irq_exit"));
+        ck_assert(strstr(pl->str, "#14 ? do_IRQ"));
+        ck_assert(strstr(pl->str, "#15 ? pci_write"));
+        ck_assert(strstr(pl->str, "#16 ? common_interrupt"));
+        ck_assert(strstr(pl->str, "#17 ? audit_make_tree"));
+        ck_assert(strstr(pl->str, "#18 ? usbdev_ioctl"));
+        ck_assert(strstr(pl->str, "#19 ? _raw_spin_unlock_irqrestore"));
+        ck_assert(strstr(pl->str, "#20 ? __setup_irq"));
+        ck_assert(strstr(pl->str, "#21 ? kmem_cache_alloc_trace"));
+        ck_assert(strstr(pl->str, "#22 ? vsnprintf"));
+        ck_assert(strstr(pl->str, "#23 ? request_threaded_irq"));
+        ck_assert(strstr(pl->str, "#24 ? usb_hcd_platform_shutdown"));
+        ck_assert(strstr(pl->str, "#25 ? request_threaded_irq"));
+        ck_assert(strstr(pl->str, "#26 ? usb_add_hcd"));
+        ck_assert(strstr(pl->str, "#27 ? usb_hcd_pci_probe"));
+        ck_assert(strstr(pl->str, "#28 ? rpm_resume"));
+        ck_assert(strstr(pl->str, "#29 ? pci_device_probe"));
+        ck_assert(strstr(pl->str, "#30 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#31 ? pci_match_device"));
+        ck_assert(strstr(pl->str, "#32 ? __driver_attach"));
+        ck_assert(strstr(pl->str, "#33 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#34 ? bus_for_each_dev"));
+        ck_assert(strstr(pl->str, "#35 ? driver_attach"));
+        ck_assert(strstr(pl->str, "#36 ? driver_probe_device"));
+        ck_assert(strstr(pl->str, "#37 ? bus_add_driver"));
+        ck_assert(strstr(pl->str, "#38 ? pci_match_id"));
+        ck_assert(strstr(pl->str, "#39 ? driver_register"));
+        ck_assert(strstr(pl->str, "#40 ? ohci_hcd_mod_init"));
+        ck_assert(strstr(pl->str, "#41 ? ohci_hcd_mod_init"));
+        ck_assert(strstr(pl->str, "#42 ? __pci_register_driver"));
+        ck_assert(strstr(pl->str, "#43 ? uhci_hcd_init"));
+        ck_assert(strstr(pl->str, "#44 ? do_one_initcall"));
+        ck_assert(strstr(pl->str, "#45 ? kernel_init_freeable"));
+        ck_assert(strstr(pl->str, "#46 ? do_early_param"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 
 }
 END_TEST
@@ -521,16 +521,16 @@ START_TEST(general_protection_fault_payload)
         ck_assert(strstr(pl->str, "Modules : tun 8021q garp mrp bridge stp llc bonding ipt_REJECT xt_comment xt_multiport"));
         ck_assert(strstr(pl->str, "lpc_ich iw_cm mfd_core nfsd auth_rpcgss wmi nfs_acl lockd ipmi_si ipmi_devintf"));
 
-        ck_assert(strstr(pl->str, "#1 alloc_pipe_info"));
+        ck_assert(strstr(pl->str, "#1 ? alloc_pipe_info"));
         ck_assert(strstr(pl->str, "#2 alloc_pipe_info"));
         ck_assert(strstr(pl->str, "#3 create_pipe_files"));
         ck_assert(strstr(pl->str, "#4 __do_pipe_flags"));
         ck_assert(strstr(pl->str, "#5 SyS_pipe2"));
-        ck_assert(strstr(pl->str, "#6 __audit_syscall_exit"));
+        ck_assert(strstr(pl->str, "#6 ? __audit_syscall_exit"));
         ck_assert(strstr(pl->str, "#7 SyS_pipe"));
         ck_assert(strstr(pl->str, "#8 system_call_fastpath"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -552,7 +552,7 @@ START_TEST(double_fault_payload)
         ck_assert(strstr(pl->str, "Modules : nls_utf8 vfat fat sit tunnel4 ext2 fuse ip6table_filter ip6_tables ebtable_nat"));
         ck_assert(strstr(pl->str, "joydev sdhci_pci sdhci usb_storage mmc_core video output [last unloaded: speedstep_lib]"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 }
 END_TEST
 
@@ -571,7 +571,7 @@ START_TEST(bad_page_map_payload)
         ck_assert(strstr(pl->str, "Kernel Version : 3.12-1-amd64 #1 Debian 3.12.9-1"));
         ck_assert(strstr(pl->str, "Tainted : Not tainted"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 
 }
 END_TEST
@@ -594,31 +594,78 @@ START_TEST(bug_kernel_handle_payload)
         ck_assert(strstr(pl->str, "Modules : nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE be2iscsi"));
         ck_assert(strstr(pl->str, "drm_kms_helper firewire_ohci drm firewire_core tg3 crc_itu_t i2c_core video usb_storage sunrpc"));
 
-        ck_assert(strstr(pl->str, "#1 _nv007312rm"));
-        ck_assert(strstr(pl->str, "#2 _nv007847rm"));
-        ck_assert(strstr(pl->str, "#3 _nv004049rm"));
-        ck_assert(strstr(pl->str, "#4 _nv004049rm"));
-        ck_assert(strstr(pl->str, "#5 _nv010019rm"));
-        ck_assert(strstr(pl->str, "#6 _nv014983rm"));
-        ck_assert(strstr(pl->str, "#7 _nv001097rm"));
-        ck_assert(strstr(pl->str, "#8 rm_init_adapter"));
+        ck_assert(strstr(pl->str, "#1 ? _nv007312rm"));
+        ck_assert(strstr(pl->str, "#2 ? _nv007847rm"));
+        ck_assert(strstr(pl->str, "#3 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#4 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#5 ? _nv010019rm"));
+        ck_assert(strstr(pl->str, "#6 ? _nv014983rm"));
+        ck_assert(strstr(pl->str, "#7 ? _nv001097rm"));
+        ck_assert(strstr(pl->str, "#8 ? rm_init_adapter"));
+        ck_assert(strstr(pl->str, "#9 ? nv_kern_open"));
+        ck_assert(strstr(pl->str, "#10 ? chrdev_open"));
+        ck_assert(strstr(pl->str, "#11 ? do_dentry_open"));
+        ck_assert(strstr(pl->str, "#12 ? cdev_put"));
+        ck_assert(strstr(pl->str, "#13 ? finish_open"));
+        ck_assert(strstr(pl->str, "#14 ? do_last"));
+        ck_assert(strstr(pl->str, "#15 ? inode_permission"));
+        ck_assert(strstr(pl->str, "#16 ? link_path_walk"));
+        ck_assert(strstr(pl->str, "#17 ? path_openat"));
+        ck_assert(strstr(pl->str, "#18 ? do_filp_open"));
+        ck_assert(strstr(pl->str, "#19 ? __alloc_fd"));
+        ck_assert(strstr(pl->str, "#20 ? do_sys_open"));
+        ck_assert(strstr(pl->str, "#21 ? __audit_syscall_entry"));
+        ck_assert(strstr(pl->str, "#22 ? sys_open"));
+        ck_assert(strstr(pl->str, "#23 ? system_call_fastpath"));
+
+        nc_string_free(pl);
+
+}
+END_TEST
+
+START_TEST(bug_kernel_handle_payload_new_format)
+{
+        char *oopsfile = NULL;
+
+        oopsfile = TESTOOPSDIR "/bug_kernel_handle_new.txt";
+        setup_payload(oopsfile);
+
+        telem_log(LOG_ERR, "Bug kernel handle backtrace: %s\n", pl->str);
+
+        ck_assert(pl->len > 0);
+        ck_assert_str_eq(reason, "BUG: unable to handle kernel paging request at 0000000000002658");
+
+        ck_assert(strstr(pl->str, "Kernel Version : 3.7.9-201.fc18.x86_64 #1 Apple Inc. MacBookPro6,2/Mac-F22586C8"));
+        ck_assert(strstr(pl->str, "Tainted : PF"));
+
+        ck_assert(strstr(pl->str, "Modules : nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE be2iscsi"));
+        ck_assert(strstr(pl->str, "drm_kms_helper firewire_ohci drm firewire_core tg3 crc_itu_t i2c_core video usb_storage sunrpc"));
+
+        ck_assert(strstr(pl->str, "#1 ? _nv007312rm"));
+        ck_assert(strstr(pl->str, "#2 ? _nv007847rm"));
+        ck_assert(strstr(pl->str, "#3 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#4 ? _nv004049rm"));
+        ck_assert(strstr(pl->str, "#5 ? _nv010019rm"));
+        ck_assert(strstr(pl->str, "#6 ? _nv014983rm"));
+        ck_assert(strstr(pl->str, "#7 ? _nv001097rm"));
+        ck_assert(strstr(pl->str, "#8 ? rm_init_adapter"));
         ck_assert(strstr(pl->str, "#9 nv_kern_open"));
-        ck_assert(strstr(pl->str, "#10 chrdev_open"));
-        ck_assert(strstr(pl->str, "#11 do_dentry_open"));
-        ck_assert(strstr(pl->str, "#12 cdev_put"));
-        ck_assert(strstr(pl->str, "#13 finish_open"));
-        ck_assert(strstr(pl->str, "#14 do_last"));
-        ck_assert(strstr(pl->str, "#15 inode_permission"));
+        ck_assert(strstr(pl->str, "#10 ? chrdev_open"));
+        ck_assert(strstr(pl->str, "#11 ? do_dentry_open"));
+        ck_assert(strstr(pl->str, "#12 ? cdev_put"));
+        ck_assert(strstr(pl->str, "#13 ? finish_open"));
+        ck_assert(strstr(pl->str, "#14 ? do_last"));
+        ck_assert(strstr(pl->str, "#15 ? inode_permission"));
         ck_assert(strstr(pl->str, "#16 link_path_walk"));
         ck_assert(strstr(pl->str, "#17 path_openat"));
-        ck_assert(strstr(pl->str, "#18 do_filp_open"));
-        ck_assert(strstr(pl->str, "#19 __alloc_fd"));
-        ck_assert(strstr(pl->str, "#20 do_sys_open"));
-        ck_assert(strstr(pl->str, "#21 __audit_syscall_entry"));
-        ck_assert(strstr(pl->str, "#22 sys_open"));
-        ck_assert(strstr(pl->str, "#23 system_call_fastpath"));
+        ck_assert(strstr(pl->str, "#18 ? do_filp_open"));
+        ck_assert(strstr(pl->str, "#19 ? __alloc_fd"));
+        ck_assert(strstr(pl->str, "#20 ? do_sys_open"));
+        ck_assert(strstr(pl->str, "#21 ? __audit_syscall_entry"));
+        ck_assert(strstr(pl->str, "#22 ? sys_open"));
+        ck_assert(strstr(pl->str, "#23 ? system_call_fastpath"));
 
-        g_string_free(pl, true);
+        nc_string_free(pl);
 
 }
 END_TEST
@@ -648,6 +695,7 @@ Suite *config_suite(void)
         tcase_add_test(t, double_fault_payload);
         tcase_add_test(t, bad_page_map_payload);
         tcase_add_test(t, bug_kernel_handle_payload);
+        tcase_add_test(t, bug_kernel_handle_payload_new_format);
 
         //TODO fix
         //tcase_add_test(t, badness_payload);

tests/local.mk

@@ -14,6 +14,8 @@ check_PROGRAMS = \
 	%D%/check_config \
 	%D%/check_daemon \
 	%D%/check_probes \
+	%D%/check_ncb64  \
+	%D%/check_journal \
 	%D%/check_libtelemetry
 
 dist_check_SCRIPTS = \
@@ -24,27 +26,25 @@ dist_check_SCRIPTS = \
 
 %C%_check_config_CFLAGS = \
 	$(AM_CFLAGS) \
-	@CHECK_CFLAGS@ \
-	@GLIB_CFLAGS@ 
+	@CHECK_CFLAGS@
 
 %C%_check_config_LDADD = \
 	@CHECK_LIBS@ \
-	@GLIB_LIBS@ \
 	$(top_builddir)/src/libtelem-shared.la
 
 %C%_check_daemon_SOURCES = \
 	%D%/check_daemon.c \
 	src/telemdaemon.c \
-	src/telemdaemon.h
+	src/telemdaemon.h \
+	src/journal/journal.c \
+	src/journal/journal.h
 
 %C%_check_daemon_CFLAGS = \
 	$(AM_CFLAGS) \
 	@CHECK_CFLAGS@ \
-	@GLIB_CFLAGS@ \
 	@CURL_CFLAGS@
 %C%_check_daemon_LDADD = \
 	@CHECK_LIBS@ \
-	@GLIB_LIBS@ \
 	@CURL_LIBS@ \
 	$(top_builddir)/src/libtelem-shared.la
 
@@ -61,6 +61,7 @@ endif
         %D%/read_oopsfile.h \
         %D%/read_oopsfile.c \
         %D%/check_probes.c \
+	src/nica/nc-string.c \
 	src/probes/klog_scanner.c \
 	src/probes/klog_scanner.h \
 	src/probes/oops_parser.c \
@@ -72,6 +73,7 @@ EXTRA_DIST += \
 	%D%/oops_test_files/badness.txt \
 	%D%/oops_test_files/bad_page_map.txt \
 	%D%/oops_test_files/bug_kernel_handle.txt \
+	%D%/oops_test_files/bug_kernel_handle_new.txt \
 	%D%/oops_test_files/double_fault_DONE.txt \
 	%D%/oops_test_files/double_fault.txt \
 	%D%/oops_test_files/general_protection_fault.txt \
@@ -94,11 +96,9 @@ EXTRA_DIST += \
 
 %C%_check_probes_CFLAGS = \
         $(AM_CFLAGS) \
-        @CHECK_CFLAGS@ \
-        @GLIB_CFLAGS@
+        @CHECK_CFLAGS@
 %C%_check_probes_LDADD = \
         @CHECK_LIBS@ \
-        @GLIB_LIBS@ \
         $(top_builddir)/src/libtelem-shared.la
 
 if LOG_SYSTEMD
@@ -110,17 +110,48 @@ if HAVE_SYSTEMD_JOURNAL
 endif
 endif
 
+%C%_check_ncb64_SOURCES = \
+	%D%/check_ncb64.c \
+	src/nica/b64enc.h \
+	src/nica/b64enc.c
+
+%C%_check_ncb64_CFLAGS = \
+	$(AM_CFLAGS) \
+	@CHECK_CFLAGS@
+
+%C%_check_ncb64_LDADD = \
+	@CHECK_LIBS@
+
+EXTRA_DIST += \
+	%D%/nc_b64enc_test_files/empty \
+	%D%/nc_b64enc_test_files/fo \
+	%D%/nc_b64enc_test_files/foob \
+	%D%/nc_b64enc_test_files/foobar \
+	%D%/nc_b64enc_test_files/long_text
+
+%C%_check_journal_SOURCES = \
+	%D%/check_journal.c \
+	src/journal/journal.h \
+	src/journal/journal.c \
+	src/util.h \
+	src/util.c
+
+%C%_check_journal_CFLAGS = \
+	$(AM_CFLAGS) \
+	@CHECK_CFLAGS@
+
+%C%_check_journal_LDADD = \
+	@CHECK_LIBS@
+
 %C%_check_libtelemetry_SOURCES = \
 	%D%/check_libtelemetry.c \
 	src/telemetry.c
 
 %C%_check_libtelemetry_CFLAGS = \
 	$(AM_CFLAGS) \
-	@CHECK_CFLAGS@ \
-	@GLIB_CFLAGS@
+	@CHECK_CFLAGS@
 %C%_check_libtelemetry_LDADD = \
 	@CHECK_LIBS@ \
-        @GLIB_LIBS@ \
 	$(top_builddir)/src/libtelemetry.la \
 	$(top_builddir)/src/libtelem-shared.la
 

tests/nc_b64enc_test_files/fo

@@ -0,0 +1 @@
+fo

tests/nc_b64enc_test_files/foob

@@ -0,0 +1 @@
+foob

tests/nc_b64enc_test_files/foobar

@@ -0,0 +1 @@
+foobarfoobar

tests/nc_b64enc_test_files/long_text

@@ -0,0 +1,2 @@
+Lorem ipsum dolor sit amet, nonummy vitae, in vivamus suspendisse ac. Semper sed pharetra scelerisque. Eget eleifend amet vel nunc volutpat cursus, auctor platea pretium, mattis risus facilisis faucibus, suspendisse diam, lectus metus nulla. Nunc justo facilisi nam felis vel laoreet, nibh leo massa suspendisse accumsan convallis, sed quis pellentesque egestas, lorem ante morbi mattis vitae. Praesent in rutrum eget ac, vivamus pede suspendisse lectus nullam, et pede eu odio pede, etiam leo egestas in, nonummy semper. A accumsan dui, vestibulum ridiculus et in, justo et placerat duis ut vivamus libero. Vestibulum eget ut mollis, nunc a sollicitudin, fringilla eros posuere mollis ac natoque pede, risus nulla rutrum turpis a vitae id. Inceptos tincidunt quidem arcu turpis nunc sollicitudin. Phasellus metus erat placerat, in erat lorem mauris nulla, at libero erat suspendisse sapien montes rhoncus, magna quam. Justo praesent lacus massa, reprehenderit nunc, risus dignissim felis non, dignissim odio tellus aliquam, enim sodales tempor quisque sodales magnis porttitor. Ligula sed risus, eu est quisque, class non aute eu, platea scelerisque id integer dignissimos integer suscipit.
+Vehicula eius nec, convallis maecenas lectus purus quisque aliquam, et et, lacinia dis dignissim lorem. Voluptatem ut, viverra aenean phasellus felis, fermentum ligula eget ullamcorper amet, mauris leo luctus. Et etiam semper pharetra nibh mi mauris. Hendrerit sed aliquam, eget id tellus magna, sed laoreet ultrices enim, mollis suspendisse in. Eu nullam lorem sit etiam, orci in libero. Nunc ipsum mauris et sem hac. Dictum faucibus dis vitae in volutpat morbi, a dictum, quisque mauris, auctor nec vel pellentesque ullamcorper etiam. Cum libero wisi accumsan aliquam consectetuer tellus, phasellus curae, at illum ante pretium nibh morbi. Sed blandit pulvinar pulvinar. Dictum est nunc eleifend, velit velit tempor accumsan lobortis laoreet congue. Sit amet, sed nibh porro neque auctor hymenaeos posuere, odio nulla blandit congue elit. Id faucibus et tempus malesuada platea.

tests/oops_test_files/bug_kernel_handle_new.txt

@@ -0,0 +1,51 @@
+<4>[    68764.975947] BUG: unable to handle kernel paging request at 0000000000002658
+<4>[    68764.975947] IP: [<ffffffffa07f92e5>] _nv007245rm+0x54/0xc4 [nvidia]
+<4>[    68764.975947] PGD 22d005067 PUD 22d006067 PMD 0 
+<4>[    68764.975947] Oops: 0000 [#1] SMP 
+<4>[    68764.975947] Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_MASQUERADE be2iscsi iscsi_boot_sysfs bnx2i cnic uio cxgb4i cxgb4 cxgb3i cxgb3 mdio ip6table_mangle libcxgbi ib_iser rdma_cm ib_addr iw_cm ib_cm ib_sa ib_mad ib_core iscsi_tcp libiscsi_tcp ip6t_REJECT libiscsi scsi_transport_iscsi nf_conntrack_ipv6 nf_defrag_ipv6 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ebtable_filter ebtables rfcomm ip6table_filter ip6_tables bnep nls_utf8 hfsplus binfmt_misc snd_hda_codec_hdmi arc4 brcmsmac cordic brcmutil mac80211 cfg80211 joydev iTCO_wdt iTCO_vendor_support nvidia(POF) coretemp applesmc input_polldev microcode uvcvideo i2c_i801 intel_ips snd_hda_codec_cirrus videobuf2_vmalloc videobuf2_memops videobuf2_core videodev btusb media bluetooth snd_hda_intel snd_hda_codec rfkill snd_hwdep bcm5974 snd_seq snd_seq_device snd_pcm lpc_ich mfd_core bcma snd_page_alloc snd_timer vhost_net tun snd macvtap macvlan soundcore kvm_intel kvm apple_gmux apple_bl uinput crc32c_intel i915 ghash_clmulni_intel i2c_algo_bit drm_kms_helper firewire_ohci drm firewire_core tg3 crc_itu_t i2c_core video usb_storage sunrpc
+<4>[    68764.975947] CPU 2 
+<4>[    68764.975947] Pid: 1059, comm: Xorg Tainted: PF          O 3.7.9-201.fc18.x86_64 #1 Apple Inc. MacBookPro6,2/Mac-F22586C8
+<4>[    68764.975947] RIP: 0010:[<ffffffffa07f92e5>]  [<ffffffffa07f92e5>] _nv007245rm+0x54/0xc4 [nvidia]
+<4>[    68764.975947] RSP: 0018:ffff88022d0639b8  EFLAGS: 00010282
+<4>[    68764.975947] RAX: 0000000000000000 RBX: ffff88022d190008 RCX: 0000000000000000
+<4>[    68764.975947] RDX: 0000000000000000 RSI: 0000000000000015 RDI: 0000000000000000
+<4>[    68764.975947] RBP: ffff88022d30df08 R08: 0000000000000002 R09: ffff88023f342508
+<4>[    68764.975947] R10: ffff88022d951030 R11: 0000000000000000 R12: 0000000000000000
+<4>[    68764.975947] R13: ffff88023f3b0008 R14: 0000000000000000 R15: ffff88022da96008
+<4>[    68764.975947] FS:  00007f771f4f2940(0000) GS:ffff88024fc80000(0000) knlGS:0000000000000000
+<4>[    68764.975947] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
+<4>[    68764.975947] CR2: 0000000000002658 CR3: 000000022d004000 CR4: 00000000000007e0
+<4>[    68764.975947] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
+<4>[    68764.975947] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
+<4>[    68764.975947] Process Xorg
+<4>[    68764.975947] Stack:
+<4>[    68764.975947]  ffff88022d33a008 ffff88022d33a008 0000000000000002 ffff88023f3b0008
+<4>[    68764.975947]  ffff88022d190008 ffffffffa07deec8 ffff88023d67b008 ffff88022d190008
+<4>[    68764.975947]  ffff88023f3b0008 ffff88023f3b0008 ffff88022da94808 ffffffffa083e77f
+<4>[    68764.975947] Call Trace:
+<4>[    68764.975947]  ? _nv007312rm+0x7f8/0x938 [nvidia]
+<4>[    68764.975947]  ? _nv007847rm+0x80/0x276 [nvidia]
+<4>[    68764.975947]  ? _nv004049rm+0x85fb/0xaef0 [nvidia]
+<4>[    68764.975947]  ? _nv004049rm+0x705f/0xaef0 [nvidia]
+<4>[    68764.975947]  ? _nv010019rm+0x25/0x40 [nvidia]
+<4>[    68764.975947]  ? _nv014983rm+0x7c9/0x943 [nvidia]
+<4>[    68764.975947]  ? _nv001097rm+0x42b/0x661 [nvidia]
+<4>[    68764.975947]  ? rm_init_adapter+0xac/0x146 [nvidia]
+<4>[    68764.975947]  nv_kern_open+0x4b4/0x800 [nvidia]
+<4>[    68764.975947]  ? chrdev_open+0x9b/0x180
+<4>[    68764.975947]  ? do_dentry_open+0x203/0x290
+<4>[    68764.975947]  ? cdev_put+0x30/0x30
+<4>[    68764.975947]  ? finish_open+0x35/0x50
+<4>[    68764.975947]  ? do_last+0x6de/0xe00
+<4>[    68764.975947]  ? inode_permission+0x18/0x50
+<4>[    68764.975947]  link_path_walk+0x77/0x870
+<4>[    68764.975947]  path_openat+0xbc/0x4d0
+<4>[    68764.975947]  ? do_filp_open+0x41/0xa0
+<4>[    68764.975947]  ? __alloc_fd+0x42/0x110
+<4>[    68764.975947]  ? do_sys_open+0xf4/0x1e0
+<4>[    68764.975947]  ? __audit_syscall_entry+0xcc/0x300
+<4>[    68764.975947]  ? sys_open+0x21/0x30
+<4>[    68764.975947]  ? system_call_fastpath+0x16/0x1b
+<4>[    68764.975947] Code: e0 05 00 00 00 00 00 00 41 bc 00 00 00 00 eb 18 44 89 e2 48 89 de 4c 89 ef ff 93 48 1a 00 00 01 83 e0 05 00 00 41 ff c4 4c 89 f7 <41> ff 96 58 26 00 00 44 39 e0 77 d9 ba 74 06 10 00 be 00 00 00 
+<4>[    68764.975947] RIP  [<ffffffffa07f92e5>] _nv007245rm+0x54/0xc4 [nvidia]
+<4>[    68764.975947]  RSP <ffff88022d0639b8>