Add github actions CI.

telemetrics-client installation starts when the package is installed,
this change makes sure that to start telemetry the first time two steps
are needed: 1- telemctl opt-in and 2- telemctl start

Signed-off-by: Alex Jaramillo <alex.jch@gmail.com>

.github/workflows/ccpp.yml

@@ -0,0 +1,29 @@
+name: C/C++ CI
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: install dependencies
+      run: sudo apt-get install libcurl4-gnutls-dev valgrind libelf-dev libdw-dev
+    - name: install check
+      run: wget https://github.com/libcheck/check/releases/download/0.12.0/check-0.12.0.tar.gz &&
+           tar xf check-0.12.0.tar.gz &&
+           pushd check-0.12.0 &&
+           ./configure --prefix=/usr --libdir=/usr/lib/x86_64-linux-gnu &&
+           make -j8 && sudo make install && popd
+    - name: autogen
+      run: sh autogen.sh
+    - name: configure
+      run: ./configure
+    - name: make
+      run: make -j8
+    - name: check
+      run: make check
+    - name: make distcheck
+      run: make distcheck

.gitignore

@@ -33,18 +33,20 @@ build-aux/m4/ltversion.m4
 *.trs
 /cscope.*
 /tags
-telemd
+telemprobd
+telempostd
 autoscan.log
 configure.scan
 tprobe
 hprobe
+bertprobe
 crashprobe
 journalprobe
 telem-record-gen
 klogscanner
-oopsprobe
 pstoreclean
 pstoreprobe
+telem_journal
 make.out
 types_c.taghl
 src/recomp.sh
@@ -53,7 +55,11 @@ src/gcc.out
 tests/demo
 tests/check_config
 tests/check_daemon
+tests/check_journal
 tests/check_libtelemetry
+tests/check_ncb64
+tests/check_postd
+tests/check_probd
 tests/check_probes
 src/data/*.path
 src/data/*.service

.travis.yml

@@ -25,9 +25,9 @@ addons:
             - libdw-dev
 
 install:
-        - wget http://downloads.sourceforge.net/project/check/check/0.10.0/check-0.10.0.tar.gz
-        - tar -xvf check-0.10.0.tar.gz
-        - pushd check-0.10.0 && ./configure --prefix=/usr --libdir=/usr/lib/x86_64-linux-gnu && make -j48 && sudo make install && popd
+        - wget https://github.com/libcheck/check/releases/download/0.12.0/check-0.12.0.tar.gz
+        - tar -xvf check-0.12.0.tar.gz
+        - pushd check-0.12.0 && ./configure --prefix=/usr --libdir=/usr/lib/x86_64-linux-gnu && make -j48 && sudo make install && popd
 
 script:
     - ./configure && make && make check

AUTHORS

@@ -3,3 +3,6 @@ Patrick McCarty <patrick.mccarty@intel.com>
 Rob Nesius <robert.a.nesius@intel.com>
 Gabrielle N. Beyer <gabrielle.n.beyer@intel.com>
 Pam Leonard <pam.leonard@intel.com>
+Alex V. Jaramillo <alex.v.jaramillo@intel.com>
+California Sullivan <california.l.sullivan@intel.com>
+Juro Bystricky <juro.bystricky@intel.com>

Makefile.am

@@ -15,11 +15,11 @@ DISTCHECK_CONFIGURE_FLAGS = \
 include $(top_srcdir)/build-aux/make/cflags.make
 include $(top_srcdir)/build-aux/make/ldflags.make
 include $(top_srcdir)/docs/local.mk
-include $(top_srcdir)/scripts/local.mk
 include $(top_srcdir)/src/local.mk
 include $(top_srcdir)/src/data/local.mk
 include $(top_srcdir)/src/nica/local.mk
 include $(top_srcdir)/src/probes/local.mk
+include $(top_srcdir)/src/journal/local.mk
 include $(top_srcdir)/tests/local.mk
 
 release:

README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/clearlinux/telemetrics-client.svg?branch=master)](https://travis-ci.org/clearlinux/telemetrics-client)
+
 Telemetrics-client
 ==================
 
@@ -6,15 +8,22 @@ solution for Linux-based operating systems. Specifically, the front end
 component includes:
 
 - telemetrics probes that collect specific types of data from the operating
-  system.
+  system. For more info on probes go [here](./src/probes).
 
 - a library, libtelemetry, that telemetrics probes use to create telemetrics
   records and send them to the daemon for further processing.
 
-- a daemon, telemd, that prepares the records to send to a telemetrics server
+- telemprobd, a daemon that handles communication with probes and forwards records
+  to a second daemon.
+
+- telempostd, this daemon sends telemetry records to a telemetrics server
   (not included in this source tree), or spools the records on disk in case
   it's unable to successfully deliver them.
 
+A telemetrics server implementation that works with this component is available
+from
+[clearlinux/telemetrics-backend](https://github.com/clearlinux/telemetrics-backend).
+
 
 Build dependencies
 ---------------------
@@ -26,7 +35,7 @@ Build dependencies
 - elfutils, which provides libelf and libdwfl libraries..
 
 - (optional) libsystemd, for syslog-style logging to the systemd journal, and
-  socket/path activation of telemd by systemd.
+  socket/path activation of telemprobd and telempostd by systemd.
 
 
 Build and installation
@@ -56,54 +65,71 @@ and modify the /etc version.
 
 Descriptions of config options are listed below in the Usage section.
 
-Starting the daemon
+Starting the client
 ---------------------
-
-Method 1 (recommended):
+To use the telemetrics client a one time explicit ```opt-in``` is required (this is
+also true when the contents of the directory ```/etc/telemetrics/``` are removed).
+To opt-in to telemetrics-client use the command:
 
 ```{r, engine='bash', count_lines}
- systemctl start telemd.socket telemd.path
+ telemctl opt-in
 ```
 
+**Note** this is a change from previous versions, before 2.3.0 installation of
+telemetrics client was enough to enable the client and if needed the client could
+be disabled with ```telemctl opt-out```. This command in previous versions created
+```/etc/telemetrics/opt-out``` file (after telemetrics-client version 2.3.0 this
+file can be safely removed).
+
+If the client was compiled with systemd support the respective activation units
+should be already in place (after a ```make install``` invocation). In this case
+the client wil start automatically when data is made available to it. i.e. when
+executing an ```/usr/bin/hprobe``` command. Otherwise use the following command:
+
+```{r, engine='bash', count_lines}
+ telemctl start
+```
 Note: the above invocation technically readies the service for both socket and
-path activation, so you may not see the daemon start right away.
-
-Method 2:
+path activation, so you may not see an "active" status. To check the status of
+telemetrics-client use:
 
 ```{r, engine='bash', count_lines}
-systemctl start telemd.service
+ telemctl is-active
+   telemprobd : active
+   telempostd : active
 ```
 
-Method 3:
+Starting individual service units ```telempostd.service``` or ```telemeprobd.service```
+is discouraged.
 
-```{r, engine='bash', count_lines}
-telemd &
-```
-
-Configure the daemon to autostart at boot
+Configure the client to autostart at boot
 ---------------------
 
+As longs as the first time ```opt-in``` was performed, the following methods are valid:
+
+
 Method 1 (recommended):
 
 Enable the socket-activated service and path unit:
 
 ```{r, engine='bash', count_lines}
-systemctl enable telemd.socket telemd.path
+systemctl enable telemprobd.socket telempostd.path
 ```
 
 Method 2:
 
-Enable the service itself, which automatically enables the socket and path
+Enable services, which automatically enables the socket and path
 units as well:
 
 ```{r, engine='bash', count_lines}
-systemctl enable telemd.service
+systemctl enable telemprobd.service
+systemctl enable telempostd.service
 ```
 
 Usage
 ---------------------
 
-Once the daemon is running, the telemetrics probes will be ready to use.
+Once the client is running, the telemetrics probes will be ready to use.
 
 Available probes:
 
@@ -113,11 +139,11 @@ that telemetrics-client works. It sends a "hello world" record to the server.
 * crash probe: A handler for core files that sends the corresponding backtraces
 to the server.
 
-The daemon uses the following configuration options from the configuration file:
+The client uses the following configuration options from the configuration file:
 
-* server: This specifies the web server that the daemon sends the telemetry records to
+* server: This specifies the web server that the telempostd sends the telemetry records to
 * socket_path: This specifies the path of the unix domain socket that the
-  daemon listens on for connections from the probes
+  telemprobd listens on for connections from the probes
 * spool_dir: This config option is related to spooling. If the daemon is not
   able to send the telemetry records to the backend server due to reasons such
   as the network availability, then it stores the records in a spool directory.
@@ -127,7 +153,7 @@ The daemon uses the following configuration options from the configuration file:
 ```{r, engine='bash', count_lines}
     mkdir -p /var/spool/telemetry
     chown -R telemetry:telemetry /var/spool/telemetry
-    systemctl restart telemd.service
+    systemctl restart telemprobd.service
 ```
 
 * record_expiry: This is the time in minutes after which the records in the
@@ -162,6 +188,56 @@ The daemon uses the following configuration options from the configuration file:
   threshold has been reached. Currently the options are 'drop' or 'spool', with
   spool being the default. If spool is chosen, records will be spooled and sent
   at a later time.
+* record_retention_enabled: When this key is enabled (true) the daemon saves a
+  copy of the payload on disk from all valid records. To avoid the excessive use
+  of disk space only the latest 100 records are kept. The default value for this
+  configuration key is false.
+* record_server_delivery_enabled: This key controls the delivery of records to
+  ```server```, when enabled (default value) the record will be posted to the
+  address in the configuration file. If this configuration key is disabled (false)
+  records will not be spooled or posted to backend. This configuration key can
+  be used in combination with ```record_retention_enabled``` to keep copies of
+  telemetry records locally only.
+
+
+Data reported
+---------------------
+
+The data reported by the telemetry client could be understood as two main sets:
+metadata and a payload.
+
+The metadata is used to report details of a machine's architecture. The
+following are the metadata values currently collected (Record Format Version 4):
+
+* record_format_version: version of the record, currently is 'Version 4'. This
+  value changes when new metadata is added.
+* classification: this field is used to identify the type of record sent by a
+  specific client probe; classifications use the format DOMAIN/PROBE/REST, where
+  DOMAIN is the vendor of the probe, PROBE is the probe name, and REST is a
+  probe-defined field to classify what is contained in the payload.
+* severity: this is an integer value between 1 and 4 where 1 is "low" and 4 is
+  "critical"
+* machine_id: a machine identifier that is rotate every 3 days for privacy reasons.
+* creation_timestamp: timestamp when the record was collected.
+* arch: a string describing machine architecture i.e. 'x86_64'.
+* host_type: a string with the combination of 'System Vendor', 'Product Name', and
+  'Product Version' read from dmi file system.
+* build: OS build number.
+* kernel_version: Kernel version.
+* payload_format_version: version of the payload, currently is 'Version 1'.
+* system_name: the value after 'ID=' from '/etc/os-release' (or distribution
+  provided folder)
+* board_name: a string read from dmi file system that combines 'Board Name' and
+  'Board Vendor'.
+* cpu_model: cpu model name extracted from '/proc/cpuinfo'.
+* bios_version: BIOS version.
+* event_id: an id to group multiple records if these were generated by a single
+  event occurrence.
+
+The payload as mentioned above is reported by probes. The telemetry library adds
+the metadata to the payload (done programatically when using library API) for
+more information about probes go [here](./src/probes).
+
 
 Machine id
 ---------------------
@@ -177,16 +253,87 @@ you can opt in by creating a static machine id file named
 # echo "unique machine ID" > /etc/telemetrics/opt-in-static-machine-id
 ```
 
-The telemetry daemon reads, at most, the first 32 characters from this file
+The telemetry client reads, at most, the first 32 characters from this file
 uses it for the machine id. You can put a string like 'my-machine-name' in this
-file to easily identify your machine.  Restart telemd for the machine id
+file to easily identify your machine.  Restart telemprobd for the machine id
 changes to take effect by running:
 
 ```{r, engine='bash', count_lines}
-# systemctl restart telemd.service
+# systemctl restart telemprobd.service
 ```
 
 You can switch back to the rotating machine id by deleting the override file
-and restarting the daemon.  You can do a quick test to check that your
+and restarting the client.  You can do a quick test to check that your
 machine-id has changed by running "hprobe" and verifying that a record has
 landed on your backend telemetrics server, with the specified machine id.
+
+Event Id
+----------------------
+
+This is a 32 character lowercase hexadecimal string i.e. '5de9de8d5f3c6a7d445d75ba01cc3322'.
+This header is used to group multiple records by an event id. Before this header
+every single record could have been thought of an event, however this is not always
+the case. There are "events" that trigger the creation of multiple records (i.e.
+updates). The ```event_id``` header was added for probes with the capability to
+detect events and group records based on such events. This header was added to
+```telem-record-gen``` and can be specified using the ```-e``` (```--event-id```
+long form) switch.
+
+```{r, engine=bash, count_lines}
+  -e, --event-id        Event id to use in the record
+```
+
+Debugging locally with telemetrics-client
+-----------------------------------------------
+
+The function of the telemetrics-client is to handle the transport of information
+reported by a probe to a backend (see ```server``` key in configuration). This
+information is helpful for developers to debug and fix reported crashes, however
+developers not always have access to the backend in these case users can leverage
+features added for local debugging. The following is a list of steps to enable
+local debug:
+
+* *Enabling record retention*: this step configures telempostd to keep
+copies of telemetry records locally. To enable record retention set the value of
+```record_retention_enabled``` from ```false``` to ```true```. Optionally set
+```record_server_delivery_enabled`` to ```false``` to keep records local only.
+Remember to restart the client after configuration values are updated
+(```telemctl restart```).
+
+* *Creating a record*: run ```hprobe``` command to create a record for the purposes
+of this step by step guide. Once we have the record or records that you need to
+capture locally you can display the data.
+
+* *Displaying record metadata*: telempostd keeps metadata of any valid record,
+to display this data a new option to telemctl was added ```telemctl journal```.
+Assuming that the last record created was the record from previous step `hprobe` we
+can use `tail -n 1` to print the last created record only, i.e.
+
+```
+$ sudo telemctl journal | tail -n 1
+$ org.clearlinux/hello/world     Mon 2018-04-02 17:48:01 UTC a19a0d41ba16788881e274b19b8a1be4 5de9de8d5f3c6a7d445d75ba01cc3322 60c014cd-4693-40f1-b334-548cd932949b
+```
+
+The headers for the metadata (along with other information) can be printed using the
+```-V``` switch with ```telemctl journal``` command, i.e.
+
+```
+$ sudo telemctl journal -V | head -n 1
+$ Classification             Time stamp              Record ID                    Event ID                     Boot ID
+```
+
+* *Displaying record payload*: to print the content of a record payload you can use
+the ```-i``` (```--include_record``` long format) option to ```telemctl journal```
+command. To print the specific record you created you can use the option ```-r```
+(```--record_id``` long format) with the Record Id of the generated record, i.e.
+
+```
+$ sudo telemctl journal --record_id a19a0d41ba16788881e274b19b8a1be4 --include_record
+$ org.clearlinux/hello/world     Mon 2018-04-02 17:48:01 UTC a19a0d41ba16788881e274b19b8a1be4 5de9de8d5f3c6a7d445d75ba01cc3322 60c014cd-4693-40f1-b334-548cd932949b
+$ hello
+```
+
+## Security Disclosures
+
+To report a security issue or receive security advisories please follow procedures
+in this [link](https://01.org/security).

TODO

@@ -1,24 +0,0 @@
-Build
-=====
-- make sure our AM_CFLAGS do not break older compilers, e.g. compilers
-  that do not support -fstack-protector-strong
-
-Probes
-======
-- add pstore probe (Archana)
-- journal probe: detect kernel oopses (Patrick)
-- crash probe: add function/register value info to crash payloads (Patrick)
-- crash probe: if some symbols are missing from backtraces, attempt the
-  debuginfo lookup again after a certain timeout (Patrick)
-- crash probe: for non-debug builds, ensure that we are invoked by the kernel,
-  and write tests to verify that attempts to run the crash probe for non-debug
-  builds fail. (Patrick)
-
-Library
-=======
-- add API for structured payloads
-
-Other
-=====
-- set fixed size for system-wide core file ulimit, and afterward notify QA team
-  to update the BAT test that checks for the ulimit value.

build-aux/make/cflags.make

@@ -12,7 +12,10 @@ AM_CFLAGS = \
 	-Wunreachable-code \
 	-funsigned-char \
 	-fPIE \
-	-fPIC
+	-fPIC \
+	-fno-strict-overflow \
+	-fno-delete-null-pointer-checks \
+	-fwrapv
 
 AM_CPPFLAGS = \
 	-D_FORTIFY_SOURCE=2 \
@@ -24,8 +27,8 @@ AM_CPPFLAGS = \
 	-DABSTOPSRCDIR=\"$(abs_top_srcdir)\" \
 	-DDATADIR=\"$(datadir)\" \
 	-DLOCALSTATEDIR=\"$(localstatedir)\" \
-	-DKERNELOOPSDIR=\"$(localstatedir)/cache/telemetry/oops\" \
 	-DPSTOREDIR=\"$(localstatedir)/cache/telemetry/pstore\" \
-	-DTESTOOPSDIR=\"$(top_srcdir)/tests/oops_test_files\"
+	-DTESTOOPSDIR=\"$(top_srcdir)/tests/oops_test_files\" \
+	-DBACKEND_ADDR=\"$(BACKEND_ADDR)\"
 
 # vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

configure.ac

@@ -2,13 +2,13 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([telemetrics-client], [1.12.4], [https://clearlinux.org/])
+AC_INIT([telemetrics-client], [2.2.3], [https://clearlinux.org/])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([1.14 -Wall -Werror -Wno-extra-portability foreign subdir-objects])
 AM_SILENT_RULES([yes])
 LT_INIT([disable-static])
 AC_CONFIG_MACRO_DIRS([build-aux/m4])
-AC_CONFIG_SRCDIR([src/main.c])
+AC_CONFIG_SRCDIR([src/probe.c])
 AC_CONFIG_HEADERS([config.h])
 
 # Checks for programs.
@@ -23,7 +23,7 @@ AC_PROG_MAKE_SET
 # Checks for libraries.
 
 # check >= 0.9.12 is required for TAP output
-PKG_CHECK_MODULES([CHECK], [check >= 0.9.12])
+PKG_CHECK_MODULES([CHECK], [check >= 0.12])
 PKG_CHECK_MODULES([CURL], [libcurl])
 AC_CHECK_LIB([elf], [elf_begin], [have_elflib=yes], [AC_MSG_ERROR([Unable to find libelf from elfutils])])
 AC_CHECK_LIB([dw], [dwfl_begin], [have_dwlib=yes], [AC_MSG_ERROR([Unable to find libdw from elfutils])])
@@ -98,6 +98,11 @@ AC_ARG_WITH([systemdsysctldir], AS_HELP_STRING([--with-systemdsysctldir=DIR],
 test -z "${sysctlpath}" && sysctlpath=/usr/lib/sysctl.d
 AC_SUBST(SYSTEMD_SYSCTLDIR, [${sysctlpath}])
 
+AC_ARG_WITH([systemdsystemctldir], AS_HELP_STRING([--with-systemdsystemctldir=DIR],
+            [path to systemd systemctl dir @<:@default=/usr/bin@:>@]), [systemctlpath=${withval}])
+test -z "${systemctlpath}" && systemctlpath=/usr/bin
+AC_SUBST(SYSTEMD_SYSTEMCTLDIR, [${systemctlpath}])
+
 # systemd.pc gives the /etc conf dir, so we can't use PKG_CONFIG for this one
 AC_ARG_WITH([systemdsystemconfdir], AS_HELP_STRING([--with-systemdsystemconfdir=DIR],
 	    [path to systemd system conf dir @<:@default=/usr/lib/systemd/system.conf.d@:>@]), [confpath=${withval}],
@@ -114,6 +119,11 @@ AC_ARG_WITH([loglevel], AS_HELP_STRING([--with-loglevel=NUM],
 	    [loglevel=5])
 AC_DEFINE_UNQUOTED([MAX_LOG_LEVEL], [${loglevel}], [Maximum log level for binaries])
 
+AC_ARG_WITH([backendserveraddr], AS_HELP_STRING([--with-backendserveraddr=URI],
+	    [uri to telemetrics backend server @<:@default=https://clr.telemetry.intel.com/v2/collector@:>@]), [backendaddr=${withval}])
+test -z "${backendaddr}" && backendaddr=https://clr.telemetry.intel.com/v2/collector
+AC_SUBST(BACKEND_ADDR, [${backendaddr}])
+
 AC_ARG_ENABLE([logtype], AS_HELP_STRING([--enable-logtype],
               [Vector for logging: stderr (default), syslog, systemd]),
 			  [case ${enableval} in

docs/README.md

@@ -1,9 +1,9 @@
 
 ## Notes on manual pages
 
-The manual pages are generated using `ronn(1)`. To recreate them,
-run `make manpages` in the toplevel  folder. If you want to edit
-the documentation for upstream changes, make sure to edit the `*.md`
+The manual pages are generated using `rst2man.py`. To recreate them,
+run `make manpages` in the toplevel folder. If you want to edit
+the documentation for upstream changes, make sure to edit the `*.rst`
 files and not the shipped nroff output files.
 
 

docs/local.mk

@@ -1,6 +1,7 @@
 MANPAGES = \
 	docs/man/telemctl.1 \
-	docs/man/telemd.1 \
+	docs/man/telemprobd.1 \
+	docs/man/telempostd.1 \
 	docs/man/telem-record-gen.1 \
 	docs/man/telemetry.3 \
 	docs/man/telemetrics.conf.5

docs/man/telem-record-gen.1

@@ -35,10 +35,10 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 \fBtelem\-record\-gen\fP <options>
 .SH DESCRIPTION
 .sp
-Generate and attempt to send a telemetry record to \fBtelemd\fP(1). This
+Generate and attempt to send a telemetry record to \fBtelemprobd\fP(1). This
 program can be used to create a telemetry record from a shell script
 or from another program using e.g. \fBsystem()\fP\&. The program connects
-to a local \fBtelemd\fP(1) daemon to spool the record for delivery to
+to a local \fBtelemprobd\fP(1) daemon to spool the record for delivery to
 the actual collection service.
 .sp
 The payload can be provided as standard input, as a file with the
@@ -77,6 +77,9 @@ File to read payload from.
 .IP \(bu 2
 \fB\-R\fP, \fB\-\-record\-version\fP <version>:
 Version number for format of payload (default 1).
+.IP \(bu 2
+\fB\-e\fP, \fB\-\-event\-id\fP <version>:
+Event id to use in the record. If not provided a randomly generated id will be assigned to record.
 .UNINDENT
 .UNINDENT
 .UNINDENT
@@ -86,7 +89,7 @@ Version number for format of payload (default 1).
 .SH SEE ALSO
 .INDENT 0.0
 .IP \(bu 2
-\fBtelemd\fP(1)
+\fBtelemprobd\fP(1)
 .IP \(bu 2
 \fI\%https://github.com/clearlinux/telemetrics\-client\fP
 .IP \(bu 2

docs/man/telem-record-gen.1.rst

@@ -19,10 +19,10 @@ SYNOPSIS
 DESCRIPTION
 ===========
 
-Generate and attempt to send a telemetry record to ``telemd``\(1). This
+Generate and attempt to send a telemetry record to ``telemprobd``\(1). This
 program can be used to create a telemetry record from a shell script
 or from another program using e.g. ``system()``. The program connects
-to a local ``telemd``\(1) daemon to spool the record for delivery to
+to a local ``telemprobd``\(1) daemon to spool the record for delivery to
 the actual collection service.
 
 The payload can be provided as standard input, as a file with the
@@ -62,6 +62,10 @@ OPTIONS
  * ``-R``, ``--record-version`` <version>:
    Version number for format of payload (default 1).
 
+ * ``-e``, ``--event-id`` <version>:
+   Event id to use in the record. If not provided a randomly generated id will be assigned to record.
+
+
 
 RETURN VALUES
 =============
@@ -72,7 +76,6 @@ RETURN VALUES
 SEE ALSO
 ========
 
-* ``telemd``\(1)
+* ``telemprobd``\(1)
 * https://github.com/clearlinux/telemetrics-client
 * https://clearlinux.org/documentation/
-

docs/man/telemctl.1

@@ -34,12 +34,11 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .sp
 \fBtelemctl\fP
 .sp
-\fB/etc/telemetrics/opt\-out\fP
+\fB/etc/telemetrics/opt\-in\fP
 .SH DESCRIPTION
 .sp
 Control actions for telemetry services. The command can be used to start,
-restart, or stop \fBtelemd\fP(1), or to opt\-in or opt\-out of telemetry delivery
-of records to a central telemetry service.
+restart, or stop \fBtelemprobd\fP(1) and \fBtelempostd\fP(1), or to opt\-in or opt\-out of telemetry delivery of records to a central telemetry service.
 .SH OPTIONS
 .INDENT 0.0
 .INDENT 3.5
@@ -49,12 +48,16 @@ of records to a central telemetry service.
 Starts, stops or restarts all running telemetry services.
 .IP \(bu 2
 \fBopt\-in\fP:
-Opts in to telemetry, and starts telemetry services. The opt\-out file
-\fB/etc/telemetrics/opt\-out\fP is removed.
+Opts in to telemetry and the opt\-in file \fB/etc/telemetrics/opt\-in\fP
+is created. Note: this is a one time required operation before
+telemetry can be used the first time.
 .IP \(bu 2
 \fBopt\-out\fP:
-Opts out of telemetry, and stops telemetry services. The opt\-out file
-\fB/etc/telemetrics/opt\-out\fP is created.
+Opts out of telemetry, and stops telemetry services. The opt\-in file
+\fB/etc/telemetrics/opt\-in\fP is deleted.
+.IP \(bu 2
+\fBis\-active\fP:
+Checks if telemetry client daemons are active (telemprobd and telempostd).
 .UNINDENT
 .UNINDENT
 .UNINDENT
@@ -64,7 +67,7 @@ Opts out of telemetry, and stops telemetry services. The opt\-out file
 .SH SEE ALSO
 .INDENT 0.0
 .IP \(bu 2
-\fBtelemd\fP(1)
+\fBtelemprobd\fP(1)
 .IP \(bu 2
 \fI\%https://github.com/clearlinux/telemetrics\-client\fP
 .IP \(bu 2

docs/man/telemctl.1.rst

@@ -15,15 +15,14 @@ SYNOPSIS
 
 ``telemctl``
 
-``/etc/telemetrics/opt-out``
+``/etc/telemetrics/opt-in``
 
 
 DESCRIPTION
 ===========
 
 Control actions for telemetry services. The command can be used to start,
-restart, or stop ``telemd``\(1), or to opt-in or opt-out of telemetry delivery
-of records to a central telemetry service.
+restart, or stop ``telemprobd``\(1) and ``telempostd``\(1), or to opt-in or opt-out of telemetry delivery of records to a central telemetry service.
 
 
 OPTIONS
@@ -32,13 +31,17 @@ OPTIONS
  * ``start``|``stop``|``restart``:
    Starts, stops or restarts all running telemetry services.
 
- * ``opt-in``: 
-   Opts in to telemetry, and starts telemetry services. The opt-out file
-   ``/etc/telemetrics/opt-out`` is removed.
+ * ``opt-in``:
+   Opts in to telemetry and the opt-in file ``/etc/telemetrics/opt-in``
+   is created. Note: this is a one time required operation before
+   telemetry can be used the first time.
 
- * ``opt-out``: 
-   Opts out of telemetry, and stops telemetry services. The opt-out file
-   ``/etc/telemetrics/opt-out`` is created.
+ * ``opt-out``:
+   Opts out of telemetry, and stops telemetry services. The opt-in file
+   ``/etc/telemetrics/opt-in`` is deleted.
+
+ * ``is-active``:
+   Checks if telemetry client daemons are active (telemprobd and telempostd).
 
 
 RETURN VALUES
@@ -50,7 +53,6 @@ RETURN VALUES
 SEE ALSO
 ========
 
-* ``telemd``\(1)
+* ``telemprobd``\(1)
 * https://github.com/clearlinux/telemetrics-client
 * https://clearlinux.org/documentation/
-

docs/man/telemetrics.conf.5

@@ -37,8 +37,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 \fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
 .SH DESCRIPTION
 .sp
-This file contains configuration parameters for the \fBtelemd\fP(1) telemetry
-service daemon. The daemon reads this file at startup if it exists.
+This file contains configuration parameters for the \fBtelemprobd\fP(1) and \fBtelempostd\fP(1) telemetry service daemons. The daemon reads this file at startup if it exists.
 .SH SYNTAX
 .sp
 The configuration file contains \fBkey=value\fP pairs, formatted as plain
@@ -54,7 +53,7 @@ Server URL including protocol designator.
 .IP \(bu 2
 \fBsocket_path=<path>\fP
 .sp
-Path to the socket that \fItelemd\fP will listen on.
+Path to the socket that \fItelemprobd\fP will listen on.
 .IP \(bu 2
 \fBcainfo=<path>\fP
 .sp
@@ -82,7 +81,7 @@ and not the actual size of the record itself.
 .IP \(bu 2
 \fBspool_process_time=<seconds>\fP
 .sp
-Time in seconds for processing spool. Valid range: 120..3600. Values
+Time in seconds for processing spool. Valid range: 120..300. Values
 outside this range are clamped.
 .IP \(bu 2
 \fBrate_limit_enabled=<true|false>\fP
@@ -114,7 +113,9 @@ delivery over network. Valid stategies: \fBspool\fP, \fBdrop\fP\&.
 .SH SEE ALSO
 .INDENT 0.0
 .IP \(bu 2
-\fBtelemd\fP(1)
+\fBtelemprobd\fP(1)
+.IP \(bu 2
+\fBtelempostd\fP(1)
 .IP \(bu 2
 \fI\%https://github.com/clearlinux/telemetrics\-client\fP
 .IP \(bu 2

docs/man/telemetrics.conf.5.rst

@@ -21,8 +21,7 @@ SYNOPSIS
 DESCRIPTION
 ===========
 
-This file contains configuration parameters for the ``telemd``\(1) telemetry
-service daemon. The daemon reads this file at startup if it exists.
+This file contains configuration parameters for the ``telemprobd``\(1) and ``telempostd``\(1) telemetry service daemons. The daemon reads this file at startup if it exists.
 
 
 SYNTAX
@@ -43,7 +42,7 @@ OPTIONS
 
 -  ``socket_path=<path>``
 
-   Path to the socket that `telemd` will listen on.
+   Path to the socket that `telemprobd` will listen on.
 
 -  ``cainfo=<path>``
 
@@ -71,7 +70,7 @@ OPTIONS
 
 -  ``spool_process_time=<seconds>``
 
-   Time in seconds for processing spool. Valid range: 120..3600. Values
+   Time in seconds for processing spool. Valid range: 120..300. Values
    outside this range are clamped.
 
 -  ``rate_limit_enabled=<true|false>``
@@ -104,7 +103,7 @@ OPTIONS
 SEE ALSO
 ========
 
-* ``telemd``\(1)
+* ``telemprobd``\(1)
+* ``telempostd``\(1)
 * https://github.com/clearlinux/telemetrics-client
 * https://clearlinux.org/documentation/
-

docs/man/telemetry.3

@@ -44,11 +44,13 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .sp
 \fBvoid tm_free_record(struct telem_ref *t_ref)\fP
 .sp
-\fBvoid tm_set_config_file(char *c_file)\fP
+\fBint tm_set_config_file(const char *c_file)\fP
+.sp
+\fBint tm_is_opted_in(void)\fP
 .SH DESCRIPTION
 .sp
 The functions in the telemetry library facilitate the delivery of
-telemetry data to the \fBtelemd\fP(1) service.
+telemetry data to the \fBtelemprobd\fP(1) service.
 .sp
 The function \fBtm_create_record()\fP initializes a telemetry record and
 sets the severity and classification of that record, as well as the
@@ -60,19 +62,23 @@ The function \fBtm_set_payload()\fP attaches the provided telemetry record
 data to the telemetry record. The current maximum payload size is 8192b.
 .sp
 The function \fBtm_send_record()\fP delivers the record to the local
-\fBtelemd\fP(1) service.
+\fBtelemprobd\fP(1) service.
 .sp
 The function \fBtm_set_config_file()\fP can be used to provide an alternate
 configuration path to the telemetry library.
+.sp
+\fBtm_is_opted_in\fP is a utility provided to check if the one time opt\-in
+has been performed.
 .SH RETURN VALUES
 .sp
 All these functions return \fB0\fP on success, or a non\-zero return value
-if an error occurred. The functions \fBtm_free_record()\fP and \fBtm_set_config_file()\fP
-do not return any values.
+if an error occurred. The function \fBtm_free_record()\fP does not return
+any value. \fBtm_is_opted_in\fP returns \fB1\fP when telemetry is opted\-in
+otherwise \fB0\fP\&.
 .SH SEE ALSO
 .INDENT 0.0
 .IP \(bu 2
-\fBtelemd\fP(1)
+\fBtelemprobd\fP(1)
 .IP \(bu 2
 \fI\%https://github.com/clearlinux/telemetrics\-client\fP
 .IP \(bu 2

docs/man/telemetry.3.rst

@@ -14,9 +14,9 @@ SYNOPSIS
 ========
 
 ``#include "telemetry.h"``
-  
+
 ``struct telem_ref { struct telem_record *record; };``
-  
+
 ``int tm_create_record(struct telem_ref **t_ref, uint32_t severity, char *classification, uint32_t payload_version)``
 
 ``int tm_set_payload(struct telem_ref *t_ref, char *payload)``
@@ -25,14 +25,16 @@ SYNOPSIS
 
 ``void tm_free_record(struct telem_ref *t_ref)``
 
-``void tm_set_config_file(char *c_file)``
+``int tm_set_config_file(const char *c_file)``
+
+``int tm_is_opted_in(void)``
 
 
 DESCRIPTION
 ===========
 
 The functions in the telemetry library facilitate the delivery of
-telemetry data to the ``telemd``\(1) service.
+telemetry data to the ``telemprobd``\(1) service.
 
 The function ``tm_create_record()`` initializes a telemetry record and
 sets the severity and classification of that record, as well as the
@@ -44,24 +46,26 @@ The function ``tm_set_payload()`` attaches the provided telemetry record
 data to the telemetry record. The current maximum payload size is 8192b.
 
 The function ``tm_send_record()`` delivers the record to the local
-``telemd``\(1) service.
+``telemprobd``\(1) service.
 
 The function ``tm_set_config_file()`` can be used to provide an alternate
 configuration path to the telemetry library.
 
+``tm_is_opted_in`` is a utility provided to check if the one time opt-in
+has been performed.
 
 RETURN VALUES
 =============
 
 All these functions return ``0`` on success, or a non-zero return value
-if an error occurred. The functions ``tm_free_record()`` and ``tm_set_config_file()``
-do not return any values.
+if an error occurred. The function ``tm_free_record()`` does not return
+any value. ``tm_is_opted_in`` returns ``1`` when telemetry is opted-in
+otherwise ``0``.
 
 
 SEE ALSO
 ========
 
-* ``telemd``\(1)
+* ``telemprobd``\(1)
 * https://github.com/clearlinux/telemetrics-client
 * https://clearlinux.org/documentation/
-

docs/man/telempostd.1

@@ -0,0 +1,97 @@
+.\" Man page generated from reStructuredText.
+.
+.TH TELEMPOSTD 1 "" "" ""
+.SH NAME
+telempostd \- Telemetry client service
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelempostd\fP <flags>
+.sp
+\fB/etc/telemetrics/telemetrics.conf\fP
+.sp
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.sp
+\fB/etc/telemetrics/opt\-in\-static\-machine\-id\fP
+.SH DESCRIPTION
+.sp
+The \fBtelempostd\fP program delivers locally generated telemetry records to a remote
+telemetry service. Telemetry data can be in any format, and is relayed as\-is.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fB\-f\fP, \fB\-\-config_file\fP [<file>]:
+Configuration file. This overrides the other parameters.
+.IP \(bu 2
+\fB\-h\fP, \fB\-\-help\fP:
+Display this help message.
+.IP \(bu 2
+\fB\-V\fP, \fB\-\-version\fP:
+Print the program version.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH FILES
+.INDENT 0.0
+.IP \(bu 2
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+If no custom configuration file is found, \fBtelempostd\fP uses the
+settings in this file.
+.UNINDENT
+.UNINDENT
+.IP \(bu 2
+\fB/etc/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+Custom configuration file that \fBtelempostd\fP reads. See \fBtelemetrics.conf\fP(5).
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH EXIT STATUS
+.sp
+0 when no errors occurred. A non\-zero exit status indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemetry\fP(3)
+.IP \(bu 2
+\fBtelemetrics.conf\fP(5)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2018 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
+.

docs/man/telempostd.1.rst

@@ -0,0 +1,71 @@
+==========
+telempostd
+==========
+
+------------------------
+Telemetry client service
+------------------------
+
+:Copyright: \(C) 2018 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telempostd`` \<flags\>
+
+``/etc/telemetrics/telemetrics.conf``
+
+``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+``/etc/telemetrics/opt-in-static-machine-id``
+
+
+DESCRIPTION
+===========
+
+The ``telempostd`` program delivers locally generated telemetry records to a remote
+telemetry service. Telemetry data can be in any format, and is relayed as-is.
+
+
+OPTIONS
+=======
+
+  * ``-f``, ``--config_file`` \[\<file\>\]:
+    Configuration file. This overrides the other parameters.
+
+  * ``-h``, ``--help``:
+    Display this help message.
+
+  * ``-V``, ``--version``:
+    Print the program version.
+
+
+FILES
+=====
+
+* ``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+    If no custom configuration file is found, ``telempostd`` uses the
+    settings in this file.
+
+* ``/etc/telemetrics/telemetrics.conf``
+
+    Custom configuration file that ``telempostd`` reads. See ``telemetrics.conf``\(5).
+
+
+EXIT STATUS
+===========
+
+0 when no errors occurred. A non-zero exit status indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemetry``\(3)
+* ``telemetrics.conf``\(5)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

docs/man/telemprobd.1

@@ -1,8 +1,8 @@
 .\" Man page generated from reStructuredText.
 .
-.TH TELEMD 1 "" "" ""
+.TH TELEMPROBD 1 "" "" ""
 .SH NAME
-telemd \- Telemetry client service
+telemprobd \- Telemetry probes service
 .
 .nr rst2man-indent-level 0
 .
@@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 ..
 .SH SYNOPSIS
 .sp
-\fBtelemd\fP <flags>
+\fBtelemprobd\fP <flags>
 .sp
 \fB/etc/telemetrics/telemetrics.conf\fP
 .sp
@@ -41,8 +41,8 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 \fB/etc/telemetrics/opt\-in\-static\-machine\-id\fP
 .SH DESCRIPTION
 .sp
-The \fBtelemd\fP program delivers locally generated telemetry records to a remote
-telemetry service. Telemetry data can be in any format, and is relayed as\-is.
+The \fBtelemprobd\fP program handles communication between telemetry client and telemetry
+probes.
 .SH OPTIONS
 .INDENT 0.0
 .INDENT 3.5
@@ -65,7 +65,7 @@ Print the program version.
 \fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
 .INDENT 2.0
 .INDENT 3.5
-If no custom configuration file is found, \fBtelemd\fP uses the
+If no custom configuration file is found, \fBtelemprobd\fP uses the
 settings in this file.
 .UNINDENT
 .UNINDENT
@@ -73,7 +73,7 @@ settings in this file.
 \fB/etc/telemetrics/telemetrics.conf\fP
 .INDENT 2.0
 .INDENT 3.5
-Custom configuration file that \fBtelemd\fP reads. See \fBtelemetrics.conf\fP(5).
+Custom configuration file that \fBtelemprobd\fP reads. See \fBtelemetrics.conf\fP(5).
 .UNINDENT
 .UNINDENT
 .IP \(bu 2

docs/man/telemprobd.1.rst

@@ -1,9 +1,9 @@
-======
-telemd
-======
+==========
+telemprobd
+==========
 
 ------------------------
-Telemetry client service
+Telemetry probes service
 ------------------------
 
 :Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
@@ -13,7 +13,7 @@ Telemetry client service
 SYNOPSIS
 ========
 
-``telemd`` \<flags\>
+``telemprobd`` \<flags\>
 
 ``/etc/telemetrics/telemetrics.conf``
 
@@ -25,8 +25,8 @@ SYNOPSIS
 DESCRIPTION
 ===========
 
-The ``telemd`` program delivers locally generated telemetry records to a remote
-telemetry service. Telemetry data can be in any format, and is relayed as-is.
+The ``telemprobd`` program handles communication between telemetry client and telemetry
+probes.
 
 
 OPTIONS
@@ -47,12 +47,12 @@ FILES
 
 * ``/usr/share/defaults/telemetrics/telemetrics.conf``
 
-    If no custom configuration file is found, ``telemd`` uses the
+    If no custom configuration file is found, ``telemprobd`` uses the
     settings in this file.
 
 * ``/etc/telemetrics/telemetrics.conf``
 
-    Custom configuration file that ``telemd`` reads. See ``telemetrics.conf``\(5).
+    Custom configuration file that ``telemprobd`` reads. See ``telemetrics.conf``\(5).
 
 
 * ``/etc/telemetrics/opt-in-static-machine-id``

scripts/local.mk

@@ -1,3 +0,0 @@
-dist_bin_SCRIPTS = %D%/telemctl
-
-# vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

scripts/telemctl

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-declare -a SPECIAL_UNITS=(
-  hprobe.timer
-  telemd.socket
-  telemd.path
-)
-
-declare -a SERVICES=(
-  hprobe.service
-  journal-probe.service
-  oops-probe.service
-  pstore-probe.service
-  klogscanner.service
-  telemd.service
-)
-
-SCRIPT="$0"
-TELEM_DIR=/etc/telemetrics
-OPT_OUT_FILE=${TELEM_DIR}/opt-out
-
-exit_ok() {
-  echo "$1" > /dev/stderr
-  exit 0
-}
-
-exit_err() {
-  echo "$1" > /dev/stderr
-  exit 1
-}
-
-notice() {
-  echo "$1" > /dev/stderr
-}
-
-for_each_service() {
-  local action=$1 && shift
-  local -a array=($*)
-  for service in "${array[@]}"; do
-    systemctl $action $service
-    [ $? -ne 0 ] && notice "Failed to $action ${service}. Continuing..."
-  done
-}
-
-telem_stop() {
-  # the special units must be stopped first so that activation no longer happens
-  for_each_service "stop" ${SPECIAL_UNITS[@]}
-  for_each_service "stop" ${SERVICES[@]}
-}
-
-telem_start() {
-  [ -f $OPT_OUT_FILE ] && exit_err "Opt out is enabled. Cannot start services."
-  # the units in SERVICES are activated as needed, so no need to start them
-  for_each_service "start" ${SPECIAL_UNITS[@]}
-}
-
-telem_opt_out() {
-  [ -f $OPT_OUT_FILE ] && exit_ok "Already opted out. Nothing to do."
-  mkdir -p $TELEM_DIR || exit_err "Failed to create ${TELEM_DIR}."
-  touch $OPT_OUT_FILE || exit_err "Failed to create ${OPT_OUT_FILE}."
-  telem_stop
-}
-
-telem_opt_in() {
-  [ ! -f $OPT_OUT_FILE ] && exit_ok "Already opted in. Nothing to do."
-  rm -f $OPT_OUT_FILE || exit_err "Failed to remove ${OPT_OUT_FILE}."
-  telem_start
-}
-
-telem_restart() {
-  telem_stop
-  telem_start
-}
-
-usage() {
-  printf "$SCRIPT - Control actions for telemetry services\n"
-  printf "\n"
-  printf "%15s   %-65s\n" "stop" "Stops all running telemetry services"
-  printf "%15s   %-65s\n" "start" "Starts all telemetry services"
-  printf "%15s   %-65s\n" "restart" "Restarts all telemetry services"
-  printf "%15s   %-65s\n" "opt-in" "Opts in to telemetry, and starts telemetry services"
-  printf "%15s   %-65s\n" "opt-out" "Opts out of telemetry, and stops telemetry services"
-  printf "\n"
-  exit 2
-}
-
-if [ $# -ne 1 ]; then
-  usage
-fi
-
-if [ $EUID -ne 0 ]; then
-  exit_err "Must be root to run this command. Exiting..."
-fi
-
-SUBCOMMAND=$1
-
-case $SUBCOMMAND in
-  opt-out)
-    telem_opt_out ;;
-  opt-in)
-    telem_opt_in ;;
-  stop)
-    telem_stop ;;
-  start)
-    telem_start ;;
-  restart)
-    telem_restart ;;
-  *)
-    notice "Unknown command passed to $SCRIPT"
-    usage ;;
-esac
-
-exit 0
-
-# vi: ts=8 sw=2 sts=2 et tw=80

src/common.c

@@ -33,7 +33,11 @@ static const char *header_names[] = {
         TM_SYSTEM_BUILD_STR,
         TM_KERNEL_VERSION_STR,
         TM_PAYLOAD_VERSION_STR,
-        TM_SYSTEM_NAME_STR
+        TM_SYSTEM_NAME_STR,
+        TM_BOARD_NAME_STR,
+        TM_CPU_MODEL_STR,
+        TM_BIOS_VERSION_STR,
+        TM_EVENT_ID_STR
 };
 
 const char *get_header_name(int ind)

src/common.h

@@ -35,6 +35,10 @@
 #define TM_KERNEL_VERSION 8
 #define TM_PAYLOAD_VERSION 9
 #define TM_SYSTEM_NAME 10
+#define TM_BOARD_NAME 11
+#define TM_CPU_MODEL 12
+#define TM_BIOS_VERSION 13
+#define TM_EVENT_ID 14
 
 #define TM_RECORD_VERSION_STR "record_format_version"
 #define TM_CLASSIFICATION_STR "classification"
@@ -47,22 +51,34 @@
 #define TM_KERNEL_VERSION_STR "kernel_version"
 #define TM_PAYLOAD_VERSION_STR "payload_format_version"
 #define TM_SYSTEM_NAME_STR "system_name"
+#define TM_BOARD_NAME_STR "board_name"
+#define TM_CPU_MODEL_STR "cpu_model"
+#define TM_BIOS_VERSION_STR "bios_version"
+#define TM_EVENT_ID_STR "event_id"
 
-#define NUM_HEADERS 11
+#define NUM_HEADERS 15
+
+#define EVENT_ID_ALPHAB "0123456789abcdef"
+#define EVENT_ID_LEN 32
+
+/* Journal */
+#define JOURNAL_PATH "/var/log/telemetry/journal"
+#define JOURNAL_TMPDIR  "/var/log/telemetry/"
+#define RECORD_RETENTION_DIR "/var/log/telemetry/records"
 
 /* For internal library usage. Bump the version whenever we change the record
  * structure (e.g. adding or removing a header field). Note that the value
  * should be an unsigned int.
  */
-static const uint32_t RECORD_FORMAT_VERSION = 2;
+static const uint32_t RECORD_FORMAT_VERSION = 4;
 
 #define TM_SITE_VERSION_FILE "/etc/os-release"
 #define TM_DIST_VERSION_FILE "/usr/lib/os-release"
 
-#define TM_OPT_OUT_FILE "/etc/telemetrics/opt-out"
+#define TM_OPT_IN_FILE "/etc/telemetrics/opt-in"
 
 /* Currently max supported payload size is 8kb */
-#define MAX_PAYLOAD_SIZE 8192
+#define MAX_PAYLOAD_LENGTH 8192
 
 /* Maximum size for a category string is 122 bytes */
 #define MAX_CLASS_LENGTH 122
@@ -70,12 +86,12 @@ static const uint32_t RECORD_FORMAT_VERSION = 2;
 /* Maximum size for a sub category 40 bytes */
 #define MAX_SUBCAT_LENGTH 40
 
-/* Max Payload size is 8 kibabytes (8192) */
-#define MAX_PAYLOAD_LENGTH 8192
+/* Exit daemon after idle for this duration in seconds */
+#define TM_DAEMON_EXIT_TIME  (5 * 60)
 
 /* SPOOL DEFINES */
-/* Spooling should run every TM_SPOOL_RUN_MAX atleast */
-#define TM_SPOOL_RUN_MAX (60 /*min*/ * 60 /*sec*/)
+/* Spooling should run at least every TM_SPOOL_RUN_MAX */
+#define TM_SPOOL_RUN_MAX TM_DAEMON_EXIT_TIME
 
 /* Spooling should not run more often than TM_SPOOL_RUN_MIN */
 #define TM_SPOOL_RUN_MIN (2 /*min*/ * 60 /*sec*/)
@@ -86,8 +102,10 @@ static const uint32_t RECORD_FORMAT_VERSION = 2;
 /* Maximum records that can be processed in a single spool run loop*/
 #define TM_SPOOL_MAX_PROCESS_RECORDS 60
 
-/* Time to check to exit from  the daemon */
-#define TM_DAEMON_EXIT_TIME  (2 /*h*/ * 60 /*m*/ * 60 /*sec*/)
+/* Definitions for config file override */
+#define CFG_PREFIX        "CFG:"
+#define CFG_PREFIX_LENGTH 4
+#define CFG_PREFIX_32BIT  0x3a474643
 
 /* Very simple structure. Array of header strings and a payload. Calling
  * program is reponsible for passing in the payload as a simple string.
@@ -101,4 +119,5 @@ struct telem_record {
 
 const char *get_header_name(int ind);
 
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/configuration.c

@@ -28,6 +28,7 @@
 #include "configuration.h"
 #include "util.h"
 #include "common.h"
+#include "log.h"
 
 #include "nica/inifile.h"
 
@@ -38,21 +39,50 @@ static NcHashmap *keyfile = NULL;
 static bool cmd_line_cfg = false;
 
 /* Conf strings, integers, and booleans expected in the conf file */
-const char *config_key_str[] = { NULL, "server", "socket_path", "spool_dir",
-                                 "rate_limit_strategy", "cainfo",
-                                 "tidheader", NULL };
+static const char *config_key_str[] = { "server",
+                                        "socket_path",
+                                        "spool_dir",
+                                        "rate_limit_strategy",
+                                        "cainfo",
+                                        "tidheader"};
 
-const char *config_key_int[] = { NULL, "record_expiry", "spool_max_size",
-                                 "spool_process_time", "record_window_length",
-                                 "byte_window_length", "record_burst_limit",
-                                 "byte_burst_limit", NULL };
+static const char *config_key_int[] = { "record_expiry",
+                                        "spool_max_size",
+                                        "spool_process_time",
+                                        "record_window_length",
+                                        "byte_window_length",
+                                        "record_burst_limit",
+                                        "byte_burst_limit" };
+
+static const char *config_key_bool[] = { "rate_limit_enabled",
+                                         "daemon_recycling_enabled",
+                                         "record_retention_enabled",
+                                         "record_server_delivery_enabled" };
+
+static const char *config_str_default[] = { DEFAULT_SERVER_ADDR,
+                                            DEFAULT_SOCKET_PATH,
+                                            DEFAULT_SPOOL_DIR,
+                                            DEFAULT_RATE_LIMIT_STRATEGY,
+                                            DEFAULT_CAINFO,
+                                            DEFAULT_TIDHEADER };
+
+static const bool config_bool_default[] = { DEFAULT_RATE_LIMIT_ENABLED,
+                                            DEFAULT_DAEMON_RECYCLING_ENABLED,
+                                            DEFAULT_RECORD_RETENTION_ENABLED,
+                                            DEFAULT_RECORD_SERVER_DELIVERY_ENABLED };
+
+static const int config_int_default[] = { DEFAULT_RECORD_EXPIRY,
+                                          DEFAULT_SPOOL_MAX_SIZE,
+                                          DEFAULT_SPOOL_PROCESS_TIME,
+                                          DEFAULT_RECORD_WINDOW_LENGTH,
+                                          DEFAULT_BYTE_WINDOW_LENGTH,
+                                          DEFAULT_RECORD_BURST_LIMIT,
+                                          DEFAULT_BYTE_BURST_LIMIT };
 
-const char *config_key_bool[] = { NULL, "rate_limit_enabled",
-                                  "daemon_recycling_enabled", NULL };
 
 static struct configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
 
-int validate_config_file(char *f)
+static int validate_config_file(const char *f)
 {
         struct stat sbuf;
 
@@ -60,26 +90,66 @@ int validate_config_file(char *f)
                 return -EINVAL;
         }
 
+        /* check if the file exists and is a regular file */
         if (stat(f, &sbuf) != 0) {
                 return -errno;
         }
 
+        if (!S_ISREG(sbuf.st_mode)) {
+                return -EINVAL;
+        }
+
         return 0;
 
 }
 
-void set_config_file(char *filename)
+const char *get_config_file(void)
 {
+        return config_file;
+}
 
-        /* TODO: This should return a status, but that will require coordinating
-         * an API change.  For now, config-entries referencing a null config
-         * file will fail, so the end result still works. */
+const char *get_cmd_line_config_file(void)
+{
+        if (cmd_line_cfg == true) {
+                return config_file;
+        }
+        return NULL;
+}
 
-        if (validate_config_file(filename) == 0) {
-                config_file = strdup(filename);
-                cmd_line_cfg = true;
+int set_config_file(const char *filename)
+{
+        int ret;
+
+        if ((ret = validate_config_file(filename)) == 0) {
+                char *filename1 = strdup(filename);
+                if (filename1 == NULL) {
+                    ret = -errno;
+                } else {
+                    config_file = filename1;
+                    cmd_line_cfg = true;
+                }
         }
 
+        return ret;
+}
+
+bool set_default_config_values(struct configuration *config)
+{
+        for (int i = 0; i < CONF_STR_MAX; i++) {
+                config->strValues[i] = strdup(config_str_default[i]);
+                        if (config->strValues[i] == NULL) {
+                                telem_log(LOG_ERR, "Could not set config item %s: %s\n",
+                                          config_key_str[i], strerror(errno));
+                                return false;
+                        }
+        }
+        for (int i = 0; i < CONF_INT_MAX; i++) {
+                config->intValues[i] = config_int_default[i];
+        }
+        for (int i = 0; i < CONF_BOOL_MAX; i++) {
+                config->boolValues[i] = config_bool_default[i];
+        }
+        return true;
 }
 
 bool read_config_from_file(char *config_file, struct configuration *config)
@@ -90,51 +160,64 @@ bool read_config_from_file(char *config_file, struct configuration *config)
 
         keyfile = nc_ini_file_parse(config_file);
         if (!keyfile) {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to read config file: %s\n",
-                        error->message);
-#endif
+                telem_log(LOG_ERR, "Failed to read config file\n");
                 return false;
         } else {
-                for (int i = CONF_STR_MIN + 1; i < CONF_STR_MAX; i++) {
-                        config->strValues[i] = strdup(nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_str[i]));
-                        if (config->strValues[i] == NULL) {
-                                return false;
+                for (int i = 0; i < CONF_STR_MAX; i++) {
+                        char *ptr;
+                        ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_str[i]);
+                        if (ptr) {
+                                config->strValues[i] = strdup(ptr);
+                                if (config->strValues[i] == NULL) {
+                                        telem_log(LOG_ERR, "Could not set config item %s: %s\n",
+                                                  config_key_str[i], strerror(errno));
+                                        return false;
+                                }
+                        } else {
+                                config->strValues[i] = strdup(config_str_default[i]);
                         }
+
                 }
-                for (int i = CONF_INT_MIN + 1; i < CONF_INT_MAX; i++) {
+
+                for (int i = 0; i < CONF_INT_MAX; i++) {
                         char *ptr;
                         ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_int[i]);
                         if (ptr) {
+                                errno = 0;
                                 config->intValues[i] = strtoll(ptr, NULL, 10);
+                                if (errno != 0) {
+                                        telem_log(LOG_ERR, "Error while parsing value of option %s: %s\n",
+                                                  config_key_int[i], strerror(errno));
+                                        return false;
+                                }
                         } else {
-                                return false;
+                                config->intValues[i] = config_int_default[i];
                         }
                 }
 
-                for (int i = CONF_BOOL_MIN + 1; i < CONF_BOOL_MAX; i++) {
+                for (int i = 0; i < CONF_BOOL_MAX; i++) {
                         char *ptr;
                         ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_bool[i]);
-                        config->boolValues[i] = false;
-
                         if (ptr) {
-                                if (strcasecmp(ptr, "TRUE") == 0) {
-                                        config->boolValues[i] = true;
-                                }
-                                if (strcasecmp(ptr, "1") == 0) {
+                                if ((strcasecmp(ptr, "TRUE") == 0) || (strcmp(ptr, "1") == 0)) {
                                         config->boolValues[i] = true;
+                                } else if ((strcasecmp(ptr, "FALSE") == 0) || (strcmp(ptr, "0") == 0)) {
+                                        config->boolValues[i] = false;
+                                } else {
+                                        telem_log(LOG_ERR, "Configuration item '%s' requires a boolean value\n",
+                                                  config_key_bool[i]);
+                                        return false;
                                 }
                         } else {
-                                return false;
+                                config->boolValues[i] = config_bool_default[i];
                         }
                 }
         }
 
-        config->initialized = true;
         return true;
 }
 
-void initialize_config(void)
+static void initialize_config(void)
 {
         if (config.initialized) {
                 return;
@@ -145,26 +228,25 @@ void initialize_config(void)
                 if (access(etc_config_file, R_OK) == 0) {
                         config_file = etc_config_file;
                 } else {
-                        if (access(default_config_file,
-                                   R_OK) == 0) {
+                        if (access(default_config_file, R_OK) == 0) {
                                 config_file = default_config_file;
-                        } else {
-                                /* If there is no default config, exit with failure */
-#ifdef DEBUG
-                                fprintf(stderr, "ERR: No configuration file"
-                                        " found, exiting\n");
-#endif
-                                exit(EXIT_FAILURE);
                         }
                 }
         }
-        if (!read_config_from_file(config_file, &config)) {
-                /* Error while parsing file  */
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Error while parsing configuration file\n");
-#endif
-                exit(EXIT_FAILURE);
+
+        if (config_file) {
+                if (!read_config_from_file(config_file, &config)) {
+                        /* Error while parsing file  */
+                        exit(EXIT_FAILURE);
+                }
         }
+        else {
+                if (!set_default_config_values(&config)) {
+                        exit(EXIT_FAILURE);
+                }
+        }
+
+        config.initialized = true;
 }
 
 void reload_config(void)
@@ -184,7 +266,7 @@ void free_configuration(void)
                 return;
         }
 
-        for (int i = CONF_STR_MIN + 1; i < CONF_STR_MAX; i++) {
+        for (int i = 0; i < CONF_STR_MAX; i++) {
                 free(config.strValues[i]);
         }
 
@@ -348,4 +430,16 @@ bool daemon_recycling_enabled_config(void)
         initialize_config();
         return config.boolValues[CONF_DAEMON_RECYCLING_ENABLED];
 }
+
+bool record_retention_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_RETENTION_ENABLED];
+}
+
+bool record_server_delivery_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED];
+}
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/configuration.h

@@ -18,11 +18,31 @@
 #include <stdbool.h>
 #include <stdint.h>
 
+/* Default configuration settings */
+#define DEFAULT_SERVER_ADDR BACKEND_ADDR
+#define DEFAULT_SOCKET_PATH "/run/telem-0"
+#define DEFAULT_SPOOL_DIR LOCALSTATEDIR "/spool/telemetry"
+#define DEFAULT_RATE_LIMIT_STRATEGY "spool"
+#define DEFAULT_CAINFO ""
+#define DEFAULT_TIDHEADER "X-Telemetry-TID: 6907c830-eed9-4ce9-81ae-76daf8d88f0f"
+
+#define DEFAULT_RECORD_EXPIRY 1200
+#define DEFAULT_SPOOL_MAX_SIZE 5120
+#define DEFAULT_SPOOL_PROCESS_TIME 120
+#define DEFAULT_RECORD_WINDOW_LENGTH 15
+#define DEFAULT_BYTE_WINDOW_LENGTH 20
+#define DEFAULT_RECORD_BURST_LIMIT 1000
+#define DEFAULT_BYTE_BURST_LIMIT -1
+
+#define DEFAULT_RATE_LIMIT_ENABLED true
+#define DEFAULT_DAEMON_RECYCLING_ENABLED true
+#define DEFAULT_RECORD_RETENTION_ENABLED false
+#define DEFAULT_RECORD_SERVER_DELIVERY_ENABLED true
+
 #define TM_MAX_WINDOW_LENGTH (1 /*h*/ * 60 /*m*/)
 
 enum config_str_keys {
-        CONF_STR_MIN = 0,
-        CONF_SERVER_ADDR,
+        CONF_SERVER_ADDR = 0,
         CONF_SOCKET_PATH,
         CONF_SPOOL_DIR,
         CONF_RATE_LIMIT_STRATEGY,
@@ -32,8 +52,7 @@ enum config_str_keys {
 };
 
 enum config_int_keys {
-        CONF_INT_MIN = 0,
-        CONF_RECORD_EXPIRY,
+        CONF_RECORD_EXPIRY = 0,
         CONF_SPOOL_MAX_SIZE,
         CONF_SPOOL_PROCESS_TIME,
         CONF_RECORD_WINDOW_LENGTH,
@@ -44,9 +63,10 @@ enum config_int_keys {
 };
 
 enum config_bool_keys {
-        CONF_BOOL_MIN = 0,
-        CONF_RATE_LIMIT_ENABLED,
+        CONF_RATE_LIMIT_ENABLED = 0,
         CONF_DAEMON_RECYCLING_ENABLED,
+        CONF_RECORD_RETENTION_ENABLED,
+        CONF_RECORD_SERVER_DELIVERY_ENABLED,
         CONF_BOOL_MAX
 };
 
@@ -59,7 +79,16 @@ typedef struct configuration {
 } configuration;
 
 /* Sets the configuration file to be used later */
-void set_config_file(char *filename);
+int set_config_file(const char *filename);
+
+/* Gets the configuration currently in use */
+const char *get_config_file(void);
+
+/* Gets the configuration specified via command line or NULL */
+const char *get_cmd_line_config_file(void);
+
+/* Sets all default configuration values to a given config */
+bool set_default_config_values(struct configuration *config);
 
 /* Parses the ini format config file */
 bool read_config_from_file(char *filename, struct configuration *config);
@@ -117,4 +146,10 @@ const char *get_tidheader_config(void);
 /* Gets whether recycling is enabled */
 bool daemon_recycling_enabled_config(void);
 
+/* Gets whether local retention of records is enabled */
+bool record_retention_enabled_config(void);
+
+/* Gets whether records should be sent to server_addr */
+bool record_server_delivery_enabled_config(void);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/data/bert-probe.service.in

@@ -0,0 +1,13 @@
+[Unit]
+Description=Telemetrics BERT Probe
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=/etc/telemetrics/opt-in
+ConditionPathExists=/sys/firmware/acpi/tables/data/BERT
+
+[Service]
+ExecStart=@bindir@/bertprobe
+User=telemetry
+
+[Install]
+WantedBy=multi-user.target

src/data/example.1.conf

@@ -0,0 +1,46 @@
+[settings]
+server=http://127.0.0.1
+
+cainfo=/tmp/cacert.crt
+
+tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
+
+socket_path=/tmp/test_telem_socket
+
+#record expiry time in minutes
+record_expiry=1200
+
+spool_dir=/tmp/spool
+
+#maximum size of the spool directory in KB
+spool_max_size=1024
+
+#time in seconds for processing spool
+spool_process_time=900
+
+#rate limiting record burst limit
+record_burst_limit=100
+
+#rate limiting record window length
+record_window_length = 15
+
+#rate limiting byte burst limit
+byte_burst_limit = 1000
+
+#rate limiting byte window length
+byte_window_length=20
+
+#rate limit enabled
+rate_limit_enabled=true
+
+#rate limit strategy if record not send
+rate_limit_strategy=spool
+
+#daemon recycling enabled
+daemon_recycling_enabled=true
+
+#server delivery disable
+record_server_delivery_enabled=false
+
+#local copy enabled
+record_retention_enabled=true

src/data/example.2.conf

@@ -0,0 +1,4 @@
+[settings]
+server=http://127.0.0.1
+
+cainfo=/tmp/cacert.crt

src/data/example.conf

@@ -16,7 +16,7 @@ spool_dir=/tmp/spool
 spool_max_size=1024
 
 #time in seconds for processing spool
-spool_process_time=900
+spool_process_time=180
 
 #rate limiting record burst limit
 record_burst_limit=100

src/data/hprobe.service.in

@@ -1,11 +1,11 @@
 [Unit]
 Description=Telemetrics Heartbeat Probe
-Requires=telemd.socket
-After=telemd.socket
-ConditionPathExists=!/etc/telemetrics/opt-out
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=/etc/telemetrics/opt-in
 
 [Service]
-ExecStart=@bindir@/hprobe -H
+ExecStart=@bindir@/hprobe -Hlu
 User=telemetry
 
 [Install]

src/data/hprobe.timer

@@ -1,6 +1,6 @@
 [Unit]
 Description=Daily Heartbeat for Telemetrics
-ConditionPathExists=!/etc/telemetrics/opt-out
+ConditionPathExists=/etc/telemetrics/opt-in
 
 [Timer]
 OnCalendar=daily

src/data/journal-probe-tail.service.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Telemetrics Systemd Journal Probe
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=/etc/telemetrics/opt-in
+
+[Service]
+ExecStart=@bindir@/journalprobe -t
+User=telemetry
+
+[Install]
+WantedBy=multi-user.target

src/data/journal-probe.service.in

@@ -1,8 +1,8 @@
 [Unit]
 Description=Telemetrics Systemd Journal Probe
-Requires=telemd.socket
-After=telemd.socket
-ConditionPathExists=!/etc/telemetrics/opt-out
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=/etc/telemetrics/opt-in
 
 [Service]
 ExecStart=@bindir@/journalprobe

src/data/klogscanner.service.in

@@ -1,7 +1,7 @@
 [Unit]
 Description=Telemetrics Kernel Log Scanner
-Requires=telemd.socket
-After=telemd.socket oops-probe.service
+Requires=telemprobd.socket
+After=telemprobd.socket
 
 [Service]
 ExecStart=@bindir@/klogscanner

src/data/local.mk

@@ -4,23 +4,30 @@ pathfix = @sed \
 	-e 's|@libdir[@]|$(libdir)|g' \
 	-e 's|@localstatedir[@]|$(localstatedir)|g' \
 	-e 's|@PACKAGE_VERSION[@]|$(PACKAGE_VERSION)|g' \
-	-e 's|@SOCKETDIR[@]|$(SOCKETDIR)|g'
+	-e 's|@SOCKETDIR[@]|$(SOCKETDIR)|g' \
+	-e 's|@systemctldir[@]|$(SYSTEMD_SYSTEMCTLDIR)|g' \
+	-e 's|@BACKEND_ADDR[@]|$(BACKEND_ADDR)|g'
 
 EXTRA_DIST += \
 	%D%/40-core-ulimit.conf \
 	%D%/40-crash-probe.conf.in \
 	%D%/example.conf \
+	%D%/example.1.conf \
+	%D%/example.2.conf \
 	%D%/hprobe.service.in \
 	%D%/hprobe.timer \
+	%D%/bert-probe.service.in \
 	%D%/journal-probe.service.in \
-	%D%/oops-probe.service.in \
+	%D%/journal-probe-tail.service.in \
 	%D%/pstore-probe.service.in \
 	%D%/klogscanner.service.in \
 	%D%/pstore-clean.service.in \
 	%D%/libtelemetry.pc.in \
-	%D%/telemd.path.in \
-	%D%/telemd.service.in \
-	%D%/telemd.socket.in \
+	%D%/telempostd.service.in \
+	%D%/telempostd.path.in \
+	%D%/telemprobd.service.in \
+	%D%/telemprobd.socket.in \
+	%D%/telemprobd-update-trigger.service.in \
 	%D%/telemetrics-dirs.conf.in \
 	%D%/telemetrics.conf.in
 
@@ -46,22 +53,28 @@ systemdunitdir = @SYSTEMD_UNITDIR@
 systemdunit_DATA = \
 	%D%/hprobe.service \
 	%D%/hprobe.timer \
+	%D%/bert-probe.service \
 	%D%/journal-probe.service \
-	%D%/oops-probe.service \
+	%D%/journal-probe-tail.service \
 	%D%/pstore-probe.service \
 	%D%/klogscanner.service \
 	%D%/pstore-clean.service \
-	%D%/telemd.service \
-	%D%/telemd.socket \
-	%D%/telemd.path
+	%D%/telemprobd.service \
+	%D%/telemprobd.socket \
+	%D%/telemprobd-update-trigger.service \
+	%D%/telempostd.service \
+	%D%/telempostd.path
 
 %D%/hprobe.service: %D%/hprobe.service.in
 	$(pathfix) < $< > $@
 
+%D%/bert-probe.service: %D%/bert-probe.service.in
+	$(pathfix) < $< > $@
+
 %D%/journal-probe.service: %D%/journal-probe.service.in
 	$(pathfix) < $< > $@
 
-%D%/oops-probe.service: %D%/oops-probe.service.in
+%D%/journal-probe-tail.service: %D%/journal-probe-tail.service.in
 	$(pathfix) < $< > $@
 
 %D%/pstore-probe.service: %D%/pstore-probe.service.in
@@ -73,13 +86,19 @@ systemdunit_DATA = \
 %D%/pstore-clean.service: %D%/pstore-clean.service.in
 	$(pathfix) < $< > $@
 
-%D%/telemd.service: %D%/telemd.service.in
+%D%/telempostd.path: %D%/telempostd.path.in
 	$(pathfix) < $< > $@
 
-%D%/telemd.socket: %D%/telemd.socket.in
+%D%/telempostd.service: %D%/telempostd.service.in
 	$(pathfix) < $< > $@
 
-%D%/telemd.path: %D%/telemd.path.in
+%D%/telemprobd.service: %D%/telemprobd.service.in
+	$(pathfix) < $< > $@
+
+%D%/telemprobd.socket: %D%/telemprobd.socket.in
+	$(pathfix) < $< > $@
+
+%D%/telemprobd-update-trigger.service: %D%/telemprobd-update-trigger.service.in
 	$(pathfix) < $< > $@
 
 sysctldir = @SYSTEMD_SYSCTLDIR@
@@ -92,17 +111,20 @@ systemconfdir = @SYSTEMD_SYSTEMCONFDIR@
 systemconf_DATA = %D%/40-core-ulimit.conf
 
 clean-local:
-	-rm -f  %D%/telemd.service \
-		%D%/telemd.socket \
-		%D%/telemd.path \
+	-rm -f  %D%/telemprobd.service \
+		%D%/telemprobd.socket \
+		%D%/telempostd.service \
+		%D%/telempostd.path \
+		%D%/telemprobd-update-trigger.service \
 		%D%/telemetrics.conf \
 		%D%/telemetrics-dirs.conf \
 		%D%/libtelemetry.pc \
 		%D%/40-crash-probe.conf \
 		%D%/journal-probe.service \
-		%D%/oops-probe.service \
+		%D%/journal-probe-tail.service \
 		%D%/pstore-probe.service \
 		%D%/klogscanner.service \
 		%D%/pstore-clean.service \
 		%D%/hprobe.service \
+		%D%/bert-probe.service \
 		%D%/.dirstamp

src/data/oops-probe.service.in

@@ -1,11 +0,0 @@
-[Unit]
-Description=Telemetrics Kernel Oops Probe
-Requires=telemd.socket
-After=telemd.socket
-
-[Service]
-ExecStart=@bindir@/oopsprobe
-User=telemetry
-
-[Install]
-WantedBy=multi-user.target

src/data/pstore-probe.service.in

@@ -1,7 +1,8 @@
 [Unit]
 Description=Telemetrics Pstore Probe
-Requires=telemd.socket
-After=telemd.socket
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=/etc/telemetrics/opt-in
 
 [Service]
 ExecStart=@bindir@/pstoreprobe

src/data/telemetrics-dirs.conf.in

@@ -1,5 +1,6 @@
 d @localstatedir@/lib/telemetry 0755 telemetry telemetry -
 d @localstatedir@/spool/telemetry 0750 telemetry telemetry -
+d @localstatedir@/log/telemetry 0750 telemetry telemetry -
+d @localstatedir@/log/telemetry/records 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry 0750 telemetry telemetry -
-d @localstatedir@/cache/telemetry/oops 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry/pstore 0750 telemetry telemetry -

src/data/telemetrics.conf.in

@@ -1,57 +1,69 @@
+# values represent their default state
+# copy to /etc/telemetrics/telemetrics.conf and modify for user config
 [settings]
 # ip address of destination server for record delivery
-server=https://clr.telemetry.intel.com/v2/collector
+#server=@BACKEND_ADDR@
 
-socket_path=@SOCKETDIR@/telem-0
+#socket_path=@SOCKETDIR@/telem-0
 
 # certificate file to use to validate ssl endpoint
-cainfo=
+#cainfo=
 
 # Telemetry id - post header used to group records in ingestion service,
 # which may be ingesting for more than one set of clients. Can be set
 # to any string.
-tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
+#tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
 
 # record expiry time in minutes
-record_expiry=1200
+#record_expiry=1200
 
-spool_dir=@localstatedir@/spool/telemetry
+#spool_dir=@localstatedir@/spool/telemetry
 
 # maximum size of the spool directory in KB, -1 = quota disabled.
 # The block size of the files in this directory is considered,
 # and not the actual size of the record itself.
-spool_max_size=5120
+#spool_max_size=5120
 
 # time in seconds for processing spool
-# Valid range: 120..3600. Values outside this range are clamped.
-spool_process_time=900
+# Valid range: 120..300. Values outside this range are clamped.
+#spool_process_time=120
 
 # rate limit enabled - if this is set to false then all rate-limiting disabled.
 # It is possible to disable each rate-limit individually below.
-rate_limit_enabled=true
+#rate_limit_enabled=true
 
 # rate limiting record burst limit
 # Valid Range:  0..INT_MAX, -1 = disabled.
-record_burst_limit=1000
+#record_burst_limit=1000
 
 # rate limiting record window length in minutes
 # Valid Range: 0..59
-record_window_length=15
+#record_window_length=15
 
 # rate limiting byte burst limit
 # Valid Range:  0..INT_MAX, -1 = disabled.
-byte_burst_limit=-1
+#byte_burst_limit=-1
 
 # rate limiting byte window length in minutes
 # Valid Range: 0..59
-byte_window_length=20
+#byte_window_length=20
 
 # rate limit strategy - what to do with record if rate-limiting prevents 
 # delivery over network
 # Valid stategies: spool, drop
-rate_limit_strategy=spool
+#rate_limit_strategy=spool
 
 # daemon recycling enabled - if daemon has been running for a while (2 hours),
 # has not any client nor spool data, then it exits.
 # this is to ensure that latest code runs.
-daemon_recycling_enabled=true
+#daemon_recycling_enabled=true
+
+# record server delivery enabled - when enabled records will be delivered
+# to server otherwise records will be ignored. This configuration can be used
+# with 'record_retention_enable' configuration value to keep records local only.
+#record_server_delivery_enabled=true
+
+# record retention enabled - when enabled a copy of reported telemetry records
+# will be kept locally. This configuration combined with 'record_server_delivery_enabled'
+# value can be used to keep records local only.
+#record_retention_enabled=false

src/data/telempostd.path.in

@@ -1,6 +1,6 @@
 [Unit]
-Description=Telemetry Daemon Spool
-ConditionPathExists=!/etc/telemetrics/opt-out
+Description=Telemetry Post Daemon staging
+ConditionPathExists=/etc/telemetrics/opt-in
 
 [Path]
 DirectoryNotEmpty=@localstatedir@/spool/telemetry

src/data/telempostd.service.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Telemetrics Post Daemon
+ConditionPathExists=/etc/telemetrics/opt-in
+
+[Service]
+ExecStart=@bindir@/telempostd
+WorkingDirectory=@localstatedir@/spool/telemetry
+User=telemetry
+
+[Install]
+Also=telempostd.path
+WantedBy=multi-user.target

src/data/telemprobd-update-trigger.service.in

@@ -0,0 +1,7 @@
+[Unit]
+Description=Restart Telemetrics Daemon On Update
+BindsTo=update-triggers.target
+
+[Service]
+Type=oneshot
+ExecStart=@systemctldir@/systemctl try-restart telemprobd.service

src/data/telemprobd.service.in

@@ -1,12 +1,12 @@
 [Unit]
 Description=Telemetrics Daemon
-ConditionPathExists=!/etc/telemetrics/opt-out
+ConditionPathExists=/etc/telemetrics/opt-in
 
 [Service]
-ExecStart=@bindir@/telemd
+ExecStart=@bindir@/telemprobd
 WorkingDirectory=@localstatedir@/spool/telemetry
 User=telemetry
 
 [Install]
-Also=telemd.socket telemd.path
+Also=telemprobd.socket
 WantedBy=multi-user.target

src/data/telemprobd.socket.in

@@ -1,6 +1,6 @@
 [Unit]
 Description=Telemetrics Daemon
-ConditionPathExists=!/etc/telemetrics/opt-out
+ConditionPathExists=/etc/telemetrics/opt-in
 
 [Socket]
 ListenStream=@SOCKETDIR@/telem-0

src/iorecord.c

@@ -0,0 +1,143 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+
+#include "log.h"
+#include "util.h"
+#include "common.h"
+#include "iorecord.h"
+
+bool _fgets(char *s, int n, FILE *stream)
+{
+        char *i;
+
+        if (!fgets(s, n, stream)) {
+                return false;
+        }
+        if ((i = strchr(s, '\n')) != NULL) {
+                *i = '\0';
+        }
+
+        return true;
+}
+
+bool read_record(char *fullpath, char *headers[], char **body, char **cfg_file)
+{
+        int i = 0;
+        bool result = false;
+        FILE *fp = NULL;
+        long offset;
+#if (LINE_MAX > PATH_MAX)
+        char line[LINE_MAX+1] = { 0 };
+#else
+        char line[PATH_MAX+1] = { 0 };
+#endif
+        long size;
+        uint32_t cfg_prefix = 0;
+
+        fp = fopen(fullpath, "r");
+        if (fp == NULL) {
+                telem_log(LOG_ERR, "Unable to open file %s in staging\n", fullpath);
+                return false;
+        }
+
+        // Get the file size
+        fseek(fp, 0 , SEEK_END);
+        size = ftell(fp);
+        fseek(fp, 0 , SEEK_SET);
+
+        // First line may contain configuration file path
+        if (fread(&cfg_prefix, CFG_PREFIX_LENGTH, 1, fp) != 1) {
+                telem_log(LOG_ERR, "Error while parsing staged record configuration info.\n");
+                goto read_error;
+        }
+
+        if (cfg_prefix == CFG_PREFIX_32BIT) {
+                if (!_fgets(line, sizeof(line), fp)) {
+                        telem_log(LOG_ERR, "Error while parsing staged record [%x]\n", cfg_prefix);
+                        goto read_error;
+                }
+
+                size_t pathlen = strlen(line);
+                *cfg_file = malloc(pathlen + 1);
+                if (*cfg_file == NULL) {
+                        telem_log(LOG_ERR, "Could not allocate memory for config file path\n");
+                        goto read_error;
+                }
+                strcpy (*cfg_file, line);
+                telem_debug("DEBUG: cfg_file specified: %s\n", *cfg_file);
+        } else {
+                telem_debug("DEBUG: no user cfg file specified, cfg_prefix: %08x\n", cfg_prefix);
+                *cfg_file = NULL;
+                rewind(fp);
+        }
+
+        for (i = 0; i < NUM_HEADERS; i++) {
+                const char *header_name = get_header_name(i);
+                if (!_fgets(line, sizeof(line), fp)) {
+                        telem_log(LOG_ERR, "Error while parsing staged record\n");
+                        fclose(fp);
+                        return false;
+                }
+                //Get rid of trailing newline
+                result = get_header(line, header_name, &headers[i]);
+                if (!result) {
+                        telem_log(LOG_ERR, "read_record: Incorrect"
+                                  " headers in record\n");
+                        goto read_error;
+                }
+        }
+
+        offset = ftell(fp);
+        if (offset == -1) {
+                telem_perror("Failed to get staged file offset of record payload");
+                result = false;
+                goto read_error;
+        }
+
+        size = size - offset + 1;
+        *body = malloc((size_t)size);
+        if (*body == NULL) {
+                telem_log(LOG_ERR, "Could not allocate memory for payload from staged file\n");
+                result = false;
+                goto read_error;
+        }
+        memset(*body, 0, (size_t)size);
+
+        //read rest of file
+        size_t newlen = fread(*body, sizeof(char), (size_t)size, fp);
+        if (newlen == 0) {
+                telem_perror("Error reading staged file");
+                result = false;
+                goto read_error;
+        } else {
+                result = true;
+        }
+
+read_error:
+        if (fp) {
+                fclose(fp);
+        }
+
+        return result;
+}

src/iorecord.h

@@ -0,0 +1,28 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#include <stdbool.h>
+
+/**
+ * Reads a telemetry record
+ *
+ * @param fullpath pointer to full path file name
+ * @param headers pointer to array of headers and values
+ * @param body record message content
+ *
+ * @return true if successful otherwise false
+ */
+bool read_record(char *fullpath, char *headers[], char **body, char **cfg);

src/journal/cli.c

@@ -0,0 +1,115 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <getopt.h>
+
+#include "common.h"
+#include "journal.h"
+
+static void print_usage(void)
+{
+        printf("Usage:\n");
+        printf("  telem_journal [-Vi] [-r <record_id>] [-e <event_id>] [-c <classification>] [-b <boot_id>]\n\n");
+        printf("Where:\n");
+        printf("  -r,  --record_id        Print record with specific record_id\n");
+        printf("  -e,  --event_id         Print records with specific event_id\n");
+        printf("  -c,  --classification   Print records with specific classification\n");
+        printf("  -b,  --boot_id          Print records with specific boot_id\n");
+        printf("  -i,  --include_record   Include record content if available.\n");
+        printf("                          Content only available when telemetry is configured\n");
+        printf("                          with \"record_retention_enabled=true\"\n");
+        printf("  -V,  --verbose          Verbose output\n");
+        printf("  -h,  --help             Display this help message\n");
+}
+
+int main(int argc, char **argv)
+{
+
+        int rc = EXIT_SUCCESS;
+        int count = 0;
+        bool verbose_output = false;
+        bool record = false;
+        char *boot_id = NULL;
+        char *record_id = NULL;
+        char *event_id = NULL;
+        char *classification = NULL;
+        struct TelemJournal *telem_journal = NULL;
+
+        /** opts */
+        int c;
+        int opt_index = 0;
+        struct option opts[] = {
+                { "record_id", 1, NULL, 'r' },
+                { "event_id", 1, NULL, 'e' },
+                { "classification", 1, NULL, 'c' },
+                { "boot_id", 1, NULL, 'b' },
+                { "verbose", 0, NULL, 'V' },
+                { "include_record", 0, NULL, 'i' },
+                { "help", 0, NULL, 'h' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "r:e:c:b:Vih", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'r':
+                                record_id = optarg;
+                                break;
+                        case 'e':
+                                event_id = optarg;
+                                break;
+                        case 'c':
+                                classification = optarg;
+                                break;
+                        case 'b':
+                                boot_id = optarg;
+                                break;
+                        case 'V':
+                                verbose_output = true;
+                                break;
+                        case 'i':
+                                record = true;
+                                break;
+                        case 'h':
+                                print_usage();
+                                exit(EXIT_SUCCESS);
+                        default:
+                                print_usage();
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        if ((telem_journal = open_journal(JOURNAL_PATH))) {
+                if (verbose_output) {
+                        fprintf(stdout, "%-30s %-27s %-32s %-32s %-36s\n", "Classification", "Time stamp",
+                                "Record ID", "Event ID", "Boot ID");
+                }
+                count = print_journal(telem_journal, classification, record_id, event_id, boot_id, record);
+                if (verbose_output) {
+                        fprintf(stdout, "Total records: %d\n", count);
+                }
+                close_journal(telem_journal);
+        } else {
+                fprintf(stderr, "Unable to open journal\n");
+                rc = EXIT_FAILURE;
+        }
+
+        return rc;
+}

src/journal/journal.c

@@ -0,0 +1,678 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#define ID_LEN 32
+#define BOOTID_LEN 37 // Includes the \n character at the end
+#define BOOTID_FILE "/proc/sys/kernel/random/boot_id"
+#define MID_BUFF 1024
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/uio.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include "log.h"
+#include "util.h"
+#include "common.h"
+#include "journal.h"
+
+/*
+ * Packs journal entry values into one line string separated
+ * by '\036' RS.
+ *
+ * @param entry A pointer to a JournalEntry struct with values
+ *              to serialize.
+ * @param buff A pointer to be initialized and set with the
+ *             serialized values.
+ *
+ * @return 0 on success, -1 on failure.
+ */
+static int serialize_journal_entry(struct JournalEntry *entry, char **buff)
+{
+        int rc = 0;
+
+        if (entry == NULL) {
+                return -1;
+        }
+
+        if (asprintf(buff, "%s\036%lld\036%s\036%s\036%s", entry->record_id, (long long int)entry->timestamp,
+                     entry->classification, entry->event_id, entry->boot_id) < 0) {
+                telem_log(LOG_ERR, "Error: Unable to serialize data in buff\n");
+                rc = -1;
+        }
+
+        return rc;
+}
+
+/**
+ *  Frees journal entry struct members and journal entry pointer.
+ *
+ *  @param the pointer to the entry structure to free.
+ */
+static void free_journal_entry(struct JournalEntry *entry)
+{
+        if (entry) {
+                free(entry->classification);
+                free(entry->record_id);
+                free(entry->event_id);
+                free(entry->boot_id);
+                free(entry);
+        }
+}
+
+/**
+ * Unpacks a record that was stored in file and initializes a
+ * JournalEntry struct with parsed values.
+ *
+ * @param line A pointer to the data to unpack.
+ * @param entry A pointer to be initialized and set with values
+ *        parsed from parameter line.
+ *
+ * @return 0 on success, -1 on failure.
+ */
+static int deserialize_journal_entry(char *line, struct JournalEntry **entry)
+{
+        int rc = -1;
+        size_t len = 0;
+        size_t llen = 0;
+        size_t offset = 0;
+        struct JournalEntry *e = NULL;
+
+        /* Assume an error... */
+        *entry = NULL;
+
+        if (line == NULL || !strlen(line)) {
+                return -1;
+        }
+
+        e = calloc(1, sizeof(struct JournalEntry));
+        if (!e) {
+                return -1;
+        }
+
+        llen = strlen(line);
+
+        for (int i = 0; i < 5; i++) {
+                char *out = NULL;
+                if (offset > llen) {
+                        rc = -1;
+                        break;
+                }
+                len = strcspn(line + offset, "\036\n");
+                out = strndup((line + offset), len);
+                offset += len + 1;
+                if (out) {
+                        switch (i) {
+                                case 0:
+                                        e->record_id = out;
+                                        break;
+                                case 1:
+                                        e->timestamp = atoll(out);
+                                        free(out);
+                                        break;
+                                case 2:
+                                        e->classification = out;
+                                        break;
+                                case 3:
+                                        e->event_id = out;
+                                        break;
+                                case 4:
+                                        e->boot_id = out;
+                                        rc = 0;
+                                        break;
+                        }
+                } else {
+                        rc = -1;
+                        break;
+                }
+        }
+
+        if (rc == 0) {
+                *entry = e;
+        } else {
+                free_journal_entry(e);
+        }
+
+        return rc;
+}
+
+/**
+ * Counts the number of records in journal.
+ *
+ * @param telem_journal Pointer to a telemetry journal
+ *        to get record count.
+ *
+ * @return record count
+ *
+ */
+static int get_record_count(struct TelemJournal *telem_journal)
+{
+        int count = 0;
+        size_t len;
+        ssize_t read;
+        char *line = NULL;
+
+        rewind(telem_journal->fptr);
+        // read until the end line by line
+        while ((read = getline(&line, &len, telem_journal->fptr)) != -1) {
+                count++;
+        }
+        free(line);
+
+        return count;
+}
+
+/**
+ * Serializes JournalEntry and saves it to file.
+ *
+ * @param fptr A FILE pointer.
+ * @param entry A pointer to data.
+ *
+ * @return 0 on succes, !=0 on failure
+ */
+static int save_entry(FILE *fptr, struct JournalEntry *entry)
+{
+        int rc = -1;
+        char *serialized_data = NULL;
+
+        if (fptr == NULL || entry == NULL) {
+                return rc;
+        }
+
+        if ((rc = serialize_journal_entry(entry, &serialized_data)) == 0) {
+                telem_debug("DEBUG: Saving: %s\n", serialized_data);
+                fprintf(fptr, "%s\n", serialized_data);
+                fflush(fptr);
+                free(serialized_data);
+        }
+
+        return rc;
+}
+
+/**
+ * Reads the boot unique identifier from BOOTID_FILE
+ *
+ * @param buff pointer to a BOOTID_LEN allocated
+ *
+ * @return 0 on success, -1 on failure
+ */
+static int read_boot_id(char buff[])
+{
+        int rc = -1;
+        FILE *fs = NULL;
+
+        fs = fopen(BOOTID_FILE, "r");
+        if (!fs) {
+                telem_log(LOG_ERR, "Error: Unable to open %s for reading: %d\n", BOOTID_FILE, errno);
+                return rc;
+        }
+
+        if (fgets(buff, BOOTID_LEN, fs)) {
+                rc = 0;
+        }
+        fclose(fs);
+
+        return rc;
+}
+
+/**
+ * Moves the file position indicator n lines.
+ *
+ * @param n Number of lines to skip.
+ * @param fptr A file pointer.
+ * @param found_record A pointer to a function to be called
+ *                     every time a valid record is found, if
+ *                     pointer is not set to null.
+ *
+ * @returns 0 on success, on failure -1 if n > existing
+ *            lines in file or errno if failure happens
+ *            during file operation.
+ */
+static int skip_n_lines(int n, FILE *fptr, int (*found_record)(char *))
+{
+        int rc = 0;
+        size_t len = 0;
+        char *line = NULL;
+        struct JournalEntry *entry = NULL;
+
+        if (fptr == NULL) {
+                telem_log(LOG_ERR, "fptr argument to skip_n_lines is invalid\n");
+                // File descriptor in bad state
+                return EBADFD;
+        }
+
+        if (fseek(fptr, 0, 0) != 0) {
+                return errno;
+        }
+
+        for (int i = 0; i < n; i++) {
+                if (getline(&line, &len, fptr) == -1) {
+                        rc = -1;
+                        break;
+                }
+                if (found_record != NULL) {
+                        if (!strlen(line))
+                            continue;
+                        deserialize_journal_entry(line, &entry);
+                        if (entry) {
+                                found_record(entry->record_id);
+                                free_journal_entry(entry);
+                                entry = NULL;
+                        }
+                }
+        }
+        free(line);
+
+        return rc;
+}
+
+/**
+ * Copies contents of fptr to a known location.
+ *
+ * @param fptr A file pointer from source file.
+ *
+ * @returns 0 on success, errno on failure
+ */
+static int copy_to_tmp(FILE *fptr, char *tmp_path)
+{
+        int rc = 0;
+        size_t len;
+        char *line = NULL;
+        FILE *fptr_tmp = NULL;
+
+        if (fptr == NULL) {
+                return -1;
+        }
+
+        fptr_tmp = fopen(tmp_path, "w");
+        if (!fptr_tmp) {
+                return errno;
+        }
+
+        // Copy lines to new file
+        while (getline(&line, &len, fptr) > -1) {
+                if (fprintf(fptr_tmp, "%s", line) < 0) {
+                        rc = errno;
+                        break;
+                }
+        }
+
+        if (fclose(fptr_tmp) != 0) {
+                rc = errno;
+                telem_perror("Error");
+        }
+        free(line);
+
+        return rc;
+}
+
+/* Exported function */
+TelemJournal *open_journal(const char *journal_file)
+{
+        FILE *fptr = NULL;
+        char boot_id[BOOTID_LEN] = { '\0' };
+        struct TelemJournal *telem_journal;
+
+        // Use default location if journal_file parameter is NULL
+        if (journal_file == NULL) {
+                fptr = fopen(JOURNAL_PATH, "a+");
+        } else {
+                fptr = fopen(journal_file, "a+");
+        }
+        if (fptr == NULL) {
+                telem_perror("Error while opening journal file");
+                return NULL;
+        }
+
+        if (read_boot_id(boot_id) != 0) {
+                telem_perror("Error while reading boot_id");
+                fclose(fptr);
+                return NULL;
+        }
+
+        telem_journal = malloc(sizeof(struct TelemJournal));
+        if (!telem_journal) {
+                telem_log(LOG_CRIT, "CRIT: Unable to allocate memory\n");
+                fclose(fptr);
+                return NULL;
+        }
+
+        telem_journal->fptr = fptr;
+        telem_journal->journal_file = \
+                (journal_file == NULL) ? strdup(JOURNAL_PATH) : strdup(journal_file);
+        /* boot_id includes \n at the end, strip RC during duplication */
+        telem_journal->boot_id = strndup(boot_id, BOOTID_LEN - 1);
+        telem_journal->record_count = get_record_count(telem_journal);
+        telem_journal->record_count_limit = RECORD_LIMIT;
+        telem_journal->latest_record_id = NULL;
+        telem_journal->prune_entry_callback = NULL;
+
+        telem_debug("Records in db: %d\n", telem_journal->record_count);
+
+        return telem_journal;
+}
+
+/* Exported function */
+void close_journal(TelemJournal *telem_journal)
+{
+        if (telem_journal) {
+                free(telem_journal->boot_id);
+                free(telem_journal->journal_file);
+                free(telem_journal->latest_record_id);
+                fclose(telem_journal->fptr);
+                free(telem_journal);
+        }
+}
+
+/**
+ * Checks if class is a prefix i.e. t/\* or t/t/\*
+ *
+ * @param class A pointer to string with classification
+ *              value to check.
+ * @return 1 if class is a prefix, 0 otherwise
+ */
+static int is_class_prefix(char *class)
+{
+        size_t class_len = strlen(class);
+        return (strcmp((char *)(class + class_len - 2), "/*") == 0) ? 1 : 0;
+}
+
+/**
+ * Print records content
+ *
+ * @param record_id Unique record identifier
+ *
+ */
+static void print_record(char *record_id)
+{
+        int rc = 0;
+        size_t read = 0;
+        char buff[MID_BUFF] = { '\0' };
+        char *filepath = NULL;
+        FILE *recordfp = NULL;
+
+        rc = asprintf(&filepath, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (rc == -1) {
+                // just bail out, there are worse problems
+                return;
+        }
+
+        recordfp = fopen(filepath, "r");
+        if (!recordfp) {
+                telem_log(LOG_INFO, "Could not open record %s: %s\n", record_id, strerror(errno));
+                return;
+        }
+
+        while ((read = fread(buff, 1, sizeof(buff), recordfp)) > 0) {
+                fwrite(buff, 1, read, stdout);
+        }
+
+        free(filepath);
+        fclose(recordfp);
+}
+
+/* Exported function */
+int print_journal(TelemJournal *telem_journal, char *classification,
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record)
+{
+        int n = 0;
+        int rc = 0;
+        int count = 0;
+        char str_time[80] = { '\0' };
+        char *line = NULL;
+        size_t len = 0;
+        FILE *journal_fileptr = NULL;
+        struct tm ts;
+        struct JournalEntry *entry = NULL;
+
+        if (telem_journal == NULL) {
+                return -1;
+        }
+
+        journal_fileptr = fopen(telem_journal->journal_file, "r");
+        if (!journal_fileptr) {
+                return -1;
+        }
+
+        // Advance file pointer to line n
+        n = telem_journal->record_count - telem_journal->record_count_limit;
+        if (n > 0) {
+                rc = skip_n_lines(n, journal_fileptr, NULL);
+                if (rc == -1) {
+                        return rc;
+                } else if (rc != 0) {
+                        telem_log(LOG_ERR, "An error occurred while advancing journal file: %s\n", strerror(rc));
+                }
+        }
+
+        while (getline(&line, &len, journal_fileptr) != -1) {
+                if (!strlen(line))
+                    continue;
+                deserialize_journal_entry(line, &entry);
+                if (entry) {
+                        /* filter entry out if one is provided */
+                        if (record_id != NULL && strcmp(entry->record_id, record_id) != 0) {
+                                goto skip_print;
+                        }
+                        if (boot_id != NULL && strcmp(entry->boot_id, boot_id) != 0) {
+                                goto skip_print;
+                        }
+                        if (event_id != NULL && strcmp(entry->event_id, event_id) != 0) {
+                                goto skip_print;
+                        }
+                        // In the case of class checking prefixes is an option
+                        if (classification != NULL) {
+                                // Check prefixes when classification ends in /*, otherwise use strcomp
+                                if (is_class_prefix(classification)) {
+                                        if (strncmp(entry->classification, classification, strlen(classification) - 1) != 0) {
+                                                goto skip_print;
+                                        }
+                                } else if (strcmp(entry->classification, classification) != 0) {
+                                        goto skip_print;
+                                }
+                        }
+                        /* end filters section */
+                        ts = *localtime(&entry->timestamp);
+                        if (strftime(str_time, sizeof(str_time), "%a %Y-%m-%d %H:%M:%S %Z", &ts) == 0) {
+                                goto skip_print;
+                        }
+                        /* print record metadata */
+                        fprintf(stdout, "%-30s %s %s %s %s\n", entry->classification, str_time, entry->record_id, entry->event_id, entry->boot_id);
+                        /* print record content */
+                        if (include_record) {
+                                print_record(entry->record_id);
+                        }
+                        count++;
+skip_print:
+                        free_journal_entry(entry);
+                }
+        }
+        free(line);
+        fclose(journal_fileptr);
+
+        return count;
+}
+
+/**
+ * Validation for event id. Thic check makes sure
+ * that an event_id is a 32 hexadecimal chars long.
+ *
+ * @param event_id A pointer to id value.
+ *
+ * @return 0 on success, 1 on failure
+ */
+static int validate_event_id(char *event_id)
+{
+        const char alphab[] = EVENT_ID_ALPHAB;
+
+        if (event_id == NULL) {
+                return 1;
+        }
+
+        if (strlen(event_id) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        if (strspn(event_id, alphab) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        return 0;
+}
+
+/* Exported function */
+int new_journal_entry(TelemJournal *telem_journal, char *classification,
+                      time_t timestamp, char *event_id)
+{
+        int rc = 1;
+        char boot_id[BOOTID_LEN] = { '\0' };
+        char *record_id = NULL;
+        struct JournalEntry *entry = NULL;
+
+        if (telem_journal == NULL) {
+                telem_log(LOG_ERR, "telem_journal was not initialized\n");
+                return rc;
+        }
+
+        if (validate_classification(classification) != 0) {
+                return rc;
+        }
+
+        if (validate_event_id(event_id) != 0) {
+                return rc;
+        }
+
+        entry = malloc(sizeof(struct JournalEntry));
+        if (!entry) {
+                telem_log(LOG_CRIT, "CRIT: unable to allocate memory\n");
+                return rc;
+        }
+        entry->classification = NULL;
+        entry->record_id = NULL;
+        entry->event_id = NULL;
+        entry->boot_id = NULL;
+
+        if (get_random_id(&record_id) != 0) {
+                telem_log(LOG_ERR, "Erorr: Unable to generate random id\n");
+                goto quit;
+        }
+
+        if (read_boot_id(boot_id) != 0) {
+                telem_log(LOG_ERR, "Error: Unable to read boot_id\n");
+                goto quit;
+        }
+
+        if ((entry->classification = strdup(classification)) == NULL) {
+                goto quit;
+        }
+        if ((entry->event_id = strdup(event_id)) == NULL) {
+                goto quit;
+        }
+
+        entry->record_id = record_id;
+        /* boot_id includes \n at the end, strip RC during duplication */
+        entry->boot_id = strndup(boot_id, BOOTID_LEN - 1);
+        entry->timestamp = timestamp;
+
+        if ((rc = save_entry(telem_journal->fptr, entry)) == 0) {
+                telem_journal->record_count = telem_journal->record_count + 1;
+                telem_debug("DEBUG: %d records in journal\n", telem_journal->record_count);
+        }
+
+        free(telem_journal->latest_record_id);
+        telem_journal->latest_record_id = strdup(entry->record_id);
+
+quit:
+        free_journal_entry(entry);
+
+        return rc;
+}
+
+/* Exported function */
+int prune_journal(struct TelemJournal *telem_journal, char *tmp_dir)
+{
+
+        int rc = 1;
+        int count = 0;
+        int deviation = DEVIATION;
+        char *tmp_file_path = NULL;
+
+        if (telem_journal == NULL) {
+                return rc;
+        }
+
+        if (telem_journal->record_count > (deviation + telem_journal->record_count_limit)) {
+                count = telem_journal->record_count - telem_journal->record_count_limit;
+
+                // jump to line# count
+                if ((rc = skip_n_lines(count, telem_journal->fptr, telem_journal->prune_entry_callback)) != 0) {
+                        telem_log(LOG_ERR, "Error skipping %d journal lines\n", count);
+                        // if rc == -1 (n > #lines in file, change to errno style)
+                        return (rc == -1) ? EADDRNOTAVAIL : rc;
+                }
+                // Use default if not tmp_dir is specified
+                if (tmp_dir == NULL) {
+                        if ((asprintf(&tmp_file_path, "%s/.journal", JOURNAL_TMPDIR) == -1)) {
+                                return ENOMEM;
+                        }
+                } else {
+                        if ((asprintf(&tmp_file_path, "%s/.journal", tmp_dir) == -1)) {
+                                return ENOMEM;
+                        }
+                }
+                // create new file with rest of file
+                if ((rc = copy_to_tmp(telem_journal->fptr, tmp_file_path)) != 0) {
+                        telem_log(LOG_ERR, "Error copying partial journal to temp journal file\n");
+                        goto quit;
+                }
+                // close file handler
+                if ((rc = fclose(telem_journal->fptr)) != 0) {
+                        telem_log(LOG_ERR, "Error closing journal file handler\n");
+                        goto quit;
+                }
+                // overwrite file
+                if ((rc = rename(tmp_file_path, telem_journal->journal_file)) != 0) {
+                        telem_log(LOG_ERR, "Error while overwriting journal file\n");
+                        goto quit;
+                }
+                // reopen file handler
+                telem_journal->fptr = fopen(telem_journal->journal_file, "a+");
+                if (!telem_journal->fptr) {
+                        telem_log(LOG_ERR, "Error re-opening journal file\n");
+                        return rc;
+                }
+                // update record count
+                telem_journal->record_count = telem_journal->record_count - count;
+                telem_debug("DEBUG: record_count: %d\n", telem_journal->record_count);
+        }
+        rc = 0;
+
+quit:
+        free(tmp_file_path);
+
+        return rc;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/journal/journal.h

@@ -0,0 +1,117 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+/* default record limit */
+#define RECORD_LIMIT 100
+#define DEVIATION 50
+
+#include <time.h>
+#include <stdio.h>
+#include <stdbool.h>
+
+/* Journal entry type */
+typedef struct JournalEntry {
+        time_t timestamp;
+        char *classification;
+        char *record_id;
+        char *event_id;
+        char *boot_id;
+} JournalEntry;
+
+/* Telemetry journal type */
+typedef struct TelemJournal {
+        FILE *fptr;
+        char *journal_file;
+        char *boot_id;
+        char *latest_record_id;
+        int record_count;
+        int record_count_limit;
+        int (*prune_entry_callback)(char *);
+} TelemJournal;
+
+/**
+ * Telemetry journal initialization.
+ *
+ * @param journal_file A pointer to a string containing
+ *        the full path to file used as journal storage.
+ *        If journal_file is set to NULL a default value
+ *        will be used.
+ *
+ * @returns a telemetry journal structure in success or
+ *          NULL in case of failure.
+ */
+TelemJournal *open_journal(const char *journal_file);
+
+/**
+ * Closes journal file and deallocates memory that was
+ * previously initialized by open_journal call.
+ *
+ * @param telem_journal A pointer to struct that has a
+ *         reference to journal file pointer and other
+ *         members.
+ */
+void close_journal(TelemJournal *telem_journal);
+
+/**
+ * Prints journal contents to stdout. Use function parameters
+ * to filter journal entries to print.
+ *
+ * @param telem_journal A pointer to struct initialized
+ *        by open_journal call.
+ * @param classification A pointer to string used as
+ *        classification filter.
+ * @param record_id A pointer to string used as record_id filter.
+ * @param event_id A pointer to string used as event_id filter.
+ * @param boot_id A pointer to string used as boor_id filter.
+ * @param include_record A flag to control record content print.
+ *
+ * @return the number of lines printed to stdout on success, -1
+ *         on failure.
+ */
+int print_journal(TelemJournal *telem_journal, char *classification,
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record);
+
+/**
+ * Creates a new entry in journal.
+ *
+ * @param telem_journal A pointer to telemetry journal.
+ * @param classification A pointer to a classification value.
+ * @param timestamp Record creation time stamp value.
+ * @param event_id A pointer to a unique id value for record
+ *        event.
+ *
+ * @return 0 on success 1 on failure.
+ */
+int new_journal_entry(TelemJournal *telem_journal, char *classification,
+                      time_t timestamp, char *event_id);
+
+/**
+ * Checks number of lines in log and prunes the oldest records
+ * if journal grows more than telem_journal->record_count_limit.
+ *
+ * @param telem_journal A pointer to telemetry journal struct
+ *        returned by open_journal call.
+ * @param tmp_dir A pointer to a string with the path to use for
+ *                temp file during pruning process.
+ *
+ * @return 0 on success, errno on failure
+ */
+int prune_journal(TelemJournal *telem_journal, char *tmp_dir);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/journal/local.mk

@@ -0,0 +1,15 @@
+bin_PROGRAMS += \
+	%D%/telem_journal
+
+%C%_telem_journal_SOURCES = %D%/cli.c \
+	%D%/journal.c \
+	src/util.c \
+	src/common.c
+%C%_telem_journal_CFLAGS = \
+	$(AM_CFLAGS)
+
+if LOG_SYSTEMD
+%C%_telem_journal_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_telem_journal_LDADD = $(SYSTEMD_JOURNAL_LIBS)
+endif
+# vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

src/local.mk

@@ -1,41 +1,96 @@
-bin_PROGRAMS = %D%/telemd
+bin_PROGRAMS = \
+	%D%/telemprobd \
+	%D%/telempostd \
+	%D%/telemctl
 
-%C%_telemd_SOURCES = \
-	%D%/main.c \
-	%D%/telemdaemon.c \
-	%D%/telemdaemon.h \
-	%D%/spool.c \
-	%D%/spool.h
+%C%_telemctl_SOURCES = \
+	%D%/telemctl.c
 
-%C%_telemd_LDADD = $(CURL_LIBS) \
-	%D%/libtelem-shared.la
-
-%C%_telemd_CFLAGS = \
+%C%_telemctl_CFLAGS = \
 	$(AM_CFLAGS)
 
-%C%_telemd_LDFLAGS = \
+%C%_telemctl_LDFLAGS = \
+	$(AM_LDFLAGS) \
+	-pie
+
+%C%_telemprobd_SOURCES = \
+	%D%/probe.c \
+	%D%/telemdaemon.c \
+	%D%/telemdaemon.h \
+	%D%/journal/journal.c \
+	%D%/journal/journal.h
+
+%C%_telemprobd_LDADD = $(CURL_LIBS) \
+	%D%/libtelem-shared.la \
+	%D%/libtelemetry.la
+
+%C%_telemprobd_CFLAGS = \
+	$(AM_CFLAGS)
+
+%C%_telemprobd_LDFLAGS = \
 	$(AM_LDFLAGS) \
 	-pie
 
 if HAVE_SYSTEMD_DAEMON
-%C%_telemd_CFLAGS += \
-        $(SYSTEMD_DAEMON_CFLAGS)
-%C%_telemd_LDADD += \
-        $(SYSTEMD_DAEMON_LIBS)
+%C%_telemprobd_CFLAGS += \
+	$(SYSTEMD_DAEMON_CFLAGS)
+%C%_telemprobd_LDADD += \
+	$(SYSTEMD_DAEMON_LIBS)
 endif
 
 if LOG_SYSTEMD
 if HAVE_SYSTEMD_JOURNAL
-%C%_telemd_CFLAGS += \
-        $(SYSTEMD_JOURNAL_CFLAGS)
-%C%_telemd_LDADD += \
-        $(SYSTEMD_JOURNAL_LIBS)
+%C%_telemprobd_CFLAGS += \
+	$(SYSTEMD_JOURNAL_CFLAGS)
+%C%_telemprobd_LDADD += \
+	$(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
+# Telemetry post daemon
+%C%_telempostd_SOURCES = \
+	%D%/post.c \
+	%D%/telempostdaemon.c \
+	%D%/telempostdaemon.h \
+	%D%/journal/journal.c \
+	%D%/journal/journal.h \
+	%D%/spool.h \
+	%D%/spool.c \
+	%D%/retention.h \
+	%D%/retention.c \
+	%D%/iorecord.c \
+	%D%/iorecord.h
+
+%C%_telempostd_LDADD = $(CURL_LIBS) \
+	%D%/libtelem-shared.la \
+	%D%/libtelemetry.la
+
+%C%_telempostd_CFLAGS = \
+	$(AM_CFLAGS)
+
+%C%_telempostd_LDFLAGS = \
+	$(AM_LDFLAGS) \
+	-pie
+
+if HAVE_SYSTEMD_DAEMON
+%C%_telempostd_CFLAGS += \
+	$(SYSTEMD_DAEMON_CFLAGS)
+%C%_telempostd_LDADD += \
+	$(SYSTEMD_DAEMON_LIBS)
+endif
+
+if LOG_SYSTEMD
+if HAVE_SYSTEMD_JOURNAL
+%C%_telempostd_CFLAGS += \
+	$(SYSTEMD_JOURNAL_CFLAGS)
+%C%_telempostd_LDADD += \
+	$(SYSTEMD_JOURNAL_LIBS)
 endif
 endif
 
 # set library version info
-SHAREDLIB_CURRENT=3
-SHAREDLIB_REVISION=0
+SHAREDLIB_CURRENT=4
+SHAREDLIB_REVISION=1
 SHAREDLIB_AGE=0
 
 noinst_LTLIBRARIES = %D%/libtelem-shared.la
@@ -46,6 +101,7 @@ noinst_LTLIBRARIES = %D%/libtelem-shared.la
 	%D%/configuration.c \
 	%D%/nica/inifile.c \
 	%D%/nica/hashmap.c \
+	%D%/nica/b64enc.c \
 	%D%/configuration.h \
 	%D%/common.c \
 	%D%/common.h

src/log.h

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -17,6 +17,7 @@
 #pragma once
 
 #include <stdio.h>
+#include <errno.h>
 
 #include "config.h"
 
@@ -35,15 +36,14 @@
 #include <systemd/sd-journal.h>
 #endif
 #else
-#include <errno.h>
 #include <string.h>
 #include <syslog.h>
 #endif
 
 #ifdef DEBUG
 #define telem_debug(...) do { \
-                (telem_log(LOG_DEBUG, "%s():[%d]", __func__, __LINE__), \
-                 telem_log(LOG_DEBUG, __VA_ARGS__)); \
+                telem_log(LOG_DEBUG, "%s():[%d] ", __func__, __LINE__); \
+                telem_log(LOG_DEBUG, __VA_ARGS__); \
 } while (0);
 #else
 #define telem_debug(...) do {} while (0);

src/nica/b64enc.c

@@ -0,0 +1,154 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdint.h>
+#include "b64enc.h"
+
+#define B64_LINE_LEN 76
+
+static int padding[] = {0, 2, 1, 0};
+static char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+
+/*
+ * base 64 encoding of 3 bytes to 4 printable characters
+ * (https://tools.ietf.org/html/rfc4648).
+ *
+ * i.e. (Reference: https://en.wikipedia.org/wiki/Base64)
+ *
+ *      [   M  ] [   a  ] [  n  ]
+ * bin  01001101 01100001 01101110
+ *
+ * b64  01001101 01100001 01101110  >> 18 & 0x3F -> 010011 -> ( 19 dec) T
+ *      ------
+ *
+ * b64  01001101 01100001 01101110  >> 12 & 0x3F -> 010110 -> ( 22 dec) W
+ *            -------
+ *
+ * b64  01001101 01100001 01101110  >>  6 & 0x3F -> 000101 -> (  5 dec) F
+ *                   -------
+ *
+ * b64  01001101 01100001 01101110  >>  0 & 0x3F -> 101110 -> (117 dec) u
+ *                          ------
+ *
+ */
+static void b64_3b(char bin[3], char *out) {
+
+    uint32_t x = (((uint32_t) bin[0] << 16) & 0xFF0000) |
+                 (((uint32_t) bin[1] <<  8) & 0xFF00)   |
+                 (((uint32_t) bin[2])       & 0xFF);
+
+    out[3] = table[x       & 0x3F];
+    out[2] = table[x >> 6  & 0x3F];
+    out[1] = table[x >> 12 & 0x3F];
+    out[0] = table[x >> 18 & 0x3F];
+
+}
+
+
+/*
+ * Provide padding to already initialized b64 output
+ */
+static void b64_padding(char *out, int pad_size) {
+
+    if (pad_size > 0) {
+        *out++ = '=';
+    }
+
+    if (pad_size > 1) {
+        *out++ = '=';
+    }
+
+    *out = '\0';
+}
+
+
+/*
+ * Encode file contents as b64 string
+ */
+int nc_b64enc_file(FILE *fh, char *buff, size_t buff_size) {
+
+    int ret_val = 1;
+    int out_count = 0;
+    int pad_size = 0;
+    char *buff_ptr = buff;
+    char bin[3] = {0, 0, 0};
+    size_t len = 0;
+
+    while ((len = fread(bin, 1, 3, fh)) > 0) {
+        b64_3b(bin, buff_ptr);
+        /*
+         Reset 2nd and 3rd bytes in case only one byte was read. If these
+         bytes are not clean b64 output will be corrupted with old bits.
+         */
+        bin[1] = bin[2] = 0;
+        /*
+         Advance pointer to the next free position of the b64 encoded
+         buffer. This happens to be len + 1 (as longs as len >= 1 and len <= 3)
+
+            len = 1 byte  -> b64 -> 2 bytes (or len + 1)
+            len = 2 bytes -> b64 -> 3 bytes (or len + 1)
+            len = 3 bytes -> b64 -> 4 bytes (or len + 1)
+        */
+        buff_ptr += (len + 1);
+        /*
+         Line break every 76 characters
+        */
+        out_count += (int) (len + 1);
+        if (out_count % B64_LINE_LEN == 0) {
+            *buff_ptr++ = '\n';
+        }
+        pad_size = padding[len];
+        /*
+         Check not to overflow buffer
+
+            buffer size <= b64 characters + padding length + null termination
+        */
+        if (buff_size <= (size_t) (buff_ptr - buff) + (size_t) pad_size + 1) {
+           ret_val = 0;
+           goto end_b64_enc;
+        }
+    }
+
+    b64_padding(buff_ptr, pad_size);
+
+end_b64_enc:
+
+    return ret_val;
+}
+
+
+int nc_b64enc_filename(const char *filename, char *buff, size_t buff_size) {
+
+    int ret = 0;
+    FILE *fh = NULL;
+
+    fh = fopen(filename, "rb");
+
+    if (fh == NULL) {
+        goto nc_b64_clean;
+    }
+
+    ret = nc_b64enc_file(fh, buff, buff_size);
+
+nc_b64_clean:
+
+    if (fh) {
+        fclose(fh);
+    }
+
+    return ret;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/nica/b64enc.h

@@ -0,0 +1,42 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * This is a chained base64 encode implementation.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include "macros.h"
+
+/*
+ * Encodes contents of file handler in base64
+ *
+ * @param fh File handler to encode as base64
+ * @param buff Output buffer where the base64 output should be written
+ * @param buff_size Size of the output buffer
+ *
+ * @returns 1 in success and 0 in failure
+ */
+_nica_public_ int nc_b64enc_file(FILE *fh, char *buff, size_t buff_size);
+
+
+/*
+ * Encodes contents of file (filename) in base64
+ *
+ * @param filename File to encode as base64
+ * @param buff Output buffer where the base64 output should be written
+ * @param buff_size Size of the output buffer
+ *
+ * @returns 1 in success and 0 in failure
+ */
+_nica_public_ int nc_b64enc_filename(const char *filename, char *buff, size_t buff_size);
+

src/nica/inifile.c

@@ -151,33 +151,38 @@ static char *string_strip(char *str, ssize_t len, ssize_t *out_len)
 /* eat \s */
 static char *string_unescape(char *str)
 {
-	char *c, *c1, *c2, *c3;;
+        char *c, *c1, *c2, *c3;;
 
-	if (!str)
-		return NULL;
+        if (!str) {
+                return NULL;
+        }
 
-	c = calloc(strlen(str) + 1, 1);
-	c1 = c;
+        c = calloc(strlen(str) + 1, 1);
+        if (c == NULL) {
+                return NULL;
+        }
 
-	c2 = str;
-	while (c2 && *c2 != 0) {
-		c3 = c2 + 1;
-		if (*c2 == '\\') {
-			if (*c3 == 's') {
-				*c = ' ';
-				c++;
-				c2 += 2;
-			} else {
-				c2++;
-			}
-		} else {
-			*c = *c2;
-			c++;
-			c2++;
-		}
-	}
+        c1 = c;
 
-	free(str);	
+        c2 = str;
+        while (c2 && *c2 != 0) {
+                c3 = c2 + 1;
+                if (*c2 == '\\') {
+                        if (*c3 == 's') {
+                                *c = ' ';
+                                c++;
+                                c2 += 2;
+                        } else {
+                                c2++;
+                        }
+                } else {
+                        *c = *c2;
+                        c++;
+                        c2++;
+                }
+        }
+
+        free(str);
         return c1;
 }
 
@@ -300,7 +305,14 @@ int nc_ini_file_parse_full(const char *path, NcHashmap **out_map, int *error_lin
                 key = strdup(buf);
                 key = string_chew_terminated(key);
                 value = string_chew_terminated(value);
-		value = string_unescape(value);
+                if (value != NULL) {
+                        value = string_unescape(value);
+                        if (value == NULL) {
+                                err_ret = NC_INI_ERROR_INTERNAL;
+                                fprintf(stderr, "[inifile] Fatal! Out of memory\n");
+                                goto fail;
+                        }
+                }
 
                 if (streq(key, "")) {
                         err_ret = NC_INI_ERROR_EMPTY_KEY;

src/post.c

@@ -0,0 +1,97 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <unistd.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+
+#include "telemetry.h"
+#include "log.h"
+#include "spool.h"
+#include "configuration.h"
+#include "telempostdaemon.h"
+
+/*
+ *  Using a function pointer for unit testing to isolate the call to actual post function.
+ *  The call to post the record to the server is stubbed out in the unit tests
+ *  using pointer to a fake function.
+ */
+
+bool (*post_record_ptr)(char *[], char *, char *) = post_record_http;
+
+void print_usage(char *prog)
+{
+        printf("%s: Usage\n", prog);
+        printf("  -f,  --config_file    Configuration file. This overides the other parameters\n");
+        printf("  -h,  --help           Display this help message\n");
+        printf("  -V,  --version        Print the program version\n");
+}
+
+int main(int argc, char **argv)
+{
+
+        int c;
+        int opt_index = 0;
+        TelemPostDaemon daemon;
+
+        struct option opts[] = {
+                { "config_file", 1, NULL, 'f' },
+                { "help", 0, NULL, 'h' },
+                { "version", 0, NULL, 'V' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "f:hV", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case 'h':
+                                print_usage(argv[0]);
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                print_usage(argv[0]);
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        initialize_post_daemon(&daemon);
+
+        daemon.current_spool_size = get_spool_dir_size();
+
+        /* When path activated this will process
+         * the activating message or previously
+         * spooled data */
+        staging_records_loop(&daemon);
+
+        /* This function is blocking, it will
+        * block until a signal is received */
+        run_daemon(&daemon);
+
+        close_daemon(&daemon);
+
+        return 0;
+}

src/probe.c

@@ -31,6 +31,7 @@
 #include <signal.h>
 #include <malloc.h>
 
+#include "telemetry.h"
 #include "config.h"
 #include "common.h"
 #ifdef  HAVE_SYSTEMD_SD_DAEMON_H
@@ -49,14 +50,6 @@
 #include "log.h"
 #include "telemdaemon.h"
 #include "configuration.h"
-#include "spool.h"
-
-/*
- *  Using a function pointer for unit testing to isolate the call to actual post function.
- *  The call to post the record to the server is stubbed out in the unit tests
- *  using pointer to a fake function.
- */
-bool (*post_record_ptr)(char *[], char *, bool) = post_record_http;
 
 void print_usage(char *prog)
 {
@@ -76,7 +69,6 @@ int main(int argc, char **argv)
         client *cl = NULL;
         client *current_client = NULL;
         int c;
-        char *config_file = NULL;
         int opt_index = 0;
         sigset_t mask;
         //bool interrupted = false;
@@ -94,18 +86,12 @@ int main(int argc, char **argv)
 
         while ((c = getopt_long(argc, argv, "f:hV", opts, &opt_index)) != -1) {
                 switch (c) {
-                        case 'f':
-                                config_file = optarg;
-                                struct stat buf;
-                                ret = stat(config_file, &buf);
-
-                                /* check if the file exists and is a regular file */
-                                if (ret == -1 || !S_ISREG(buf.st_mode)) {
-                                        telem_log(LOG_ERR, "Configuration file"
+                         case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
                                                   " path not valid\n");
-                                        exit(EXIT_FAILURE);
+                                    exit(EXIT_FAILURE);
                                 }
-                                set_config_file(config_file);
                                 break;
                         case 'h':
                                 print_usage(argv[0]);
@@ -119,7 +105,7 @@ int main(int argc, char **argv)
                                 exit(EXIT_FAILURE);
                 }
         }
-        initialize_daemon(&daemon);
+        initialize_probe_daemon(&daemon);
 
         sigemptyset(&mask);
 
@@ -220,19 +206,9 @@ int main(int argc, char **argv)
 
         telem_log(LOG_INFO, "Listening on socket...\n");
 
-        /* Calculate the spool directory size */
-        if (is_spool_valid()) {
-                daemon.is_spool_valid = true;
-                daemon.current_spool_size = get_spool_dir_size();
-        }
-
-        int spool_process_time = spool_process_time_config();
-
-        spool_records_loop(&(daemon.current_spool_size));
-        time_t last_spool_run_time = time(NULL);
-        time_t last_daemon_start_time = time(NULL);
-
         bool daemon_recycling_enabled = daemon_recycling_enabled_config();
+        int spool_process_time = spool_process_time_config();
+        time_t last_record_received = time(NULL);
 
         ret = update_machine_id();
         if (ret == -1) {
@@ -267,8 +243,8 @@ int main(int argc, char **argv)
                                         }
 
                                         if (fdsi.ssi_signo == SIGTERM || fdsi.ssi_signo == SIGINT) {
-                                                telem_log(LOG_INFO, "Received either a \
-                                                                     SIGINT/SIGTERM signal\n");
+                                                telem_log(LOG_INFO, "Received either a "
+                                                                     "SIGINT/SIGTERM signal\n");
                                                 goto clean_exit;
                                         }
 
@@ -276,12 +252,6 @@ int main(int argc, char **argv)
                                                 telem_log(LOG_INFO, "Received a SIGHUP signal\n");
                                                 /* reload configuration file */
                                                 reload_config();
-
-                                                if (is_spool_valid()) {
-                                                        daemon.is_spool_valid = true;
-                                                        //calc_spool_dir_size();
-                                                        daemon.current_spool_size = get_spool_dir_size();
-                                                }
                                         }
                                 }
 
@@ -333,25 +303,20 @@ int main(int argc, char **argv)
                                         }
                                         assert(current_client);
                                         handle_client(&daemon, i, current_client);
+                                        last_record_received = time(NULL);
                                 }
                         }
                 } else {
                         time_t now = time(NULL);
-                        /* If spool is empty and time to recycle the daemon has elapsed*/
-                        if (daemon_recycling_enabled && (daemon.current_spool_size == 0) &&
-                            difftime(now, last_daemon_start_time) >= TM_DAEMON_EXIT_TIME) {
+                        /* time to recycle the daemon has elapsed*/
+                        if (daemon_recycling_enabled &&
+                            difftime(now, last_record_received) >= TM_DAEMON_EXIT_TIME) {
                                 /* Exit */
                                 telem_log(LOG_INFO, "Daemon exiting for recycling\n");
                                 goto clean_exit;
                         }
                 }
-
                 time_t now = time(NULL);
-                if (difftime(now, last_spool_run_time) >= spool_process_time) {
-                        spool_records_loop(&(daemon.current_spool_size));
-                        last_spool_run_time = time(NULL);
-                }
-
                 if (difftime(now, last_refresh_time) >= TM_REFRESH_RATE) {
                         int ret = update_machine_id();
                         if (ret == -1) {
@@ -362,12 +327,13 @@ int main(int argc, char **argv)
         }
 
 clean_exit:
+
         /* Free memory before exiting */
         while ((cl = LIST_FIRST(&(daemon.client_head))) != NULL) {
                 remove_client(&(daemon.client_head), cl);
         }
         free(daemon.pollfds);
-
+        free(daemon.machine_id_override);
         if (LIST_EMPTY(&(daemon.client_head))) {
                 telem_log(LOG_INFO, "Client list cleared\n");
         }

src/probes/README.md

@@ -0,0 +1,53 @@
+# Probes
+
+The default probes provided along the telemetry client code are:
+
+* bertprobe: this probe reports on the Boot Error Region Table if such
+  entry is found in ```/sys/firmware/acpi/tables/BERT```.
+
+* crashprobe: This probe processes core dump files. It can be registered as
+  the kernel core file handler in /proc/sys/kernel/core_pattern.
+
+* hprobe: a simple probe that sends a keep alive message. This probe can also
+  be useful for testing
+
+* journalprobe: a probe that monitors the systemd journal for log messages
+  from failed services.
+
+* klogscanner: a probe to collect 'oops messages' when the kernel detects a
+  problem.
+
+* pstoreprobe: probe to collect messages left on pstore filesystem.
+
+* telem-record-gen: this is a "general-purpose probe" for sending custom
+  records on-the-fly. This tool can be used stand alone or as a part of a
+  script to implement a probe.
+
+For example to generate an hprobe like message we can execute the following
+command:
+
+```
+$ telem-record-gen --class org.example/hello/world --payload hello
+```
+
+for more options use the ```--help``` switch i.e.:
+
+```
+$ telem-record-gen --help
+Usage:
+  telem-record-gen [OPTIONS] - create and send a custom telemetry record
+
+Help Options:
+  -h, --help            Show help options
+
+Application Options:
+  -f, --config-file     Path to configuration file (not implemented yet)
+  -V, --version         Print the program version
+  -s, --severity        Severity level (1-4) - (default 1)
+  -c, --class           Classification level_1/level_2/level_3
+  -p, --payload         Record body (max size = 8k)
+  -P, --payload-file    File to read payload from
+  -R, --record-version  Version number for format of payload (default 1)
+  -e, --event-id        Event id to use in the record
+
+```

src/probes/bert_probe.c

@@ -0,0 +1,117 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <getopt.h>
+#include "common.h"
+#include "telemetry.h"
+#include "nica/b64enc.h"
+
+#include "config.h"
+#include "log.h"
+
+static const char bert_record_file[] = "/sys/firmware/acpi/tables/data/BERT";
+static const char telem_record_class[] = "org.clearlinux/bert/debug";
+static uint32_t severity = 2;
+static uint32_t payload_version = 1;
+
+void print_usage(char *prog)
+{
+        printf("%s: Usage\n", prog);
+        printf("  -f,  --config_file    Specify a configuration file other than default\n");
+        printf("  -h,  --help           Display this help message\n");
+        printf("  -V,  --version        Print the program version\n");
+}
+
+int main(int argc, char **argv)
+{
+        struct telem_ref *tm_handle = NULL;
+        char *classification = (char *)telem_record_class;
+        char *payload;
+        int ret;
+
+        // Following vars are for arg parsing.
+        int c;
+        int opt_index = 0;
+        struct option opts[] = {
+                { "config_file", 1, NULL, 'f' },
+                { "help", 0, NULL, 'h' },
+                { "version", 0, NULL, 'V' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "f:hV", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 'h':
+                                print_usage(argv[0]);
+                                exit(EXIT_SUCCESS);
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        payload = calloc(sizeof(char), MAX_PAYLOAD_LENGTH);
+        if (!payload) {
+                printf("Unable to allocate more memory.\n");
+                return -ENOMEM;
+        }
+
+        if (!(ret = nc_b64enc_filename(bert_record_file, payload, MAX_PAYLOAD_LENGTH))) {
+                printf("Failed to read payload from: %s\n", bert_record_file);
+                ret = EXIT_FAILURE;
+                goto done;
+        }
+
+        if ((ret = tm_create_record(&tm_handle, severity, classification,
+                                    payload_version)) < 0) {
+                printf("Failed to create record: %s\n", strerror(-ret));
+                goto done;
+        }
+
+        if ((ret = tm_set_payload(tm_handle, payload)) < 0) {
+                printf("Failed to set record payload: %s\n", strerror(-ret));
+                goto done;
+        }
+
+        if ((ret = tm_send_record(tm_handle)) < 0) {
+                printf("Failed to send record to daemon: %s\n", strerror(-ret));
+                goto done;
+        }
+
+        ret = EXIT_SUCCESS;
+done:
+        free(payload);
+        tm_free_record(tm_handle);
+        tm_handle = NULL;
+
+        return ret;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/crash_probe.c

@@ -41,6 +41,8 @@
 #include "probe.h"
 #include "telemetry.h"
 
+#define FRAMES_MAX 64
+
 static Dwfl *d_core = NULL;
 
 /*
@@ -68,6 +70,8 @@ static char clr_build_class[40] = "org.clearlinux/crash/clr-build";
 static char error_class[30] = "org.clearlinux/crash/error";
 static char unknown_class[30] = "org.clearlinux/crash/unknown";
 
+static char temp_core[] = "/tmp/corefile-XXXXXX";
+
 static const Dwfl_Callbacks cb =
 {
         .find_elf = dwfl_build_id_find_elf,
@@ -147,23 +151,14 @@ static int temp_core_file(void)
         int tmp;
         ssize_t ret;
 
-        char core[PATH_MAX] = "/tmp/corefile-XXXXXX";
-
         /* mkstemp() opens the file with O_EXCL and 0600 permissions, so no need
          * to change umask or manipulate the fd to meet those requirements.
          */
-        if ((tmp = mkstemp(core)) < 0) {
+        if ((tmp = mkstemp(temp_core)) < 0) {
                 telem_perror("Failed to create temp core file");
                 return -1;
         }
 
-#ifndef DEBUG
-        if (unlink(core) < 0) {
-                telem_perror("Failed to unlink temp core file");
-                return -1;
-        }
-#endif
-
         while (true) {
                 // Use Linux-specific splice(2) here;
                 // simplifies copying data from pipe->file
@@ -255,7 +250,7 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
                                " current frame: %s\n",
                                dwfl_errmsg(-1));
                 if (ret < 0) {
-                        return DWARF_CB_ABORT;
+                        errorstr = NULL;
                 }
                 return DWARF_CB_ABORT;
         }
@@ -305,6 +300,12 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
                 }
         }
         nc_string_append_printf(*bt, "\n");
+
+        if (frame_counter >= FRAMES_MAX) {
+                errorstr = NULL;
+                return DWARF_CB_ABORT;
+        }
+
         return DWARF_CB_OK;
 }
 
@@ -397,6 +398,12 @@ static int process_corefile(nc_string **backtrace)
         *backtrace = nc_string_dup("");
 
         if (dwfl_getthreads(d_core, thread_cb, backtrace) != DWARF_CB_OK) {
+                /* We aborted unwinding, due to too many frames.
+                 * We don't consider this as an error.
+                 */
+                if (frame_counter >= FRAMES_MAX) {
+                        return 0;
+                }
                 /* When errors occur during the unwinding, we reach this point.
                  * If an error string is set for the particular error, send an
                  * "error" record to capture at least a partial backtrace if
@@ -416,6 +423,18 @@ fail:
         return -1;
 }
 
+static bool startswith(const char *full, const char *prefix)
+{
+        while (*prefix) {
+                if (*prefix != *full) {
+                        return false;
+                }
+                full++;
+                prefix++;
+        }
+        return true;
+}
+
 static bool in_clr_build(char *fullpath)
 {
         // Global override for privacy filters
@@ -427,8 +446,8 @@ static bool in_clr_build(char *fullpath)
          * The build environment for Clear Linux packages is set up by 'mock',
          * and the chroot in which rpmbuild builds the packages has this prefix.
          */
-        if ((strstr(fullpath, "/builddir/build/BUILD/")) ||
-            (strstr(fullpath, "!builddir!build!BUILD!"))) {
+        if (startswith(fullpath, "/builddir/build/BUILD/") ||
+            startswith(fullpath, "!builddir!build!BUILD!")) {
                 return true;
         }
 
@@ -444,20 +463,19 @@ static bool is_banned_path(char *fullpath)
 
         // Anything outside of /usr/, or in /usr/local/, we consider third-party
 
-        if ((strncmp(fullpath, "/usr/", 5) != 0) &&
-            (strncmp(fullpath, "!usr!", 5) != 0)) {
+        if (!startswith(fullpath, "/usr/") &&
+            !startswith(fullpath, "!usr!")) {
                 return true;
         }
 
-        if ((strncmp(fullpath, "/usr/local/", 11) == 0) ||
-            (strncmp(fullpath, "!usr!local!", 11) == 0)) {
+        if (startswith(fullpath, "/usr/local/") ||
+            startswith(fullpath, "!usr!local!")) {
                 return true;
         }
 
         return false;
 }
 
-static char *config_file = NULL;
 static char *core_file = NULL;
 static char *proc_path = NULL;
 static long int signal_num = -1;
@@ -484,7 +502,7 @@ static void print_help(void)
         printf("  -h, --help            Show help options\n");
         printf("\n");
         printf("Application Options:\n");
-        printf("  -f, --config-file     Path to configuration file (not implemented yet)\n");
+        printf("  -f, --config_file     Specify a configuration file other than default\n");
         printf("  -c, --core-file       Path to core file to process\n");
         printf("  -p, --process-name    Name of process for crash report (required)\n");
         printf("  -E, --process-path    Absolute path of crashed process, with ! or / delimiters\n");
@@ -526,7 +544,11 @@ int main(int argc, char **argv)
                                 printf(PACKAGE_VERSION "\n");
                                 goto success;
                         case 'f':
-                                config_file = strdup(optarg);
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
                                 break;
                         case 'c':
                                 core_file = strdup(optarg);
@@ -563,28 +585,6 @@ int main(int argc, char **argv)
                 exit(EXIT_FAILURE);
         }
 
-        if (proc_path && in_clr_build(proc_path)) {
-                telem_log(LOG_NOTICE, "Ignoring core (from mock build)\n");
-
-                backtrace = nc_string_dup("Crash from Clear package build\n");
-
-                if (!send_data(&backtrace, unknown_severity, clr_build_class)) {
-                        goto fail;
-                }
-                goto success;
-        }
-
-        if (proc_path && is_banned_path(proc_path)) {
-                telem_log(LOG_NOTICE, "Ignoring core (third-party binary)\n");
-
-                backtrace = nc_string_dup("Crash from third party\n");
-
-                if (!send_data(&backtrace, unknown_severity, unknown_class)) {
-                        goto fail;
-                }
-                goto success;
-        }
-
         if (core_file) {
                 core_fd = open(core_file, O_RDONLY);
                 if (core_fd == -1) {
@@ -617,6 +617,28 @@ int main(int argc, char **argv)
                 }
         }
 
+        if (proc_path && in_clr_build(proc_path)) {
+                telem_log(LOG_NOTICE, "Ignoring core (from mock build)\n");
+
+                backtrace = nc_string_dup("Crash from Clear package build\n");
+
+                if (!send_data(&backtrace, unknown_severity, clr_build_class)) {
+                        goto fail;
+                }
+                goto success;
+        }
+
+        if (proc_path && is_banned_path(proc_path)) {
+                telem_log(LOG_NOTICE, "Ignoring core (third-party binary)\n");
+
+                backtrace = nc_string_dup("Crash from third party\n");
+
+                if (!send_data(&backtrace, unknown_severity, unknown_class)) {
+                        goto fail;
+                }
+                goto success;
+        }
+
         elf_version(EV_CURRENT);
 
         if (prepare_corefile(&e_core, core_fd) < 0) {
@@ -631,16 +653,12 @@ int main(int argc, char **argv)
                 nc_string_append_printf(header, "Signal: %ld\n", signal_num);
         }
 
-        if (process_corefile(&backtrace) < 0) {
-                goto fail;
-        }
-
         /* On Clear Linux OS, missing symbols may appear if automatic debuginfo
          * downloads are still in flight. So if any missing symbols appear on
          * the first run (indicated by the presence of "??? - ["), wait 10
-         * seconds and try again.
+         * seconds and try again. Also retry if errors occur in the first run.
          */
-        if (strstr(backtrace->str, "??? - [")) {
+        if ((process_corefile(&backtrace) < 0) || (strstr(backtrace->str, "??? - ["))) {
                 sleep(10);
 
                 if (prepare_corefile(&e_core, core_fd) < 0) {
@@ -652,6 +670,11 @@ int main(int argc, char **argv)
                 }
         }
 
+        if (frame_counter >= FRAMES_MAX) {
+                telem_log(LOG_ERR, "Too many frames. Backtrace truncated.\n");
+                nc_string_append_printf(header, "Too many frames. Backtrace truncated.\n");
+        }
+
         nc_string_prepend(backtrace, header->str);
 
         if (!send_data(&backtrace, default_severity, clr_class)) {
@@ -695,6 +718,12 @@ fail:
                 close(core_fd);
         }
 
+        // Remove the core file by default, except when the --core-file option
+        // is specified.
+        if (!core_file) {
+                unlink(temp_core);
+        }
+
         return ret;
 }
 

src/probes/hello.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -20,63 +20,156 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <getopt.h>
+#include <locale.h>
+#include <sys/sysinfo.h>
+#include <dirent.h>
+#include "common.h"
 #include "telemetry.h"
-
 #include "config.h"
+#include "log.h"
 
-void print_usage(char *prog)
+#define PAYLOAD_LOCALE  0x01
+#define PAYLOAD_UPTIME  0x02
+#define PAYLOAD_BUNDLES 0x04
+
+static void print_usage(char *prog)
 {
         printf("%s: Usage\n", prog);
         printf("  -f,  --config_file    Configuration file. This overides the other parameters\n");
         printf("  -H,  --heartbeat      Create a \"heartbeat\" record instead\n");
         printf("  -h,  --help           Display this help message\n");
+        printf("  -l,  --locale         Include locale in the record\n");
+        printf("  -u,  --uptime         Include uptime in the record\n");
+        printf("  -b,  --bundles        Include installed bundles in the record\n");
         printf("  -V,  --version        Print the program version\n");
 }
 
+static long get_uptime(void)
+{
+        struct sysinfo s_info;
+        if (sysinfo(&s_info) == 0) {
+                return s_info.uptime;
+        }
+        return 0;
+}
+
+static int nodots(const struct dirent *dp)
+{
+        return (dp->d_name[0] != '.');
+}
+
+static char *create_payload(unsigned options)
+{
+        int n, buflen;
+        char *ppos, *payload;
+
+        payload = calloc(sizeof(char), MAX_PAYLOAD_LENGTH);
+        if (payload == NULL) {
+                return NULL;
+        }
+
+        ppos = payload;
+        buflen = MAX_PAYLOAD_LENGTH;
+
+        n = snprintf(ppos, (size_t)buflen, "hello\n");
+        buflen -= n, ppos += n;
+
+        if (options & PAYLOAD_LOCALE) {
+                n = snprintf(ppos, (size_t)buflen, "LC_ALL: %s\n", setlocale(LC_ALL, NULL));
+                buflen -= n, ppos += n;
+        }
+
+        if (options & PAYLOAD_UPTIME) {
+                n = snprintf(ppos, (size_t)buflen, "uptime: %ld\n", get_uptime());
+                buflen -= n, ppos += n;
+        }
+
+        if (options & PAYLOAD_BUNDLES) {
+                int num_bundles, i;
+                struct dirent **entries;
+
+                num_bundles = scandir("/usr/share/clear/bundles", &entries, nodots, alphasort);
+                if (num_bundles < 0) {
+                        telem_perror("scandir failed");
+                        free(payload);
+                        return NULL;
+                }
+                n = snprintf(ppos, (size_t)buflen, "\nBundles (%d):\n", num_bundles);
+                buflen -= n, ppos += n;
+
+                for (i = 0; i < num_bundles; i++) {
+                        n = snprintf(ppos, (size_t)buflen, "%s\n", entries[i]->d_name);
+                        /* Test if truncated output */
+                        if (n >= buflen) {
+                                int lastix = MAX_PAYLOAD_LENGTH-1;
+                                payload[lastix] = '\0';
+                                payload[lastix-1] = '.';
+                                payload[lastix-2] = '.';
+                                payload[lastix-3] = '.';
+                                payload[lastix-4] = '\n';
+                                break;
+                        }
+                        buflen -= n, ppos += n;
+                }
+
+                for (i= 0; i < num_bundles; i++) {
+                        free(entries[i]);
+                }
+
+                free(entries);
+        }
+
+        return payload;
+}
+
 int main(int argc, char **argv)
 {
         uint32_t severity = 1;
         uint32_t payload_version = 1;
         char classification[30] = "org.clearlinux/hello/world";
         struct telem_ref *tm_handle = NULL;
-        char *payload2;
+        char *payload;
         char *str;
+        int ret;
+        unsigned payload_options = 0;
 
         // Following vars are for arg parsing.
         int c;
-        char *cfile = NULL;
         int opt_index = 0;
-        int ret = 0;
 
         struct option opts[] = {
-                { "cfile", 1, NULL, 'f' },
+                { "config_file", 1, NULL, 'f' },
                 { "heartbeat", 0, NULL, 'H' },
                 { "help", 0, NULL, 'h' },
                 { "version", 0, NULL, 'V' },
+                { "locale", 0, NULL, 'l' },
+                { "uptime", 0, NULL, 'u' },
+                { "bundles", 0, NULL, 'b' },
                 { NULL, 0, NULL, 0 }
         };
 
-        while ((c = getopt_long(argc, argv, "f:hHV", opts, &opt_index)) != -1) {
+        while ((c = getopt_long(argc, argv, "f:hHVlub", opts, &opt_index)) != -1) {
                 switch (c) {
                         case 'f':
-                                if (asprintf(&cfile, "%s", (char *)optarg) < 0) {
-                                        exit(EXIT_FAILURE);
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
                                 }
-
-                                struct stat buf;
-                                ret = stat(cfile, &buf);
-
-                                /* check if the file exists and is a regular file */
-                                if (ret == -1 || !S_ISREG(buf.st_mode)) {
-                                        exit(EXIT_FAILURE);
-                                }
-
-                                tm_set_config_file(cfile);
                                 break;
                         case 'H':
                                 str = "org.clearlinux/heartbeat/ping";
                                 memcpy(classification, str, strlen(str));
                                 break;
+                        case 'l':
+                                payload_options |= PAYLOAD_LOCALE;
+                                break;
+                        case 'u':
+                                payload_options |= PAYLOAD_UPTIME;
+                                break;
+                        case 'b':
+                                payload_options |= PAYLOAD_BUNDLES;
+                                break;
                         case 'h':
                                 print_usage(argv[0]);
                                 exit(EXIT_SUCCESS);
@@ -88,34 +181,39 @@ int main(int argc, char **argv)
                 }
         }
 
-        if (asprintf(&payload2, "hello\n") < 0) {
+        setlocale(LC_ALL, "");
+        payload = create_payload(payload_options);
+        if (payload == NULL) {
                 exit(EXIT_FAILURE);
         }
 
         if ((ret = tm_create_record(&tm_handle, severity, classification,
                                     payload_version)) < 0) {
-                printf("Failed to create record: %s\n", strerror(-ret));
-                ret = 1;
-                goto fail;
+                telem_log(LOG_ERR, "Failed to create record: %s\n",
+                       strerror(-ret));
+                ret = EXIT_FAILURE;
+                goto done;
         }
 
-        if ((ret = tm_set_payload(tm_handle, payload2)) < 0) {
-                printf("Failed to set record payload: %s\n", strerror(-ret));
-                ret = 1;
-                goto fail;
+        if ((ret = tm_set_payload(tm_handle, payload)) < 0) {
+                telem_log(LOG_ERR, "Failed to set record payload: %s\n",
+                        strerror(-ret));
+                ret = EXIT_FAILURE;
+                goto done;
         }
 
-        free(payload2);
+        free(payload);
 
         if ((ret = tm_send_record(tm_handle)) < 0) {
-                printf("Failed to send record to daemon: %s\n", strerror(-ret));
-                ret = 1;
-                goto fail;
+                telem_log(LOG_ERR, "Failed to send record to daemon: %s\n",
+                        strerror(-ret));
+                ret = EXIT_FAILURE;
+                goto done;
         } else {
                 printf("Successfully sent record to daemon.\n");
-                ret = 0;
+                ret = EXIT_SUCCESS;
         }
-fail:
+done:
         tm_free_record(tm_handle);
         tm_handle = NULL;
 

src/probes/journal.c

@@ -38,6 +38,7 @@
 #include "config.h"
 #include "log.h"
 #include "telemetry.h"
+#include "probe.h"
 #include "nica/nc-string.h"
 #define BOOT_ID_LEN 33
 
@@ -227,9 +228,14 @@ static bool add_filters(sd_journal *journal)
         /* The semantics of how journal entry matching works is described in
          * detail in sd_journal_add_match(3).
          *
-         * The matches declared here correspond to the logical expression:
+         * When the privacy filter override is enabled, the matches declared
+         * here correspond to this logical expression:
          *
-         * BOOTID && ((P0 || P1 || P2 || P3) || EXITED)
+         *   BOOTID && ((P0 || P1 || P2 || P3) || EXITED)
+         *
+         * Otherwise, the expression is:
+         *
+         *   BOOTID && EXITED
          *
          * BOOTID is short for _BOOT_ID=VAL, where VAL is the boot ID for the
          * current boot. P0, P1, etc stand for PRIORITY=0, etc. And EXITED is
@@ -239,19 +245,27 @@ static bool add_filters(sd_journal *journal)
         JOURNAL_MATCH(data);
         free(data);
         JOURNAL_AND;
-        // The four highest log levels, all indicating errors
-        JOURNAL_MATCH("PRIORITY=0");
-        JOURNAL_MATCH("PRIORITY=1");
-        JOURNAL_MATCH("PRIORITY=2");
-        JOURNAL_MATCH("PRIORITY=3");
-        JOURNAL_OR;
+
+        // Filter messages with the four highest log levels, all indicating
+        // errors, but only when the privacy filters override is in effect;
+        // because the log messages contain arbitrary strings, and this probe
+        // does not yet keep a whitelist of allowed patterns or a blacklist of
+        // banned patterns.
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                JOURNAL_MATCH("PRIORITY=0");
+                JOURNAL_MATCH("PRIORITY=1");
+                JOURNAL_MATCH("PRIORITY=2");
+                JOURNAL_MATCH("PRIORITY=3");
+                JOURNAL_OR;
+        }
+
         // Only set for service-level error conditions
         JOURNAL_MATCH("EXIT_CODE=exited");
 
         return true;
 }
 
-static bool process_journal(void)
+static bool process_journal(bool process_existing)
 {
         int r;
 
@@ -265,8 +279,14 @@ static bool process_journal(void)
                 return false;
         }
 
-        if (!process_existing_entries(journal)) {
-                return false;
+        if (process_existing) {
+                if (!process_existing_entries(journal)) {
+                        return false;
+                }
+        } else {
+                if ((sd_journal_seek_tail(journal) < 0) || (sd_journal_previous(journal) < 0)) {
+                        return false;
+                }
         }
 
         struct pollfd pfd;
@@ -298,12 +318,11 @@ static bool process_journal(void)
         }
 }
 
-static char *config_file = NULL;
-
 static const struct option prog_opts[] = {
         { "help", no_argument, 0, 'h' },
         { "config-file", required_argument, 0, 'f' },
         { "version", no_argument, 0, 'V' },
+        { "tail", no_argument, 0, 't' },
         { 0, 0, 0, 0 }
 };
 
@@ -316,22 +335,19 @@ static void print_help(void)
         printf("  -h, --help            Show help options\n");
         printf("\n");
         printf("Application Options:\n");
-        printf("  -f, --config-file     Path to configuration file (not implemented yet)\n");
+        printf("  -f, --config_file     Specify a configuration file other than default\n");
+        printf("  -t, --tail            Don't process entries already in the journal\n");
         printf("  -V, --version         Print the program version\n");
         printf("\n");
 }
 
-static void free_strings(void)
-{
-        free(config_file);
-}
-
 int main(int argc, char **argv)
 {
         int ret = EXIT_FAILURE;
         int opt;
+        bool process_existing = true;
 
-        while ((opt = getopt_long(argc, argv, "hf:V", prog_opts, NULL)) != -1) {
+        while ((opt = getopt_long(argc, argv, "hf:Vt", prog_opts, NULL)) != -1) {
                 switch (opt) {
                         case 'h':
                                 print_help();
@@ -340,20 +356,25 @@ int main(int argc, char **argv)
                                 printf(PACKAGE_VERSION "\n");
                                 goto success;
                         case 'f':
-                                config_file = strdup(optarg);
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 't':
+                                process_existing = false;
                                 break;
                 }
         }
 
-        if (!process_journal()) {
+        if (!process_journal(process_existing)) {
                 goto fail;
         }
 
 success:
         ret = EXIT_SUCCESS;
 fail:
-        free_strings();
-
         if (journal) {
                 sd_journal_close(journal);
         }

src/probes/klog_main.c

@@ -1,9 +1,24 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright © 2015-2019 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
 #define _GNU_SOURCE
 
 #include <stdlib.h>
 #include <stdio.h>
 #include <sys/klog.h>
-#include <fcntl.h>
 #include <errno.h>
 #include <unistd.h>
 #include <malloc.h>
@@ -22,54 +37,48 @@ int main(void)
         int log_size = 0;
         char *bufp = NULL;
         size_t buflen = 0;
-        int bytes = 0;
 
-        oops_parser_init(write_oops_to_file);
+        oops_parser_init(klog_process_oops_msgs);
 
-        // Allocate space to write the oops file to
-        if (allocate_filespace() < 0) {
-                telem_log(LOG_ERR, "%s\n", strerror(errno));
-                return 1;
-        }
-
-        // Signal Handling to close and unlink empty files
-        if (signal (SIGINT, signal_handler) == SIG_ERR) {
+        // Signal Handling to terminate the probe.
+        if (signal (SIGINT, signal_handler_fail) == SIG_ERR) {
                 telem_log(LOG_ERR, "Error handling interrupt signal\n");
         }
-        if (signal (SIGTERM, signal_handler) == SIG_ERR) {
+        if (signal (SIGTERM, signal_handler_success) == SIG_ERR) {
                 telem_log(LOG_ERR, "Error handling terminating signal\n");
         }
 
-        while (1) {
-                // Gets the size of the kernel ring buffer
-                log_size = klogctl(SYSLOG_ACTION_SIZE_BUFFER, NULL, 0);
-                if (log_size < 0) {
-                        telem_log(LOG_ERR, "Cannot read size of kernel ring buffer: %s\n", strerror(errno));
-                        return 1;
-                }
-
-                buflen = (size_t)log_size;
-
-                if (buflen > MAX_BUF) {
-                        buflen = MAX_BUF;
-                }
-
-                // Gets the contents of the kernel ring buffer
-                bufp = (char *)calloc(buflen, sizeof(char));
-
-                malloc_trim(0);
-                bytes = klogctl(SYSLOG_ACTION_READ, bufp, (int)buflen);
-                if (bytes < 0) {
-                        telem_log(LOG_ERR, "Cannot read contents of kernel ring buffer: %s\n", strerror(errno));
-                        return 1;
-                }
-
-                // Splits the buffer into separate lines with /0 terminating
-                split_buf_by_line(bufp, (size_t)bytes);
-
-                free(bufp);
+        // Gets the size of the kernel ring buffer
+        log_size = klogctl(SYSLOG_ACTION_SIZE_BUFFER, NULL, 0);
+        if (log_size < 0) {
+                telem_perror("Cannot read size of kernel ring buffer");
+                return 1;
         }
 
+        buflen = (size_t)log_size;
+        if (buflen > MAX_BUF) {
+                buflen = MAX_BUF;
+        }
+
+        // Gets the contents of the kernel ring buffer
+        bufp = (char *)malloc(buflen);
+
+        while (1) {
+                int bytes_read;
+ 
+                malloc_trim(0);
+                memset(bufp, 0, buflen);
+                bytes_read = klogctl(SYSLOG_ACTION_READ, bufp, (int)buflen);
+                if (bytes_read < 0) {
+                        telem_perror("Cannot read contents of kernel ring buffer");
+                        return 1;
+                }
+
+                klog_process_buffer(bufp, (size_t)bytes_read);
+        }
+
+        // Not reached
+        free(bufp);
         return 0;
 }
 

src/probes/klog_scanner.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright © 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -15,11 +15,10 @@
  */
 
 #define _GNU_SOURCE
-
 #include <stdlib.h>
 #include <stdio.h>
+#include <sys/types.h>
 #include <sys/klog.h>
-#include <fcntl.h>
 #include <errno.h>
 #include <unistd.h>
 #include <sys/stat.h>
@@ -27,20 +26,54 @@
 #include "log.h"
 #include "oops_parser.h"
 #include "klog_scanner.h"
+#include "telemetry.h"
+#include "nica/nc-string.h"
 
-#define MAX_OOPS_MSG_LENGTH 1024
-#define FILE_PATH KERNELOOPSDIR "/kerneloopsXXXXXX"
+static bool oops_processed = false;
 
-static int fd;
-char filename[] = FILE_PATH;
-int last_line_length;
+static uint32_t version = 1;
+
+static bool send_data(char *backtrace, char *class, uint32_t severity)
+{
+        struct telem_ref *handle = NULL;
+        int ret;
+
+        if ((ret = tm_create_record(&handle, severity, class, version)) < 0) {
+                telem_log(LOG_ERR, "Failed to create record: %s",
+                          strerror(-ret));
+                return false;
+        }
+
+        if ((ret = tm_set_payload(handle, backtrace)) < 0) {
+                telem_log(LOG_ERR, "Failed to set payload: %s", strerror(-ret));
+                tm_free_record(handle);
+                return false;
+        }
+
+        if ((ret = tm_send_record(handle)) < 0) {
+                telem_log(LOG_ERR, "Failed to send record: %s", strerror(-ret));
+                tm_free_record(handle);
+                return false;
+        }
+
+        tm_free_record(handle);
+
+        return true;
+}
+
+int klog_process_buffer(char *bufp, int bytes)
+{
+        // Splits the buffer into separate lines with /0 terminating
+        split_buf_by_line(bufp, (size_t)bytes);
+        return 0;
+}
 
-bool oops_written = false;
 void split_buf_by_line(char *bufp, int bytes)
 {
         char *start;
         char *eol;
         size_t linelength = 0;
+        oops_processed = false;
 
         while (bytes > 0) {
                 eol = memchr(bufp, '\n', (size_t)bytes);
@@ -57,87 +90,77 @@ void split_buf_by_line(char *bufp, int bytes)
                 if (bytes >= 0) {
                         start[linelength] = '\0';
                         parse_single_line(start, linelength);
+                        if (oops_processed == true) {
+                                telem_debug("oops_processed detected !\n");
+                                break;
+                        }
                         linelength = 0;
                 }
         }
 }
 
-int allocate_filespace()
+void klog_process_oops_msgs(struct oops_log_msg *msg)
 {
-        int allocate_disk = 0;
-        strcpy(filename, FILE_PATH);
-
-        fd = mkstemp(filename);
-        if (fchmod(fd, 0644) == -1) {
-                telem_log(LOG_ERR, "Error while changing permissions for file %s: %s\n",
-                          filename, strerror(errno));
-        }
-
-        if (fd == -1) {
-                telem_log(LOG_ERR, "fd error: %s\n", strerror(errno));
-                return -1;
-        }
-
-        // Allocates disk space to write kernel oops to
-        allocate_disk = fallocate(fd, 0, 0, MAX_OOPS_MSG_LENGTH);
-        if (allocate_disk != 0) {
-                telem_log(LOG_ERR, "Error allocating disk space: %s\n", strerror(errno));
-                return -1;
-        }
-        return 0;
-}
-
-void write_oops_to_file(struct oops_log_msg *msg)
-{
-        ssize_t writer;
-        size_t linelength;
+        size_t linelength, size;
         char *line;
+        char *contents = NULL;
+        nc_string *payload;
 
+        // pass 1: calculate buffer size
+        size = 0;
+        for (int i = 0; i < msg->length; i++) {
+                size += strlen(msg->lines[i]) + 1;
+        }
+
+        contents = malloc(size);
+        if (!contents) {
+                telem_log(LOG_ERR, "Call to malloc failed");
+                oops_processed = true;
+                return;
+        }
+
+        // pass 2: copy strings into the buffer
+        size_t done = 0;
+        char *bp = contents;
         for (int i = 0; i < msg->length; i++) {
                 //Add the newline character to the end of each line
                 line = msg->lines[i];
                 linelength = strlen(line);
-                line[linelength] = '\n';
-                linelength++;
+                // Copy the line (without the terminating NULL)
+                memcpy(bp, line, linelength);
+                bp[linelength] = '\n';
+                done += linelength + 1;
+                bp = contents + done;
+        }
 
-                // Write each line of oops to file
-                writer = write(fd, line, linelength);
-                if (writer == -1) {
-                        telem_log(LOG_ERR, "Error writing buffer to file: %s\n", strerror(errno));
-                        exit(EXIT_FAILURE);
+        struct oops_log_msg oops_msg;
+        if (handle_entire_oops(contents, (long)size, &oops_msg)) {
+#ifdef DEBUG
+                telem_debug("DEBUG: Raw oops message:\n");
+                for (int i = 0; i < oops_msg.length; i++) {
+                        telem_log(LOG_DEBUG, "%s\n", oops_msg.lines[i]);
                 }
+#endif
+                payload = parse_payload(&oops_msg);
+                telem_debug("DEBUG: Payload Parsed :%s\n", payload->str);
+                oops_msg_cleanup(&oops_msg);
+                send_data(payload->str, (char *)oops_msg.pattern->classification, (uint32_t)oops_msg.pattern->severity);
+                nc_string_free(payload);
+        } else {
+                /* File does no contain an oops! */
+                telem_log(LOG_ERR, "Did not find an oops in the oops directory\n");
         }
 
-        telem_log(LOG_INFO, "Oops written to file\n");
-        oops_written = true;
-        close(fd);
-
-        // Allocate more space to prepare for another oops
-        if (allocate_filespace() < 0) {
-                telem_log(LOG_ERR, "%s\n", strerror(errno));
-                exit(EXIT_FAILURE);
-        }
+        free(contents);
+        oops_processed = true;
 }
 
-bool get_oops_written(void)
+void signal_handler_success(int signum)
 {
-        return oops_written;
+        exit(EXIT_SUCCESS);
 }
 
-char *get_file_name(void)
-{
-        return filename;
-}
-
-__attribute__((destructor))
-void clean_up(void)
-{
-        // Unlink empty file and close any open file
-        unlink(filename);
-        close(fd);
-}
-
-void signal_handler(int signum)
+void signal_handler_fail(int signum)
 {
         exit(EXIT_FAILURE);
 }

src/probes/klog_scanner.h

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright © 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -17,45 +17,41 @@
 #pragma once
 #include "oops_parser.h"
 
+/**
+ * Process the buffer and send it to the backend
+ *
+ * @param *bufp A pointer to a buffer containing the bytes read
+ *		from the kernel ring buffer
+ * @param bytes Number of bytes read from ring buffer into bufp
+ * 
+ * @return 0 on succes, -1 on failure
+ *
+ */
+int klog_process_buffer(char *bufp, int bytes);
 /**
  * Splits the buffer into individual lines
  *
  * @param *bufp A pointer to a buffer containing the bytes read
  *		from the kernel ring buffer
- * @param bytes Bytes read from ring buffer into bufp
+ * @param bytes Number of bytes read from ring buffer into bufp
  *
  */
 void split_buf_by_line(char *bufp, int bytes);
 
 /**
- * Allocates memory for oops message to write to file
- *
- * @return 0 on succes, -1 on failure
- */
-int allocate_filespace(void);
-
-/**
- * Writes the oops message to tmp file
+ * Process the oops message and send it to the backend
  *
  * @param msg Struct that contains oops message lines and linecount
  *
  */
-void write_oops_to_file(struct oops_log_msg *msg);
-
+void klog_process_oops_msgs(struct oops_log_msg *msg);
 /**
  * Signal handler for cleanup purposes
  *
  * @param signum An integer for the signal type
  *
  */
-void signal_handler(int signum);
-
-/**
- * For Unit Tests Purposes
- * Determines whether an oops has been written to a file
- *
- * @return true if oops is written, false if not
- */
-bool get_oops_written(void);
+void signal_handler_fail(int signum);
+void signal_handler_success(int signum);
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/local.mk

@@ -4,15 +4,38 @@ bin_PROGRAMS += \
 	%D%/telem-record-gen \
 	%D%/klogscanner \
 	%D%/pstoreprobe \
-	%D%/oopsprobe \
-	%D%/pstoreclean
+	%D%/pstoreclean \
+	%D%/bertprobe
 
 %C%_hprobe_SOURCES = %D%/hello.c
 %C%_hprobe_LDADD = $(top_builddir)/src/libtelemetry.la
+%C%_hprobe_CFLAGS = $(AM_CFLAGS)
 %C%_hprobe_LDFLAGS = \
 	$(AM_LDFLAGS) \
 	-pie
 
+if LOG_SYSTEMD
+if HAVE_SYSTEMD_JOURNAL
+%C%_hprobe_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_hprobe_LDADD += $(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
+%C%_bertprobe_SOURCES = %D%/bert_probe.c \
+	src/nica/b64enc.c
+%C%_bertprobe_CFLAGS = $(AM_CFLAGS)
+%C%_bertprobe_LDADD = $(top_builddir)/src/libtelemetry.la
+%C%_bertprobe_LDFLAGS = \
+        $(AM_LDFLAGS) \
+        -pie
+
+if LOG_SYSTEMD
+if HAVE_SYSTEMD_JOURNAL
+%C%_bertprobe_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_bertprobe_LDADD += $(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
 %C%_telem_record_gen_SOURCES = %D%/telem_record_gen.c
 %C%_telem_record_gen_CFLAGS = \
 	$(AM_CFLAGS)
@@ -21,6 +44,14 @@ bin_PROGRAMS += \
 	$(AM_LDFLAGS) \
 	-pie
 
+if LOG_SYSTEMD
+if HAVE_SYSTEMD_JOURNAL
+%C%_telem_record_gen_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_telem_record_gen_LDADD += $(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
+
 %C%_pstoreclean_SOURCES = %D%/pstore_clean.c
 %C%_pstoreclean_CFLAGS = \
         $(AM_CFLAGS)
@@ -99,13 +130,6 @@ endif
         -pie
 
 if LOG_SYSTEMD
-if HAVE_SYSTEMD_JOURNAL
-%C%_klogscanner_CFLAGS += \
-        $(SYSTEMD_JOURNAL_CFLAGS)
-%C%_klogscanner_LDADD += \
-        $(SYSTEMD_JOURNAL_LIBS)
-endif
-
 if HAVE_SYSTEMD_JOURNAL
 %C%_klogscanner_CFLAGS += \
         $(SYSTEMD_JOURNAL_CFLAGS)
@@ -114,25 +138,6 @@ if HAVE_SYSTEMD_JOURNAL
 endif
 endif
 
-%C%_oopsprobe_SOURCES = \
-        %D%/oops_probe.c \
-	%D%/oops_parser.c \
-	src/nica/nc-string.c \
-	%D%/probe.h
-%C%_oopsprobe_CFLAGS = \
-        $(AM_CFLAGS)
-%C%_oopsprobe_LDADD = \
-        $(top_builddir)/src/libtelemetry.la \
-        $(top_builddir)/src/libtelem-shared.la
-
-if LOG_SYSTEMD
-if HAVE_SYSTEMD_JOURNAL
-%C%_oopsprobe_CFLAGS += \
-        $(SYSTEMD_JOURNAL_CFLAGS)
-%C%_oopsprobe_LDADD += \
-        $(SYSTEMD_JOURNAL_LIBS)
-endif
-endif
 
 if HAVE_SYSTEMD_JOURNAL
 if HAVE_SYSTEMD_ID128
@@ -154,6 +159,10 @@ bin_PROGRAMS += \
 %C%_journalprobe_LDFLAGS = \
 	$(AM_LDFLAGS) \
 	-pie
+if LOG_SYSTEMD
+%C%_journalprobe_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_journalprobe_LDADD += $(SYSTEMD_JOURNAL_LIBS)
+endif
 endif
 endif
 

src/probes/oops_parser.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -150,11 +150,23 @@ struct oops_pattern oops_patterns_arr[] = {
                 TM_MEDIUM,
                 false,
         },
+        {
+                "ACPI Error:",
+                "org.clearlinux/kernel/warning",
+                TM_MEDIUM,
+                false,
+        },
+        {
+                "Kernel panic - not syncing:",
+                "org.clearlinux/kernel/panic",
+                TM_CRITICAL,
+                false,
+        },
 };
 
-int oops_patterns_cnt = sizeof(oops_patterns_arr) / sizeof(struct oops_pattern);
+static int oops_patterns_cnt = sizeof(oops_patterns_arr) / sizeof(struct oops_pattern);
 
-char *skip_log_level(char *line)
+static char *skip_log_level(char *line)
 {
         char *start = line;
 
@@ -169,7 +181,7 @@ char *skip_log_level(char *line)
         return start;
 }
 
-char *skip_timestamp(char *line)
+static char *skip_timestamp(char *line)
 {
         char *start = line;
 
@@ -184,7 +196,7 @@ char *skip_timestamp(char *line)
         return start;
 }
 
-char *skip_space(char *line)
+static char *skip_space(char *line)
 {
         char *start = line;
 
@@ -194,7 +206,7 @@ char *skip_space(char *line)
         return start;
 }
 
-char *skip_spaces(char *line)
+static char *skip_spaces(char *line)
 {
         char *start = line;
 
@@ -204,7 +216,7 @@ char *skip_spaces(char *line)
         return start;
 }
 
-bool starts_with(const char *line, const char *line_end, const char *substr)
+static bool starts_with(const char *line, const char *line_end, const char *substr)
 {
         size_t substrlen = strlen(substr);
 
@@ -214,7 +226,7 @@ bool starts_with(const char *line, const char *line_end, const char *substr)
         return false;
 }
 
-bool str_starts_with_casei(const char *line, const char *substr)
+static bool str_starts_with_casei(const char *line, const char *substr)
 {
         size_t len = strlen(substr);
 
@@ -224,7 +236,7 @@ bool str_starts_with_casei(const char *line, const char *substr)
         return false;
 }
 
-bool pattern_matches(char *line, char *line_end, struct oops_pattern *pattern)
+static bool pattern_matches(char *line, char *line_end, struct oops_pattern *pattern)
 {
         if (pattern->is_regex) {
                 return (regexec(&(pattern->regex), line, 0, NULL, 0) == 0);
@@ -233,7 +245,7 @@ bool pattern_matches(char *line, char *line_end, struct oops_pattern *pattern)
         }
 }
 
-void init_pattern_regex(void)
+static void init_pattern_regex(void)
 {
         for (int i = 0; i < oops_patterns_cnt; i++) {
                 struct oops_pattern *pattern = &oops_patterns_arr[i];
@@ -244,7 +256,7 @@ void init_pattern_regex(void)
         }
 }
 
-void free_pattern_regex(void)
+static void free_pattern_regex(void)
 {
         for (int i = 0; i < oops_patterns_cnt; i++) {
                 struct oops_pattern *pattern = &oops_patterns_arr[i];
@@ -261,7 +273,6 @@ bool handle_entire_oops(char *buf, long size, struct oops_log_msg *msg)
         int i;
 
         msg->length = 0;
-        init_pattern_regex();
 
         line_end = memchr(buf, '\n', (size_t)size);
         if (line_end == NULL) {
@@ -318,7 +329,6 @@ void oops_msg_cleanup(struct oops_log_msg *msg)
                 free(msg->lines[i]);
         }
         msg->length = 0;
-        free_pattern_regex();
 }
 
 static struct oops_log_msg oops_msg;
@@ -333,7 +343,7 @@ void oops_parser_init(oops_handler_t handler)
         init_pattern_regex();
 }
 
-void handle_msg_end(void)
+static void handle_msg_end(void)
 {
         for (int i = 0; i < oops_msg.length; i++) {
                 free(oops_msg.lines[i]);
@@ -426,7 +436,6 @@ void parse_single_line(char *line, size_t size)
                         }
                 }
         }
-
 }
 
 struct stack_frame {
@@ -437,7 +446,7 @@ struct stack_frame {
         struct stack_frame *next;
 };
 
-void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, char *start)
+static void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, char *start)
 {
         /*
          * Format is:
@@ -472,7 +481,7 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
                 while (*start && !isspace(*start)) {
                         start++;
                 }
-                if (start && *start == '\0') {
+                if (*start == '\0') {
                         return;
                 }
 
@@ -507,7 +516,7 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
                 while (*start && !isspace(*start)) {
                         start++;
                 }
-                if (start && *start == '\0') {
+                if (*start == '\0') {
                         return;
                 }
 
@@ -536,7 +545,7 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
         frame->module = "kernel";
 }
 
-void stack_frame_free(struct stack_frame **head)
+static void stack_frame_free(struct stack_frame **head)
 {
         struct stack_frame *frame;
 
@@ -553,7 +562,7 @@ void stack_frame_free(struct stack_frame **head)
  * CPU: 2 PID: 0 Comm: swapper/2 Not tainted  3.10.4-100.fc18.x86_64 #1
  * CPU: 3 PID: 0 Comm: swapper/3 Not tainted 4.0.5-300.fc22.x86_64 #1
  */
-void parse_kernel_cpu_line(char *line, char **kernel_version, char **tainted)
+static void parse_kernel_cpu_line(char *line, char **kernel_version, char **tainted)
 {
         char *end = NULL;
 
@@ -626,7 +635,7 @@ static struct reg_s *reg_head = NULL;
  *
  */
 
-void parse_registers(char *line)
+static void parse_registers(char *line)
 {
         uint64_t value;
         struct reg_s *reg_entry = NULL;
@@ -739,7 +748,7 @@ void parse_registers(char *line)
 
 }
 
-void append_registers_to_bt(nc_string **backtrace)
+static void append_registers_to_bt(nc_string **backtrace)
 {
         struct reg_s *reg_entry = reg_head;
 
@@ -762,7 +771,7 @@ void append_registers_to_bt(nc_string **backtrace)
         }
 }
 
-nc_string *parse_backtrace(struct oops_log_msg *msg)
+static nc_string *parse_backtrace(struct oops_log_msg *msg)
 {
         struct stack_frame *head = NULL, *tail = NULL, *elem = NULL;
         //int in_stack_dump = 0;

src/probes/oops_probe.c

@@ -1,229 +0,0 @@
-/*
- * This program is part of the Clear Linux Project
- *
- * Copyright 2015 Intel Corporation
- *
- * This program is free software; you can redistribute it and/or modify it under
- * the terms and conditions of the GNU Lesser General Public License, as
- * published by the Free Software Foundation; either version 2.1 of the License,
- * or (at your option) any later version.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
- * details.
- */
-
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <sys/types.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/inotify.h>
-#include <poll.h>
-
-#include "oops_parser.h"
-#include "log.h"
-#include "telemetry.h"
-
-#include "nica/nc-string.h"
-
-char *oops_dir_path = KERNELOOPSDIR;
-
-static uint32_t version = 1;
-
-static bool send_data(char *backtrace, char *class, uint32_t severity)
-{
-        struct telem_ref *handle = NULL;
-        int ret;
-
-        if ((ret = tm_create_record(&handle, severity, class, version)) < 0) {
-                telem_log(LOG_ERR, "Failed to create record: %s",
-                          strerror(-ret));
-                return false;
-        }
-
-        if ((ret = tm_set_payload(handle, backtrace)) < 0) {
-                telem_log(LOG_ERR, "Failed to set payload: %s", strerror(-ret));
-                tm_free_record(handle);
-                return false;
-        }
-
-        if ((ret = tm_send_record(handle)) < 0) {
-                telem_log(LOG_ERR, "Failed to send record: %s", strerror(-ret));
-                tm_free_record(handle);
-                return false;
-        }
-
-        tm_free_record(handle);
-
-        return true;
-}
-
-void handle_oops_file(const char *fname)
-{
-        int ret;
-        FILE *fp = NULL;
-        char *filename = NULL, *contents = NULL;
-        long sz;
-        size_t size, bytes_read;
-        nc_string *payload;
-
-        ret = asprintf(&filename, "%s/%s", oops_dir_path, fname);
-        if (ret == -1) {
-                exit(EXIT_FAILURE);
-        }
-
-        #ifdef DEBUG
-        printf("File to be handled: %s\n", filename);
-        #endif
-
-        fp = fopen(filename, "r");
-        if (fp == NULL) {
-                telem_log(LOG_ERR, "Failed to open oops log file %s: %s", fname, strerror(errno));
-                goto file_end;
-        }
-
-        if (fseek(fp, 0, SEEK_END) == -1) {
-                telem_log(LOG_ERR, "Failed to seek file %s: %s\n", fname, strerror(errno));
-                goto file_end;
-        }
-
-        sz = ftell(fp);
-        if (sz == -1L) {
-                telem_perror("ftell failed");
-                goto file_end;
-        }
-
-        /* If file is empty */
-        if (sz == 0L) {
-                telem_perror("File is empty\n");
-                goto file_end;
-        }
-
-        if (fseek(fp, 0, SEEK_SET) < 0) {
-                telem_log(LOG_ERR, "Fseek failed for file %s:%s\n", fname, strerror(errno));
-                goto  file_end;
-        }
-
-        size = (size_t)sz;
-        contents = malloc(size);
-        if (!contents) {
-                telem_log(LOG_ERR, "Call to malloc failed");
-                goto file_end;
-        }
-        memset(contents, 0, size);
-
-        bytes_read = fread(contents, sizeof(char), size, fp);
-
-        if (bytes_read < size) {
-                telem_log(LOG_ERR, "Error while reading file %s:%s\n", fname, strerror(errno));
-                goto file_end;
-        }
-
-        struct oops_log_msg oops_msg;
-        if (handle_entire_oops(contents, (long)size, &oops_msg)) {
-            #ifdef DEBUG
-                printf("Raw message:\n");
-                for (int i = 0; i < oops_msg.length; i++) {
-                        printf("%s\n", oops_msg.lines[i]);
-                }
-            #endif
-
-                payload = parse_payload(&oops_msg);
-            #ifdef DEBUG
-                printf("Payload Parsed :%s\n", payload->str);
-            #endif
-
-                oops_msg_cleanup(&oops_msg);
-                send_data(payload->str, (char *)oops_msg.pattern->classification, (uint32_t)oops_msg.pattern->severity);
-                nc_string_free(payload);
-        } else {
-                /* File does no contain an oops! */
-                telem_log(LOG_ERR, "Did not find an oops in the oops directort\n");
-        }
-file_end:
-        if (contents) {
-                free(contents);
-        }
-        if (fp) {
-                fclose(fp);
-        }
-        if (unlink(filename) == -1) {
-                telem_log(LOG_ERR, "Failed to unlink %s: %s\n", filename, strerror(errno));
-        }
-        free(filename);
-}
-
-void handle_inotify_event(const struct inotify_event *event)
-{
-
-        if (!event) {
-                telem_perror("Null event received");
-                return;
-        }
-
-        if (event->len == 0) {
-                telem_perror("inotify event received with no file.");
-                return;
-        }
-
-        if (event->mask & IN_CLOSE_WRITE) {
-                if (event->mask & IN_ISDIR) {
-                        return;
-                }
-
-                #ifdef DEBUG
-                telem_log(LOG_DEBUG, "New file written to the oops location: %s\n", event->name);
-                #endif
-                handle_oops_file(event->name);
-        }
-}
-
-int main(int argc, char **argv)
-{
-        int inotify_fd;
-        char buf[4096];
-        char *ptr = NULL;
-        ssize_t len;
-        const struct inotify_event *event;
-
-        inotify_fd = inotify_init();
-        if (inotify_fd == -1) {
-                telem_log(LOG_ERR, "Failed to create inotify instance: %s\n", strerror(errno));
-                exit(EXIT_FAILURE);
-        }
-
-        if (inotify_add_watch(inotify_fd, oops_dir_path, IN_CLOSE_WRITE) == -1) {
-                telem_perror("Error adding watch to the inotify instance");
-        }
-
-        while (1) {
-                len = read(inotify_fd, buf, sizeof buf);
-                if (len <= 0) {
-                        telem_perror("error reading event");
-                        continue;
-                }
-
-                if (len < sizeof(struct inotify_event)) {
-                        telem_perror("Incomplete event received");
-                        continue;
-                }
-
-                telem_log(LOG_INFO, "Got inotify event/s\n");
-
-                /* Loop over all events in the buffer */
-                for (ptr = buf; ptr < buf + len;
-                     ptr += sizeof(struct inotify_event) + event->len) {
-
-                        event = (const struct inotify_event *)ptr;
-                        /* handle the event */
-                        handle_inotify_event(event);
-                }
-        }
-
-        return 0;
-}
-
-/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/pstore_clean.c

@@ -132,7 +132,7 @@ int main(int argc, char **argv)
         if (mkdir(MOUNT, 0755) < 0) {
                 //If there is an error other than an existing directory
                 if (errno != EEXIST) {
-                        telem_log(LOG_ERR, "Error creating directory /dev/pstore : %s\n", strerror(errno));
+                        telem_perror("Error creating directory /dev/pstore");
                         //bail out?
                         exit(EXIT_FAILURE);
                 }

src/probes/pstore_probe.c

@@ -15,6 +15,7 @@
 #define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
+#include <getopt.h>
 #include <dirent.h>
 #include <string.h>
 #include <errno.h>
@@ -65,14 +66,12 @@ void handle_complete_oops_message(struct oops_log_msg *msg)
 
 #ifdef DEBUG
         for (int i = 0; i < msg->length; i++) {
-                printf("%s\n", msg->lines[i]);
+                telem_debug("DEBUG: %s\n", msg->lines[i]);
         }
 #endif
         payload = parse_payload(msg);
 
-#ifdef DEBUG
-        printf("Payload Parsed :%s\n", payload->str);
-#endif
+        telem_debug("DEBUG: Payload Parsed :%s\n", payload->str);
         send_data(payload->str, (char *)msg->pattern->classification, (uint32_t)msg->pattern->severity);
         nc_string_free(payload);
 }
@@ -197,6 +196,13 @@ struct chunk_list {
         struct chunk_list *next;
 };
 
+static void print_usage(char *prog)
+{
+        printf("%s: Usage\n", prog);
+        printf("  -f,  --config_file    This overides the other parameters\n");
+        printf("  -h,  --help           Display this help message\n");
+}
+
 int main(int argc, char **argv)
 {
         DIR *pstore_dir;
@@ -207,6 +213,32 @@ int main(int argc, char **argv)
         int max_part;
         size_t totalsize = 0;
         char *crash_dump = NULL;
+        int c;
+        int opt_index = 0;
+
+        struct option opts[] = {
+                { "config_file", 1, NULL, 'f' },
+                { "help", 0, NULL, 'h' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "f:h", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                        telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                        exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 'h':
+                                print_usage(argv[0]);
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                print_usage(argv[0]);
+                                exit(EXIT_FAILURE);
+                }
+        }
 
         /*
          * Hash table used to store chunks belonging to a oops counter
@@ -232,9 +264,13 @@ int main(int argc, char **argv)
                 // Look for only dmesg logs. Ignore other types of pstore dumps for now.
                 if (sscanf(entry->d_name, "dmesg-efi-%" PRIu64, &id) == 1) {
                         parse_id(id, &count, &part);
-                        #ifdef DEBUG
-                        printf("Extracted count :%d, part : %d\n", count, part);
-                        #endif
+                        telem_debug("dmesg-efi Extracted count :%d, part : %d\n",
+                                    count, part);
+                } else if (sscanf(entry->d_name, "dmesg-ramoops-%" PRIu64, &id) == 1) {
+                        count = (int)(id + 1);
+                        part = 1;
+                        telem_debug("dmesg-ramoops Extracted count :%d, part : %d\n",
+                                    count, part);
                 } else {
                         continue;
                 }
@@ -262,9 +298,7 @@ int main(int argc, char **argv)
 
         nc_hashmap_iter_init(hash, &iter);
         while (nc_hashmap_iter_next(&iter, (void **)&key, (void **)&value)) {
-                #ifdef DEBUG
-                printf("Count in the hash: %d\n", NC_UNHASH_KEY(key));
-                #endif
+                telem_debug("DEBUG: Count in the hash: %d\n", NC_UNHASH_KEY(key));
 
                 head = (struct chunk_list *)value;
                 elem = head;

src/probes/telem_record_gen.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -32,22 +32,27 @@
 #include "common.h"
 #include "telemetry.h"
 
-static char *config_file = NULL;
 static uint32_t severity = 1;
 static char *opt_class = NULL;
 static char *opt_payload = NULL;
 static uint32_t payload_version = 1;
 static char *opt_payload_file = NULL;
+static char *opt_event_id = NULL;
+static bool opt_echo = false;
+static bool opt_nopost = false;
 
 static const struct option prog_opts[] = {
         { "help", no_argument, 0, 'h' },
-        { "config-file", required_argument, 0, 'f' },
+        { "echo", no_argument, 0, 'o' },
+        { "no-post", no_argument, 0, 'n' },
         { "version", no_argument, 0, 'V' },
         { "severity", required_argument, 0, 's' },
         { "class", required_argument, 0, 'c' },
         { "payload", required_argument, 0, 'p' },
         { "payload-file", required_argument, 0, 'P' },
         { "record-version", required_argument, 0, 'R' },
+        { "event-id", required_argument, 0, 'e' },
+        { "config-file", required_argument, 0, 'f' },
         { 0, 0, 0, 0 }
 };
 
@@ -60,38 +65,27 @@ static void print_help(void)
         printf("  -h, --help            Show help options\n");
         printf("\n");
         printf("Application Options:\n");
-        printf("  -f, --config-file     Path to configuration file (not implemented yet)\n");
         printf("  -V, --version         Print the program version\n");
         printf("  -s, --severity        Severity level (1-4) - (default 1)\n");
-        printf("  -c, --class           Classification level_1/level_2/level_3\n");
-        printf("  -p, --payload         Record body (max size = 8k)\n");
+        printf("  -c, --class           Classification level_1/level_2/level_3 (required)\n");
+        printf("  -p, --payload         Record body (max size = 8k) (required)\n");
         printf("  -P, --payload-file    File to read payload from\n");
         printf("  -R, --record-version  Version number for format of payload (default 1)\n");
+        printf("  -e, --event-id        Event id to use in the record\n");
+        printf("  -o, --echo            Echo record to stdout\n");
+        printf("  -n, --no-post         Do not post record just print\n");
+        printf("  -f, --config_file     Specify a configuration file other than default\n");
         printf("\n");
 }
 
-const unsigned int count_chars(const char *check, const char character)
+static bool parse_options(int argc, char **argv)
 {
-        unsigned int count = 0U;
-        if (check == NULL) {
-                return count;
-        }
-        for (; *check != '\0'; ++check) {
-                if (*check == character) {
-                        ++count;
-                }
-        }
-        return count;
-}
-
-int parse_options(int argc, char **argv)
-{
-        int ret = 0;
+        bool ret = false;
         char *endptr = NULL;
         long unsigned int tmp = 0;
 
         int opt;
-        while ((opt = getopt_long(argc, argv, "hc:Vs:c:p:P:R:", prog_opts, NULL)) != -1) {
+        while ((opt = getopt_long(argc, argv, "hc:Vs:c:p:P:R:e:onf:", prog_opts, NULL)) != -1) {
                 switch (opt) {
                         case 'h':
                                 print_help();
@@ -116,12 +110,21 @@ int parse_options(int argc, char **argv)
                                 severity = (uint32_t)tmp;
                                 break;
                         case 'c':
+                                if (opt_class != NULL) {
+                                    free(opt_class);
+                                }
                                 opt_class = strdup(optarg);
                                 break;
                         case 'p':
+                                if (opt_payload != NULL) {
+                                    free(opt_payload);
+                                }
                                 opt_payload = strdup(optarg);
                                 break;
                         case 'P':
+                                if (opt_payload_file != NULL) {
+                                    free(opt_payload_file);
+                                }
                                 opt_payload_file = strdup(optarg);
                                 break;
                         case 'R':
@@ -140,34 +143,57 @@ int parse_options(int argc, char **argv)
 
                                 payload_version = (uint32_t)tmp;
                                 break;
+                        case 'e':
+                                if (opt_event_id != NULL) {
+                                    free(opt_event_id);
+                                }
+                                opt_event_id = strdup(optarg);
+                                if (opt_event_id == NULL) {
+                                        goto fail;
+                                }
+                                break;
+                        case 'o':
+                                opt_echo = true;
+                                break;
+                        case 'n':
+                                opt_nopost = true;
+                                break;
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
                 }
         }
 
-        ret = 1;
+        ret = true;
 fail:
         return ret;
 }
 
-int validate_opts(void)
+static bool validate_opts(void)
 {
-        size_t i;
-        int ret = 0;
+        size_t i, len;
+        int x, slashes = 0;
+        const char alphab[] = EVENT_ID_ALPHAB;
+        bool ret = false;
 
         /* classification */
         if (opt_class == NULL) {
                 fprintf(stderr, "Error: Classification required. See --help.\n");
                 return ret;
-
         }
-        i = strlen(opt_class);
 
-        if ((i == 0) || (i > 120)) {
+        len = strlen(opt_class);
+
+        if ((len == 0) || (len > MAX_CLASS_LENGTH)) {
                 fprintf(stderr, "Error: Valid size for classification "
-                        "is 1-120 chars\n");
+                        "is 1-%d chars\n", MAX_CLASS_LENGTH);
                 return ret;
         }
 
-        for (int c = 0; c < strlen(opt_class); c++) {
+        for (int c = 0; c < len; c++) {
                 if (isascii(opt_class[c]) == 0) {
                         fprintf(stderr, "Error: Non-ascii characters detected "
                                 "in classification - aborting\n");
@@ -175,41 +201,53 @@ int validate_opts(void)
                 }
         }
 
-        if (count_chars(opt_class, '/') != 2) {
+        for (i = 0, x = 0; i <= (len - 1); i++, x++) {
+                if (opt_class[i] == '/') {
+                        slashes++;
+                        x = 0;
+                } else {
+                        if (x > MAX_SUBCAT_LENGTH) {
+                                fprintf(stderr, "Error: Classification strings"
+                                        " between slashes should have at most"
+                                        " %d chars\n", MAX_SUBCAT_LENGTH);
+                                return ret;
+                        }
+                }
+        }
+
+        if (slashes != 2) {
                 fprintf(stderr, "Error: Classification needs to be in "
                         "most/to/least specific format, 2 \'/\' required.\n");
                 return ret;
         }
 
+
         /* Severity */
         if ((severity) < 1 || (severity > 4)) {
                 fprintf(stderr, "Error: Valid range for severity is 1-4\n");
                 return ret;
         }
 
-        return 1;
-}
-
-int allocate_payload_buffer(char **payload)
-{
-        int ret = 0;
-
-        *payload = (char *)malloc(MAX_PAYLOAD_SIZE);
-
-        if (*payload == NULL) {
-                goto out1;
+        /* Event id  */
+        if (opt_event_id) {
+                size_t result = 0;
+                if ((result = strlen(opt_event_id)) != EVENT_ID_LEN) {
+                        fprintf(stderr, "Error: event_id length %zu it should have %d characters\n", result, EVENT_ID_LEN);
+                        return ret;
+                }
+                if ((result = strspn(opt_event_id, alphab)) != EVENT_ID_LEN) {
+                        fprintf(stderr, "Error: event_id contains invalid character '%c' at position %zu, valid characters are %s\n",
+                                opt_event_id[result], result, alphab);
+                        return ret;
+                }
         }
 
-        *payload = memset(*payload, 0, MAX_PAYLOAD_SIZE);
-        ret = 1;
-
-out1:
-        return ret;
+        return true;
 }
 
-int get_payload_from_file(char **payload)
+static bool get_payload_from_file(char **payload)
 {
-        int ret = 1;
+        bool ret = false;
         FILE *fp = NULL;
         size_t bytes_in = 0;
 
@@ -222,14 +260,14 @@ int get_payload_from_file(char **payload)
         }
 
         bytes_in = fread(*payload, 1,
-                         MAX_PAYLOAD_SIZE - 1, fp);
+                         MAX_PAYLOAD_LENGTH - 1, fp);
 
         /* if fread fails */
         if (bytes_in == 0 && ferror(fp) != 0) {
-                ret = 0;
                 goto out;
         }
 
+        ret = true;
 out:
         if (fp) {
                 fclose(fp);
@@ -238,25 +276,27 @@ out:
 
 }
 
-void get_payload_from_opt(char **payload)
+static void get_payload_from_opt(char **payload)
 {
         size_t len = 0;
 
         len = strlen(opt_payload);
-        if (len >= MAX_PAYLOAD_SIZE) {
-                len = MAX_PAYLOAD_SIZE - 1;
+        if (len >= MAX_PAYLOAD_LENGTH) {
+                len = MAX_PAYLOAD_LENGTH - 1;
         }
-        strncpy(*payload, opt_payload, len);
+
+        /* "payload" is pre-allocated zeroed buffer of MAX_PAYLOAD_LENGTH */
+        memcpy(*payload, opt_payload, len);
 }
 
-void get_payload_from_stdin(char **payload)
+static void get_payload_from_stdin(char **payload)
 {
         size_t bytes_in = 0;
         int c;
 
-        bytes_in = fread(*payload, 1, MAX_PAYLOAD_SIZE - 1, stdin);
+        bytes_in = fread(*payload, 1, MAX_PAYLOAD_LENGTH - 1, stdin);
 
-        if (bytes_in == MAX_PAYLOAD_SIZE - 1) {
+        if (bytes_in == MAX_PAYLOAD_LENGTH - 1) {
                 /* Throw away the rest of stdin */
                 while (EOF != (c = fgetc(stdin))) {
                         continue;
@@ -265,60 +305,95 @@ void get_payload_from_stdin(char **payload)
 
 }
 
-int get_payload(char **payload)
+static bool get_payload(char **payload)
 {
-        int ret = 0;
+        bool ret = false;
 
-        if (!allocate_payload_buffer(payload)) {
-                goto out1;
-        }
+        *payload = (char *)calloc(sizeof(char), MAX_PAYLOAD_LENGTH);
+
+        if (*payload == NULL)
+                return false;
 
         if (opt_payload_file) {
 
                 if (get_payload_from_file(payload)) {
-                        ret = 1;
+                        ret = true;
                 }
 
         } else if (opt_payload) {
 
                 get_payload_from_opt(payload);
-                ret = 1;
+                ret = true;
 
         } else {
                 get_payload_from_stdin(payload);
-                ret = 1;
+                ret = true;
         }
 
-        if (ret == 0) {
+        if (ret == false) {
                 free(*payload);
                 *payload = NULL;
         }
 
+        return ret;
+}
+
+static int instanciate_record(struct telem_ref **t_ref, char *payload)
+{
+        int ret = 0;
+
+        if ((ret = tm_create_record(t_ref, (uint32_t)severity,
+                                    opt_class, payload_version)) < 0) {
+                goto out1;
+        }
+
+        if (opt_event_id) {
+                if ((ret = tm_set_event_id(*t_ref, opt_event_id)) < 0) {
+                        goto out1;
+                }
+        }
+
+        ret = tm_set_payload(*t_ref, payload);
 out1:
         return ret;
 }
 
-int send_record(char *payload)
+static int send_record(char *payload)
 {
-        struct telem_ref *t_ref;
+        struct telem_ref *t_ref = NULL;
         int ret = 0;
 
-        if (tm_create_record(&t_ref, (uint32_t)severity,
-                             opt_class, payload_version) < 0) {
-                goto out1;
+        if ((ret = instanciate_record(&t_ref, payload)) < 0) {
+                goto out;
         }
 
-        if (tm_set_payload(t_ref, payload) < 0) {
-                goto out2;
+        if ((ret = tm_send_record(t_ref)) < 0) {
+                if (ret == -ECONNREFUSED) {
+                        fprintf(stderr, "Unable to send record. Make sure to opt-in to telemetry first 'telemctl opt-in'\n");
+                }
+                goto out;
+        }
+
+out:
+        tm_free_record(t_ref);
+
+        return ret;
+}
+
+static int print_record(char *payload)
+{
+        struct telem_ref *t_ref = NULL;
+        int ret;
+
+        if ((ret = instanciate_record(&t_ref, payload)) == 0) {
+                int i;
+                for (i = 0; i < NUM_HEADERS; i++) {
+                        fprintf(stdout, "%s", t_ref->record->headers[i]);
+                }
+                fprintf(stdout, "%s\n", t_ref->record->payload);
         }
 
-        if (tm_send_record(t_ref) >= 0) {
-                ret = 1;
-        }
-
-out2:
         tm_free_record(t_ref);
-out1:
         return ret;
 }
 
@@ -339,15 +414,20 @@ int main(int argc, char **argv)
                 goto fail;
         }
 
-        if (!send_record(payload)) {
+        if (opt_echo == true) {
+                print_record(payload);
+        }
+
+        if (opt_nopost == false && send_record(payload) != 0) {
                 goto fail;
         }
 
         ret = EXIT_SUCCESS;
 fail:
-        free(config_file);
         free(opt_class);
         free(opt_payload);
+        free(opt_event_id);
+        free(payload);
 
         return ret;
 }

src/retention.c

@@ -0,0 +1,46 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+
+#include "log.h"
+#include "common.h"
+
+int delete_record_by_id(char *record_id)
+{
+        int ret = 0;
+        char *record_path = NULL;
+
+        ret = asprintf(&record_path, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (ret == -1) {
+                return 1;
+        }
+        ret = unlink(record_path);
+        if (ret == -1) {
+                telem_perror("Error deleting saved record");
+        }
+        free(record_path);
+
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/retention.h

@@ -0,0 +1,27 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+/**
+ * Delete record identified by record unique id
+ *
+ * @param record_id Unique identifier of entry
+ *
+ */
+int delete_record_by_id(char *record_id);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/spool.c

@@ -27,7 +27,7 @@
 #include <errno.h>
 
 #include "spool.h"
-#include "telemdaemon.h"
+#include "telempostdaemon.h"
 #include "log.h"
 #include "configuration.h"
 #include "util.h"
@@ -202,9 +202,15 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
         FILE *fp = NULL;
         char *headers[NUM_HEADERS];
         char *payload = NULL;
-        int i, k;
-        char line[LINE_MAX] = { 0 };
+        int num_headers = 0, k;
+#if (LINE_MAX > PATH_MAX)
+        char line[LINE_MAX+1] = { 0 };
+#else
+        char line[PATH_MAX+1] = { 0 };
+#endif
         long offset;
+        char *cfg_file = NULL;
+        uint32_t cfg_prefix = 0;
 
         fp = fopen(record_path, "r");
         if (fp == NULL) {
@@ -212,16 +218,47 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
                 return;
         }
 
-        for (i = 0; i < NUM_HEADERS; i++) {
-                const char *header_name = get_header_name(i);
-                if (!fgets(line, LINE_MAX, fp)) {
+        // First line optionally contains configuration file path
+        if (fread(&cfg_prefix, CFG_PREFIX_LENGTH, 1, fp) != 1) {
+                telem_log(LOG_ERR, "Error while parsing spooled record configuration info.\n");
+                goto read_error;
+        }
+
+        if (cfg_prefix == CFG_PREFIX_32BIT) {
+                size_t pathlen;
+                char *nl;
+
+                if (!fgets(line, sizeof(line), fp)) {
                         telem_log(LOG_ERR, "Error while parsing record file\n");
-                        fclose(fp);
-                        return;
+                        goto read_error;
+                }
+
+                if ((nl = strchr(line, '\n')) != NULL) {
+                        *nl = '\0';
+                }
+
+                pathlen = strlen(line);
+                cfg_file = malloc(pathlen + 1);
+                if (cfg_file == NULL) {
+                        telem_log(LOG_ERR, "Could not allocate memory for config file path\n");
+                        goto read_error;
+                }
+                strcpy (cfg_file, line);
+                telem_debug("DEBUG: cfg_file: %s\n", cfg_file);
+        } else {
+                cfg_file = NULL;
+                rewind(fp);
+        }
+
+        for (num_headers = 0; num_headers < NUM_HEADERS; num_headers++) {
+                const char *header_name = get_header_name(num_headers);
+                if (!fgets(line, sizeof(line), fp)) {
+                        telem_log(LOG_ERR, "Error while parsing record file\n");
+                        goto read_error;
                 }
                 //Get rid of trailing newline
                 strtok(line, "\n");
-                if (get_header(line, header_name, &headers[i], strlen(header_name))) {
+                if (get_header(line, header_name, &headers[num_headers])) {
                         continue;
                 } else {
                         telem_log(LOG_ERR, "transmit_spooled_record: Incorrect"
@@ -242,8 +279,7 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
 
         if (!payload) {
                 telem_log(LOG_ERR, "Could not allocate memory for payload\n");
-                fclose(fp);
-                return;
+                goto read_error;
         }
         memset(payload, 0, (size_t)size);
 
@@ -254,7 +290,7 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
                 goto read_error;
         }
 
-        *post_succeeded = post_record_http(headers, payload, false);
+        *post_succeeded = post_record_http(headers, payload, cfg_file);
         if (*post_succeeded) {
                 unlink(record_path);
         }
@@ -267,9 +303,13 @@ read_error:
                 fclose(fp);
         }
 
-        for (k = 0; k < i; k++) {
+        for (k = 0; k < num_headers; k++) {
                 free(headers[k]);
         }
+
+        if (cfg_file) {
+                free(cfg_file);
+        }
 }
 
 int spool_record_compare(const void *entrya, const void *entryb, void *path)

src/telemctl.c

@@ -0,0 +1,554 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2019 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <ftw.h>
+#include <unistd.h>
+#include <locale.h>
+#include <pwd.h>
+#include <grp.h>
+#include <errno.h>
+
+#define TELEM_DIR       "/etc/telemetrics"
+#define TM_OPT_IN       TELEM_DIR"/opt-in"
+
+#define TELEM_WRK_DIRS_CONF "/usr/lib/tmpfiles.d/telemetrics-dirs.conf"
+
+#define cmd_create_tmp_files "systemd-tmpfiles --create " TELEM_WRK_DIRS_CONF
+#define cmd_mk_telemdir      "mkdir -p " TELEM_DIR
+
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
+
+static char *SPECIAL_UNITS[] = {
+        "hprobe.timer",
+        "telemprobd.socket",
+        "telempostd.path",
+        "klogscanner.service",
+        "journal-probe-tail.service",
+};
+
+#define NUM_SPECIAL_UNITS ARRAY_SIZE(SPECIAL_UNITS)
+
+static char *SERVICES[] = {
+        "hprobe.service",
+        "pstore-probe.service",
+        "telemprobd.service",
+        "telempostd.service",
+        "journal-probe.service",
+};
+
+#define NUM_SERVICES ARRAY_SIZE(SERVICES)
+
+static int telemctl_start(void);
+static int telemctl_stop(void);
+static int telemctl_restart(void);
+static int telemctl_is_active(void);
+static int telemctl_opt_out(void);
+static int telemctl_opt_in(void);
+static int telemctl_journal(char *);
+
+struct telemcmd {
+        char *cmd;
+        union {
+        int  (*f1)(void);
+        int  (*f2)(char *);
+        } f;
+        char *doc;
+};
+
+static struct telemcmd commands[] = {
+        {"stop",      {.f1=telemctl_stop},     "Stops all running telemetry services" },
+        {"start",     {.f1=telemctl_start},    "Starts all telemetry services" },
+        {"restart",   {.f1=telemctl_restart},  "Restarts all telemetry services" },
+        {"is-active", {.f1=telemctl_is_active},"Checks if telemprobd and telempostd are active" },
+        {"opt-in",    {.f1=telemctl_opt_in},   "Opts in to telemetry, and starts telemetry services" },
+        {"opt-out",   {.f1=telemctl_opt_out},  "Opts out of telemetry, and stops telemetry services" },
+        {"journal",   {.f2=telemctl_journal},  "Prints telemetry journal contents. Use -h argument with\n            command for more options"}
+};
+
+static int syscmd(char *cmd, char *buff, int bufflen)
+{
+        int status;
+#ifdef DEBUG
+        printf("[debug] [%s] %s\n", __func__, cmd);
+#endif
+        FILE *fp = popen(cmd, "r");
+        if (fp == NULL) {
+                perror("popen");
+                return -1;
+        }
+
+        while (fgets(buff, bufflen, fp) != NULL) {
+                ;
+        }
+
+        status = pclose(fp);
+        if (status == -1) {
+                perror("pclose");
+        }
+
+        return status;
+}
+
+/*
+Script code:
+
+  for_each_service() {
+  local action=$1 && shift
+  local -a array=($*)
+  for service in "${array[@]}"; do
+    systemctl $action $service
+    [ $? -ne 0 ] && notice "Failed to $action ${service}. Continuing..."
+  done
+}
+*/
+static int for_each_service(const char *command, char* services[], int numservices)
+{
+        char buff[512];
+        char cmd[256];
+        int ret = 0, status;
+
+        for (int i = 0; i < numservices; i++) {
+                snprintf(cmd, sizeof(cmd), "systemctl %s %s", command, services[i]);
+                memset(buff, 0, sizeof(buff));
+                status = syscmd(cmd, buff, sizeof(buff));
+                if (status != 0 || buff[0] != 0) {
+                        fprintf(stderr, "%s", buff);
+                        fprintf(stderr, "Failed to %s %s. Continuing...", command, services[i]);
+                        ret = 1;
+                }
+         }
+
+         return ret;
+}
+
+/*
+Script code:
+
+telem_start() {
+  [ -f $OPT_OUT_FILE ] && exit_err "Opt out is enabled. Cannot start services."
+  create_work_dirs
+  for_each_service "start" ${SPECIAL_UNITS[@]}
+}
+
+Modified to add explicit opt-in
+*/
+static int telemctl_start(void)
+{
+        char buff[512];
+        int status, ret;
+
+        if (access(TM_OPT_IN, F_OK) != 0) {
+                fprintf(stderr, "Opt in to telemetry first.\n");
+                return 1;
+        }
+
+        /* Creates dirs if missing, adjust ownership if exists */
+        memset(buff, 0, sizeof(buff));
+
+        /* Execute "systemd-tmpfiles --create ${TELEM_WRK_DIRS_CONF}" */
+        status = syscmd(cmd_create_tmp_files, buff, sizeof(buff));
+        fprintf(stderr, "%s", buff);
+
+        if (status == -1) {
+                return 1;
+        }
+
+        ret = for_each_service("start", SPECIAL_UNITS, NUM_SPECIAL_UNITS);
+        return ret;
+}
+
+
+/*
+Script code:
+
+telem_stop() {
+  for_each_service "stop" ${SPECIAL_UNITS[@]}
+  for_each_service "stop" ${SERVICES[@]}
+}
+*/
+static int telemctl_stop(void)
+{
+        int ret;
+
+        /* the special units must be stopped first so that activation no longer happens */
+        ret =  for_each_service("stop", SPECIAL_UNITS, NUM_SPECIAL_UNITS);
+        ret |= for_each_service("stop", SERVICES, NUM_SERVICES);
+        return ret;
+}
+
+/*
+Script code:
+*
+telem_restart() {
+  telem_stop
+  telem_start
+}
+*/
+static int telemctl_restart(void)
+{
+        if (telemctl_stop() == 0) {
+                return telemctl_start();
+        }
+        return 1;
+}
+
+/*
+Script code:
+
+telem_is_active() {
+  echo "telemprobd :" $(systemctl is-active telemprobd.socket)
+  echo "telempostd :" $(systemctl is-active telempostd.path)
+}
+*/
+
+static int telemctl_is_active(void)
+{
+        char buff[256];
+
+        memset(buff, 0, sizeof(buff));
+        if (syscmd("systemctl is-active telemprobd.socket", buff, sizeof(buff)) != -1) {
+                printf("telemprobd : %s", buff);
+                memset(buff, 0, sizeof(buff));
+                if (syscmd("systemctl is-active telempostd.path",buff, sizeof(buff)) != -1) {
+                        printf("telempostd : %s", buff);
+                        return 0;
+                }
+        }
+        return 1;
+}
+
+
+static int unlink_cb(const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf)
+{
+        int rv = remove(fpath);
+#ifdef DEBUG
+        fprintf(stderr, "[debug] [%s] remove fpath:%s rv:%d\n", __func__, fpath, rv);
+#endif
+        return rv;
+
+}
+
+/* rm -rf path
+ *
+ * We run as root, so be extra careful when deleting folders.
+ * We only delete folders owned by telemetry:telemetry.
+ * As for the files, they can be owned by root.
+ */
+static int rm_rf(char *path)
+{
+        struct stat info;
+        if (stat(path, &info) == 0) {
+                struct passwd *pw = getpwuid(info.st_uid);
+                if (pw == NULL) {
+                        perror("getpwuid");
+                        return -1;
+                }
+                if (strcmp(pw->pw_name, "telemetry") != 0) {
+                        fprintf(stderr, "Not removing \"%s\": Incorrect folder owner \"%s\"\n",
+                                path, pw->pw_name);
+                        return -1;
+                }
+                struct group  *gr = getgrgid(info.st_gid);
+                if (gr == NULL) {
+                        perror("getgrgid");
+                        return -1;
+                }
+                if (strcmp(gr->gr_name, "telemetry") != 0) {
+                        fprintf(stderr, "Not removing \"%s\": Incorrect folder group \"%s\"\n",
+                                path, gr->gr_name);
+                        return -1;
+                }
+
+                return nftw(path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
+        } else {
+                // Some folders are subfolders of folders already recursivley deleted.
+                // So they may not exist anymore.
+                if (errno != ENOENT) {
+                        fprintf(stderr, "file: %s :", path);
+                        perror("stat");
+                        return -1;
+                }
+                return 0;
+        }
+}
+
+/*
+ Script code:
+
+ telem_remove_work_dirs() {
+  # Remove dirs
+  awk '/^d/{print $2}' ${TELEM_WRK_DIRS_CONF} | xargs rm -rf;
+}
+
+Notes:
+  Parse /usr/lib/tmpfiles.d/telemetrics-dirs.conf
+  This file was used to create the folders needed by telemetry.
+  Now we want to delete them. The file contents is as:
+
+   d /usr/local/var/lib/telemetry 0755 telemetry telemetry -
+   d /usr/local/var/spool/telemetry 0750 telemetry telemetry -
+   d /usr/local/var/log/telemetry 0750 telemetry telemetry -
+   d /usr/local/var/log/telemetry/records 0750 telemetry telemetry -
+   d /usr/local/var/cache/telemetry 0750 telemetry telemetry -
+   d /usr/local/var/cache/telemetry/pstore 0750 telemetry telemetry -
+*/
+static int telemctl_remove_work_dirs(void)
+{
+        FILE *fp = fopen(TELEM_WRK_DIRS_CONF, "r");
+        if (fp == NULL){
+                fprintf(stderr, "Could not open file %s", TELEM_WRK_DIRS_CONF);
+                return 1;
+        }
+
+        while (true) {
+                char folder[PATH_MAX+1];
+                char line[PATH_MAX+30];
+
+                if (fgets(line, sizeof(line), fp) == NULL) {
+                        if (ferror(fp)) {
+                                fprintf(stderr, "Error reading file %s\n", TELEM_WRK_DIRS_CONF);
+                                fclose(fp);
+                                return 1;
+                        }
+                        // EOF
+                        break;
+                }
+
+                int ret = sscanf(line, "d %s %*s telemetry telemetry - \n", folder);
+                if (ret == 1) {
+                        /* Delete the folder. We may get an error trying to delete
+                         * folders already deleted. We ignore them, so don't bother
+                         * checking rmrtf return value */
+                        rm_rf(folder);
+                }
+        }
+
+        fclose(fp);
+        return 0;
+}
+
+static int mk_telem_dir(void)
+{
+        char buff[512];
+        int status;
+        struct stat sb;
+
+        if (stat(TELEM_DIR, &sb) == 0 && S_ISDIR(sb.st_mode)) {
+#ifdef DEBUG
+                printf("[debug] [%s] Folder %s exists\n", __func__, TELEM_DIR);
+#endif
+                return 0;
+        }
+
+        memset(buff, 0, sizeof(buff));
+
+        /* Execute "mkdir -p TM_DIR" */
+        status = syscmd(cmd_mk_telemdir, buff, sizeof(buff));
+        fprintf(stderr, "%s", buff);
+
+        if (status == -1 || buff[0] != 0) {
+                return 1;
+        }
+
+        return 0;
+}
+
+/*
+Script code:
+
+telem_opt_out() {
+  [ -f $OPT_OUT_FILE ] && exit_ok "Already opted out. Nothing to do."
+  mkdir -p $TELEM_DIR || exit_err "Failed to create ${TELEM_DIR}."
+  touch $OPT_OUT_FILE || exit_err "Failed to create ${OPT_OUT_FILE}."
+  telem_stop
+  telem_remove_work_dirs
+}
+*/
+static int telemctl_opt_out(void)
+{
+        int ret;
+
+        /* Remove TM_OPT_IN file */
+        if (unlink(TM_OPT_IN) != 0) {
+                if (errno == ENOENT) {
+                        fprintf(stderr, "Already opted out. Nothing to do.\n");
+                        return 0;
+                }
+		fprintf(stderr, "Failed to remove %s.\n", TM_OPT_IN);
+                return 1;
+	}
+
+        ret = telemctl_stop();
+        ret |= telemctl_remove_work_dirs();
+        return ret;
+}
+
+/*
+Script code:
+
+telem_opt_in() {
+  [ ! -f $OPT_OUT_FILE ] && exit_ok "Already opted in. Nothing to do."
+  rm -f $OPT_OUT_FILE || exit_err "Failed to remove ${OPT_OUT_FILE}."
+  telem_start
+}
+
+Modified to add explicit opt-in
+*/
+static int telemctl_opt_in(void)
+{
+
+        /* Ensure TELEM_DIR exists */
+        if (mk_telem_dir() != 0) {
+                fprintf(stderr, "Failed to create %s\n", TELEM_DIR);
+                return 1;
+        }
+
+	/* Create a brand new file TM_OPT_IN, we maight fail because the file exists already.
+         * In that case we are already opted in and we are done here. */
+        int fd = open(TM_OPT_IN, O_CREAT|O_EXCL|O_WRONLY, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
+        if (fd == -1) {
+                if (errno == EEXIST) {
+                        fprintf(stderr, "Already opted in. Nothing to do.\n");
+                        return 0;
+                } else {
+                        fprintf(stderr, "Failed to create %s.\n", TM_OPT_IN);
+                        return 1;
+                }
+        }
+        close(fd);
+
+        return 0;
+}
+
+
+static int journal_cmd(char *cmd)
+{
+        int status;
+        char buff[256];
+#ifdef DEBUG
+        printf("[debug] [%s] %s\n", __func__, cmd);
+#endif
+        FILE *fp = popen(cmd, "r");
+        if (fp == NULL) {
+                perror("popen");
+                return -1;
+        }
+
+        while (fgets(buff, sizeof(buff), fp) != NULL) {
+                printf("%s", buff);
+        }
+
+        status = pclose(fp);
+        if (status == -1) {
+                perror("pclose");
+        }
+
+        return status;
+}
+
+static char* concatargs(int argc, char** argv)
+{
+        size_t len = strlen("telem_journal") + 1;
+        for (int i = 2; i < argc; i++) {
+                len += strlen(argv[i]) + 1;
+        }
+
+        char *buff = malloc(len*sizeof(char));
+
+        if (buff != NULL) {
+                buff[0] = '\0';
+                strcat(buff,"telem_journal ");
+
+                for (int i = 2; i < argc; i++) {
+                        strcat(buff, argv[i]);
+                        strcat(buff, " ");
+                }
+        }
+
+        return buff;
+}
+
+static int telemctl_journal(char * x)
+{
+        journal_cmd(x);
+        return 1;
+}
+
+static void print_usage(char *str)
+{
+        printf("%s - Control actions for telemetry services\n\n", str);
+
+        for (int i = 0; i < sizeof(commands)/sizeof(commands[0]); i++) {
+                printf("  %-9s %s\n", commands[i].cmd, commands[i].doc);
+        }
+}
+
+int main(int argc, char **argv)
+{
+        int i, ret = EXIT_SUCCESS;
+        bool is_root;
+
+        setlocale(LC_ALL, "");
+
+        if (argc == 1) {
+                print_usage(argv[0]);
+                exit(2);
+        }
+
+        is_root = getuid()?false:true;
+
+        for (i = 0; i < sizeof(commands)/sizeof(commands[0]); i++) {
+                if (strcmp(commands[i].cmd, argv[1]) == 0) {
+                        if (!is_root) {
+                                fprintf(stderr, "Must be root to run this command. Exiting...\n");
+                                exit(1);
+                        }
+                        /* Special treatment for "journal", it can have cmd line arguments */
+                        if (strcmp(argv[1],"journal") != 0) {
+                                if (argc != 2) {
+                                        print_usage(argv[0]);
+                                        exit(2);
+                                }
+                                ret = commands[i].f.f1();
+                        } else {
+                                char *buff = concatargs(argc, argv);
+                                if (buff != NULL) {
+                                        ret = commands[i].f.f2(buff);
+                                        free(buff);
+                                }
+                        }
+
+                        break;
+                }
+        }
+
+        if (i == sizeof(commands)/sizeof(commands[0])) {
+                printf("Unknown command passed to %s\n\n", argv[0]);
+                print_usage(argv[0]);
+                exit(2);
+        }
+
+        exit(ret);
+}

src/telemdaemon.c

@@ -27,38 +27,25 @@
 #include <fcntl.h>
 #include <time.h>
 #include <malloc.h>
+#include <sys/uio.h>
 
+#include "iorecord.h"
 #include "telemdaemon.h"
 #include "common.h"
 #include "util.h"
 #include "log.h"
 #include "configuration.h"
 
-void initialize_daemon(TelemDaemon *daemon)
+static void process_record(TelemDaemon *daemon, client *cl);
+
+void initialize_probe_daemon(TelemDaemon *daemon)
 {
         client_list_head head;
         LIST_INIT(&head);
         daemon->nfds = 0;
         daemon->pollfds = NULL;
         daemon->client_head = head;
-        daemon->bypass_http_post_ts = 0;
-        daemon->is_spool_valid = false;
-        initialize_rate_limit(daemon);
-        daemon->current_spool_size = 0;
-}
-
-void initialize_rate_limit(TelemDaemon *daemon)
-{
-        for (int i = 0; i < TM_RATE_LIMIT_SLOTS; i++) {
-                daemon->record_burst_array[i] = 0;
-                daemon->byte_burst_array[i] = 0;
-        }
-        daemon->rate_limit_enabled = rate_limit_enabled_config();
-        daemon->record_burst_limit = record_burst_limit_config();
-        daemon->record_window_length = record_window_length_config();
-        daemon->byte_burst_limit = byte_burst_limit_config();
-        daemon->byte_window_length = byte_window_length_config();
-        daemon->rate_limit_strategy = rate_limit_strategy_config();
+        daemon->machine_id_override = NULL;
 }
 
 client *add_client(client_list_head *client_head, int fd)
@@ -95,7 +82,8 @@ bool is_client_list_empty(client_list_head *client_head)
         return (client_head->lh_first == NULL);
 }
 
-void terminate_client(TelemDaemon *daemon, client *cl, nfds_t index)
+
+static void terminate_client(TelemDaemon *daemon, client *cl, nfds_t index)
 {
         /* Remove fd from the pollfds array */
         del_pollfd(daemon, index);
@@ -106,24 +94,40 @@ void terminate_client(TelemDaemon *daemon, client *cl, nfds_t index)
         remove_client(&(daemon->client_head), cl);
 }
 
-bool handle_client(TelemDaemon *daemon, nfds_t ind, client *cl)
+/*
+ See "tm_send_record" for record retails.
+
+ recv buffer layout:
+         * <uint32_t record_size>    : so recv knows how much to read
+         * <custom cfg file field>   : optional, variable size (string)
+         * <uint32_t header_size>
+         * <headers + Payload>
+         * <null-byte>
+
+ The routine handle_client only cares about "record_size".
+ However, we need to validate if the record_size is reasonable. We assume the
+ worst case scenario would be a record with max cfg file field. There is no
+ exact way to determine header_size, so we assume each line at most 80 chars.
+ 
+*/
+
+#define MAX_RECORD_SIZE (2*sizeof(uint32_t) + CFG_PREFIX_LENGTH + PATH_MAX + \
+        MAX_PAYLOAD_LENGTH + NUM_HEADERS*80)
+bool handle_client(TelemDaemon *daemon, nfds_t index, client *cl)
 {
         /* For now  read data from fd */
         ssize_t len;
-        size_t record_size = 0;
+        size_t buf_size;
         bool processed = false;
+        uint32_t record_size;
 
-        if (!cl->buf) {
-                cl->buf = malloc(RECORD_SIZE_LEN);
-                cl->size = RECORD_SIZE_LEN;
-        }
-        if (!cl->buf) {
-                telem_log(LOG_ERR, "Unable to allocate memory, exiting\n");
-                exit(EXIT_FAILURE);
+        if (cl->buf != NULL) {
+                free(cl->buf);
+                cl->buf = NULL;
         }
 
         malloc_trim(0);
-        len = recv(cl->fd, cl->buf, RECORD_SIZE_LEN, MSG_PEEK | MSG_DONTWAIT);
+        len = recv(cl->fd, &record_size, RECORD_SIZE_LEN, MSG_PEEK | MSG_DONTWAIT);
         if (len < 0) {
                 telem_log(LOG_ERR, "Failed to talk to client %d: %s\n", cl->fd,
                           strerror(errno));
@@ -135,6 +139,39 @@ bool handle_client(TelemDaemon *daemon, nfds_t ind, client *cl)
                 goto end_client;
         }
 
+        /* Read the record size first */
+        len = recv(cl->fd, &record_size, RECORD_SIZE_LEN, 0);
+        if (len < 0) {
+                telem_log(LOG_ERR, "Failed to receive data from client"
+                                  " %d: %s\n", cl->fd, strerror(errno));
+                goto end_client;
+        } else if (len == 0) {
+                telem_log(LOG_DEBUG, "End of transmission for client"
+                          " %d\n", cl->fd);
+                goto end_client;
+        }
+
+        /* Now that we know the record size, allocate a new buffer
+         * for the record body. We don't need to record size itself in the body.
+         */
+
+        if (record_size <= RECORD_SIZE_LEN || record_size > MAX_RECORD_SIZE) {
+                telem_log(LOG_ERR, "Record size %u greater tham maximum allowed %lu."
+                                    "Recored ignored\n", record_size,
+                                    MAX_RECORD_SIZE);
+                goto end_client;      
+        }
+
+        buf_size = record_size - RECORD_SIZE_LEN;
+        cl->buf = calloc(1, buf_size);
+        if (!cl->buf) {
+                telem_log(LOG_ERR, "Unable to allocate memory, exiting\n");
+                exit(EXIT_FAILURE);
+        }
+        cl->size = buf_size;
+        cl->offset = 0;
+
+        /* Read the actual record*/
         do {
                 malloc_trim(0);
                 len = recv(cl->fd, cl->buf + cl->offset, cl->size - cl->offset, 0);
@@ -149,45 +186,20 @@ bool handle_client(TelemDaemon *daemon, nfds_t ind, client *cl)
                 }
 
                 cl->offset += (size_t)len;
-                if (cl->offset < RECORD_SIZE_LEN) {
-                        continue;
-                }
-                if (cl->size == RECORD_SIZE_LEN) {
-                        record_size = *(uint32_t *)(cl->buf);
-                        telem_log(LOG_DEBUG, "Total size of record: %zu\n", record_size);
-                        if (record_size == 0) {     //record_size < RECORD_MIN_SIZE || record_size > RECORD_MAX_SIZE
-                                goto end_client;
-                        }
-                        // We just processed the record size field, so the remaining format
-                        // is (header size field + record body + terminating '\0' byte)
-                        cl->size = sizeof(uint32_t) + record_size + 1;
-                        cl->buf = realloc(cl->buf, cl->size);
-                        memset(cl->buf, 0, cl->size);
-                        cl->offset = 0;
-                        if (!cl->buf) {
-                                telem_log(LOG_ERR, "Unable to allocate memory, exiting\n");
-                                exit(EXIT_FAILURE);
-                        }
-                }
-                if (cl->offset != cl->size) {
-                        /* full record not received yet */
-                        continue;
-                }
-                if (cl->size != RECORD_SIZE_LEN) {
-                        /* entire record has been received */
+
+                if (cl->offset == cl->size) {
                         process_record(daemon, cl);
-                        /* TODO: cleanup or terminate? */
-                        cl->offset = 0;
-                        cl->size = RECORD_SIZE_LEN;
+                        free(cl->buf);
+                        cl->buf = NULL;
                         processed = true;
-                        telem_log(LOG_DEBUG, "Record processed for client %d\n", cl->fd);
+                        telem_debug("DEBUG: Record processed for client %d\n", cl->fd);
                         break;
                 }
         } while (len > 0);
 
 end_client:
         telem_log(LOG_DEBUG, "Processed client %d: %s\n", cl->fd, processed ? "true" : "false");
-        terminate_client(daemon, cl, ind);
+        terminate_client(daemon, cl, index);
         return processed;
 }
 
@@ -237,14 +249,14 @@ char *read_machine_id_override()
         return machine_override;
 }
 
-void machine_id_replace(char **machine_header, char *machine_id_override)
+static void machine_id_replace(char **machine_header, char *machine_id_override)
 {
         char machine_id[33] = { 0 };
         char *old_header;
         int ret;
 
         if (machine_id_override) {
-                strcpy(machine_id, machine_id_override);
+                strncpy(machine_id, machine_id_override, sizeof(machine_id)-1);
         } else {
                 if (!get_machine_id(machine_id)) {
                         // TODO: decide if error handling is needed here
@@ -262,33 +274,93 @@ void machine_id_replace(char **machine_header, char *machine_id_override)
         free(old_header);
 }
 
-void process_record(TelemDaemon *daemon, client *cl)
+static void stage_record(char *filepath, char *headers[], char *body, char *cfg_file)
+{
+        int tmpfd;
+        FILE *tmpfile = NULL;
+
+        telem_debug("DEBUG: filepath:%s\n", filepath);
+        telem_debug("DEBUG: body:%s\n", body);
+        telem_debug("DEBUG: cfg:%s\n", cfg_file);
+
+        if (filepath == NULL) {
+                telem_log(LOG_ERR, "filepath value must be provided, aborting\n");
+                exit(EXIT_FAILURE);
+        }
+
+        tmpfd = mkstemp(filepath);
+        if (tmpfd < 0) {
+                telem_perror("Error opening staging file");
+                goto clean_exit;
+        }
+
+        // access the opened file as a stream
+        tmpfile = fdopen(tmpfd, "a");
+        if (!tmpfile) {
+                telem_perror("Error opening temp stage file");
+                close(tmpfd);
+                if (unlink(filepath)) {
+                        telem_perror("Error deleting temp stage file");
+                }
+                goto clean_exit;
+        }
+
+        // write cfg info if exists
+        if (cfg_file != NULL) {
+                fprintf(tmpfile, "%s%s\n", CFG_PREFIX, cfg_file);
+        }
+
+        // write headers
+        for (int i = 0; i < NUM_HEADERS; i++) {
+                fprintf(tmpfile, "%s\n", headers[i]);
+        }
+
+        // write body
+        fprintf(tmpfile, "%s\n", body);
+        fflush(tmpfile);
+        fclose(tmpfile);
+
+clean_exit:
+
+        return;
+}
+
+static void process_record(TelemDaemon *daemon, client *cl)
 {
         int i = 0;
+        int ret = 0;
         char *headers[NUM_HEADERS];
-        bool do_spool = false;
-        bool record_sent = false;
         char *tok = NULL;
         size_t header_size = 0;
         size_t message_size = 0;
         char *temp_headers = NULL;
         char *msg;
         char *body;
-        int current_minute = 0;
-        bool record_check_passed = true;
-        bool byte_check_passed = true;
-        bool record_burst_enabled = true;
-        bool byte_burst_enabled = true;
+        char *recordpath = NULL;
+        char *cfg_file = NULL;;
+        size_t cfg_info_size = 0;
+        uint8_t *buf;
 
-        /* Gets the current minute of time */
-        time_t temp = time(NULL);
-        struct tm *tm_s = localtime(&temp);
-        current_minute = tm_s->tm_min;
+        buf = cl->buf;
 
-        header_size = *(uint32_t *)cl->buf;
-        message_size = cl->size - header_size;
+        /* Check for an optional CFG_PREFIX in the first 32 bits */
+        if (*(uint32_t *)buf == CFG_PREFIX_32BIT) {
+                char *cfg  = (char *)cl->buf;
+
+                cfg_file = cfg + CFG_PREFIX_LENGTH;
+                cfg_info_size = CFG_PREFIX_LENGTH + strlen(cfg_file) + 1;
+                telem_debug("DEBUG: cfg_file: %s\n", cfg_file);
+        }
+
+        buf += cfg_info_size;
+        header_size = *(uint32_t *)buf;
+        message_size = cl->size - (cfg_info_size + header_size);
+        telem_debug("DEBUG: cl->size: %zu\n", cl->size);
+        telem_debug("DEBUG: header_size: %zu\n", header_size);
+        telem_debug("DEBUG: message_size: %zu\n", message_size);
+        telem_debug("DEBUG: cfg_info_size: %zu\n", cfg_info_size);
         assert(message_size > 0);      //TODO:Check for min and max limits
-        msg = (char *)cl->buf + sizeof(uint32_t);
+        msg = (char *)buf + sizeof(uint32_t);
 
         /* Copying the headers as strtok modifies the orginal buffer */
         temp_headers = strndup(msg, header_size);
@@ -297,7 +369,7 @@ void process_record(TelemDaemon *daemon, client *cl)
         for (i = 0; i < NUM_HEADERS; i++) {
                 const char *header_name = get_header_name(i);
 
-                if (get_header(tok, header_name, &headers[i], strlen(header_name))) {
+                if (get_header(tok, header_name, &headers[i])) {
                         if (strcmp(header_name, TM_MACHINE_ID_STR) == 0) {
                                 machine_id_replace(&headers[i], daemon->machine_id_override);
                         }
@@ -311,67 +383,15 @@ void process_record(TelemDaemon *daemon, client *cl)
         /* TODO : check if the body is within the limits. */
         body = msg + header_size;
 
-        if (inside_direct_spool_window(daemon, time(NULL))) {
-                telem_log(LOG_INFO, "process_record: delivering directly to spool\n");
-                spool_record(daemon, headers, body);
-                goto end;
-        }
-        if (daemon->record_window_length == -1 ||
-            daemon->byte_window_length == -1) {
-                telem_log(LOG_ERR, "Invalid value for window length\n");
+        /* Save record to stage */
+        ret = asprintf(&recordpath, "%s/XXXXXX", spool_dir_config());
+        if (ret == -1) {
+                telem_log(LOG_ERR, "Failed to allocate memory for record name in staging folder, aborting\n");
                 exit(EXIT_FAILURE);
         }
-        /* Checks if entirety of rate limiting is enabled */
-        if (daemon->rate_limit_enabled) {
-                /* Checks whether record and byte bursts are enabled individually */
-                record_burst_enabled = burst_limit_enabled(daemon->record_burst_limit);
-                byte_burst_enabled = burst_limit_enabled(daemon->byte_burst_limit);
-
-                if (record_burst_enabled) {
-                        record_check_passed = rate_limit_check(current_minute,
-                                                               daemon->record_burst_limit, daemon->record_window_length,
-                                                               daemon->record_burst_array, TM_RECORD_COUNTER);
-                }
-                if (byte_burst_enabled) {
-                        byte_check_passed = rate_limit_check(current_minute,
-                                                             daemon->byte_burst_limit, daemon->byte_window_length,
-                                                             daemon->byte_burst_array, cl->size);
-                }
-                /* If both record and byte burst disabled, rate limiting disabled */
-                if (!record_burst_enabled && !byte_burst_enabled) {
-                        daemon->rate_limit_enabled = false;
-                }
-        }
-        /* Sends record if rate limiting is disabled, or all checks passed */
-        if (!daemon->rate_limit_enabled || (record_check_passed && byte_check_passed)) {
-                /* Send the record as https post */
-                record_sent = post_record_ptr(headers, body, true);
-        }
-
-        // Get rate-limit strategy
-        do_spool = spool_strategy_selected(daemon);
-
-        // Drop record
-        if (!record_sent && !do_spool) {
-                goto end;
-        }
-        // Spool Record
-        else if (!record_sent && do_spool) {
-                start_network_bypass(daemon);
-                telem_log(LOG_INFO, "process_record: initializing direct-spool window\n");
-                spool_record(daemon, headers, body);
-        } else {
-                /* Updates rate limiting arrays if record sent */
-                if (record_burst_enabled) {
-                        rate_limit_update (current_minute, daemon->record_window_length,
-                                           daemon->record_burst_array, TM_RECORD_COUNTER);
-                }
-                if (byte_burst_enabled) {
-                        rate_limit_update (current_minute, daemon->byte_window_length,
-                                           daemon->byte_burst_array, cl->size);
-                }
-        }
 
+        stage_record(recordpath, headers, body, cfg_file);
+        free(recordpath);
 end:
         free(temp_headers);
         for (int k = 0; k < i; k++)
@@ -379,242 +399,6 @@ end:
         return;
 }
 
-bool burst_limit_enabled(int64_t burst_limit)
-{
-        return (burst_limit > -1) ? true : false;
-}
-
-bool rate_limit_check(int current_minute, int64_t burst_limit, int
-                      window_length, size_t *array, size_t incValue)
-{
-        size_t count = 0;
-
-        /* Variable to determine last minute allowed in burst window */
-        int window_start = (TM_RATE_LIMIT_SLOTS + (current_minute - window_length + 1))
-                           % TM_RATE_LIMIT_SLOTS;
-
-        /* The modulo is not placed in the for loop inself, because if
-         * the current minute is less than the window start (from wrapping)
-         * the for loop will never be entered.
-         */
-
-        /* Counts all elements within burst_window in array */
-        for (int i = window_start; i < (window_start + window_length); i++) {
-                count += array[i % TM_RATE_LIMIT_SLOTS];
-
-                if ((array[i % TM_RATE_LIMIT_SLOTS] + incValue) >= SIZE_MAX) {
-                        /* Exceeds maximum size array can hold */
-                        return false;
-                }
-        }
-        /* Include current record being processed to count */
-        count += incValue;
-
-        /* Determines whether the count has exceeded the limit or not */
-        return (count > burst_limit) ? false : true;
-}
-
-void rate_limit_update(int current_minute, int window_length, size_t *array,
-                       size_t incValue)
-{
-        /* Variable to determine the amount of zero'd out spots needed */
-        int blank_slots = (TM_RATE_LIMIT_SLOTS - window_length);
-
-        /* Update specified array depending on increment value */
-        array[current_minute] += incValue;
-
-        /* Zero out expired records for record array */
-        for (int i = current_minute + 1; i < (blank_slots + current_minute + 1); i++) {
-                array[i % TM_RATE_LIMIT_SLOTS] = 0;
-        }
-}
-
-bool spool_strategy_selected(TelemDaemon *daemon)
-{
-        /* Performs action depending on strategy choosen */
-        return (strcmp(daemon->rate_limit_strategy, "spool") == 0) ? true : false;
-}
-
-bool inside_direct_spool_window(TelemDaemon *daemon, time_t current_time)
-{
-        return (current_time < daemon->bypass_http_post_ts + 1800) ? true : false;
-}
-
-void start_network_bypass(TelemDaemon *daemon)
-{
-        daemon->bypass_http_post_ts = time(NULL);
-}
-
-size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
-{
-        telem_log(LOG_DEBUG, "Received data:\n%s\n", ptr);
-        return size * nmemb;
-}
-
-bool post_record_http(char *headers[], char *body, bool spool)
-{
-        CURL *curl;
-        int res = 0;
-        char *content = "Content-Type: application/text";
-        struct curl_slist *custom_headers = NULL;
-        char errorbuf[CURL_ERROR_SIZE];
-        long http_response = 0;
-        const char *cert_file = get_cainfo_config();
-        const char *tid_header = get_tidheader_config();
-
-        // Initialize the libcurl global environment once per POST. This lets us
-        // clean up the environment after each POST so that when the daemon is
-        // sitting idle, it will be consuming as little memory as possible.
-        curl_global_init(CURL_GLOBAL_ALL);
-
-        curl = curl_easy_init();
-        if (!curl) {
-                telem_log(LOG_ERR, "curl_easy_init(): Unable to start libcurl"
-                          " easy session, exiting\n");
-                exit(EXIT_FAILURE);
-                /* TODO: check if memory needs to be released */
-        }
-
-        // Errors for any curl_easy_* functions will store nice error messages
-        // in errorbuf, so send log messages with errorbuf contents
-        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf);
-
-        curl_easy_setopt(curl, CURLOPT_URL, server_addr_config());
-        curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 5L);
-        curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
-        curl_easy_setopt(curl, CURLOPT_POST, 1);
-#ifdef DEBUG
-        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
-#endif
-
-        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
-
-        for (int i = 0; i < NUM_HEADERS; i++) {
-                custom_headers = curl_slist_append(custom_headers, headers[i]);
-        }
-        custom_headers = curl_slist_append(custom_headers, tid_header);
-        // This should be set by probes/libtelemetry in the future
-        custom_headers = curl_slist_append(custom_headers, content);
-
-        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, custom_headers);
-        curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body);
-        curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strlen(body));
-        curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);
-
-        if (strlen(cert_file) > 0) {
-                if (access(cert_file, F_OK) != -1) {
-                        curl_easy_setopt(curl, CURLOPT_CAINFO, cert_file);
-                        telem_log(LOG_INFO, "cafile was set to %s\n", cert_file);
-                }
-        }
-
-        telem_log(LOG_DEBUG, "Executing curl operation...\n");
-        errorbuf[0] = 0;
-        res = curl_easy_perform(curl);
-        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_response);
-
-        if (res) {
-                size_t len = strlen(errorbuf);
-                if (len) {
-                        telem_log(LOG_ERR, "Failed sending record: %s%s", errorbuf,
-                                  ((errorbuf[len - 1] != '\n') ? "\n" : ""));
-                } else {
-                        telem_log(LOG_ERR, "Failed sending record: %s\n",
-                                  curl_easy_strerror(res));
-                }
-        } else if (http_response != 201 && http_response != 200) {
-                /*  201 means the record was  successfully created
-                 *  200 is a generic "ok".
-                 *  TODO: Make list of white-listed response codes configurable?
-                 */
-                telem_log(LOG_ERR, "Encountered error %ld on the server\n",
-                          http_response);
-                // We treat HTTP error codes the same as libcurl errors
-                res = 1;
-        } else {
-                telem_log(LOG_INFO, "Record sent successfully\n");
-        }
-
-        curl_slist_free_all(custom_headers);
-        curl_easy_cleanup(curl);
-
-        curl_global_cleanup();
-
-        return res ? false : true;
-}
-
-void spool_record(TelemDaemon *daemon, char *headers[], char *body)
-{
-        int ret = 0;
-        struct stat stat_buf;
-        int64_t max_spool_size = 0;
-        char *tmpbuf = NULL;
-        FILE *tmpfile = NULL;
-        int tmpfd = 0;
-
-        if (!daemon->is_spool_valid) {
-                /* If spool is not valid, simply drop the record */
-                return;
-        }
-
-        //check if the size is greater than 1 MB/spool max size
-        max_spool_size = spool_max_size_config();
-        if (max_spool_size != -1) {
-                telem_log(LOG_DEBUG, "Total size of spool dir: %ld\n", daemon->current_spool_size);
-                if (daemon->current_spool_size < 0) {
-                        telem_log(LOG_ERR, "Error getting spool directory size: %s\n",
-                                  strerror(-(int)(daemon->current_spool_size)));
-                        return;
-                } else if (daemon->current_spool_size >= (max_spool_size * 1024)) {
-                        telem_log(LOG_INFO, "Spool dir full, dropping record\n");
-                        return;
-                }
-        }
-
-        // create file with record
-        ret = asprintf(&tmpbuf, "%s/XXXXXX", spool_dir_config());
-        if (ret == -1) {
-                telem_log(LOG_ERR, "Failed to allocate memory for record name, aborting\n");
-                exit(EXIT_FAILURE);
-        }
-
-        tmpfd = mkstemp(tmpbuf);
-        if (tmpfd == -1) {
-                telem_perror("Error while creating temp file");
-                goto spool_err;
-        }
-
-        //fp = fopen(spool_dir_path, const char *mode);
-        tmpfile = fdopen(tmpfd, "a");
-        if (!tmpfile) {
-                telem_perror("Error opening temp file");
-                close(tmpfd);
-                if (unlink(tmpbuf)) {
-                        telem_perror("Error deleting temp file");
-                }
-                goto spool_err;
-        }
-
-        // write headers
-        for (int i = 0; i < NUM_HEADERS; i++) {
-                fprintf(tmpfile, "%s\n", headers[i]);
-        }
-
-        //write body
-        fprintf(tmpfile, "%s\n", body);
-        fflush(tmpfile);
-        fclose(tmpfile);
-
-        /* The stat should not fail here unless it is ENOMEM */
-        if (stat(tmpbuf, &stat_buf) == 0) {
-                daemon->current_spool_size += (stat_buf.st_blocks * 512);
-        }
-
-spool_err:
-        free(tmpbuf);
-        return;
-}
-
 void add_pollfd(TelemDaemon *daemon, int fd, short events)
 {
         assert(daemon);
@@ -669,7 +453,7 @@ bool get_machine_id(char *machine_id)
                 return false;
         }
 
-        ret = fscanf(id_file, "%s", machine_id);
+        ret = fscanf(id_file, "%32s", machine_id);
         if (ret != 1) {
                 telem_perror("Could not read machine id from file");
                 fclose(id_file);
@@ -679,7 +463,7 @@ bool get_machine_id(char *machine_id)
         return true;
 }
 
-int machine_id_write(uint64_t upper, uint64_t lower)
+int machine_id_write(char *new_id)
 {
         FILE *fp;
         int ret = 0;
@@ -691,7 +475,7 @@ int machine_id_write(uint64_t upper, uint64_t lower)
                 goto file_error;
         }
 
-        ret = fprintf(fp, "%.16" PRIx64 "%.16" PRIx64, upper, lower);
+        ret = fprintf(fp, "%s", new_id);
         if (ret < 0) {
                 telem_perror("Unable to write to machine id file");
                 goto file_error;
@@ -708,49 +492,17 @@ file_error:
 
 int generate_machine_id(void)
 {
-        int frandom = -1;
-        struct stat stat_buf;
-        int ret = 0;
-        struct iovec random_id[2];
-        uint64_t random_id_upper;
-        uint64_t random_id_lower;
-        int result = -1;
+        int result = 0;
+        char *new_id = NULL;
 
-        ret = stat("/dev/urandom", &stat_buf);
-        if (ret == -1) {
-                telem_log(LOG_ERR, "Unable to stat urandom device\n");
+        if ((result = get_random_id(&new_id)) != 0) {
                 goto rand_err;
         }
 
-        if (!S_ISCHR(stat_buf.st_mode)) {
-                telem_log(LOG_ERR, "/dev/urandom is not a character device file\n");
-                goto rand_err;
-        }
-
-        /* TODO : check for major and minor numbers to be extra sure ??
-           ((stat_buffer.st_rdev == makedev(1, 8)) || (stat_buffer.st_rdev == makedev(1, 9))*/
-
-        frandom = open("/dev/urandom", O_RDONLY);
-        if (frandom < 0) {
-                telem_perror("Error opening /dev/urandom");
-                goto rand_err;
-        }
-
-        random_id[0].iov_base = &random_id_upper;
-        random_id[0].iov_len = 8;
-        random_id[1].iov_base = &random_id_lower;
-        random_id[1].iov_len = 8;
-
-        if (readv(frandom, random_id, 2) < 0) {
-                telem_perror("Error while reading /dev/urandom");
-                goto rand_err;
-        }
-
-        result = machine_id_write(random_id_upper, random_id_lower);
+        result = machine_id_write(new_id);
 rand_err:
-        if (frandom >= 0) {
-                close(frandom);
-        }
+        free(new_id);
+
         return result;
 }
 

src/telemdaemon.h

@@ -54,23 +54,6 @@ typedef struct TelemDaemon {
         size_t current_alloc;
         /* client list head */
         client_list_head client_head;
-        /* Time of last failed post */
-        time_t bypass_http_post_ts;
-
-        /* Rate limit record and byte arrays */
-        size_t record_burst_array[TM_RATE_LIMIT_SLOTS];
-        size_t byte_burst_array[TM_RATE_LIMIT_SLOTS];
-        /* Rate Limit Configurations */
-        bool rate_limit_enabled;
-        int64_t record_burst_limit;
-        int record_window_length;
-        int64_t byte_burst_limit;
-        int byte_window_length;
-        const char *rate_limit_strategy;
-
-        bool is_spool_valid;
-        long current_spool_size;
-
         char *machine_id_override;
 } TelemDaemon;
 
@@ -79,14 +62,7 @@ typedef struct TelemDaemon {
  *
  * @param daemon A pointer to the daemon structure.
  */
-void initialize_daemon(TelemDaemon *daemon);
-
-/**
- * Initialize rate-limiting in the daemon
- *
- * @param daemon A pointer to the daemon structure
- */
-void initialize_rate_limit(TelemDaemon *daemon);
+void initialize_probe_daemon(TelemDaemon *daemon);
 
 /**
  * Add poll fd struct to the array of pollfds.
@@ -149,126 +125,6 @@ void remove_client(client_list_head *client_head, client *cl);
  */
 bool is_client_list_empty(client_list_head *client_head);
 
-/**
- * Terminate a client connection
- *
- * @param daemon Pointer to the daemon
- * @param cl Pointer to the client in the client list
- * @param index The index of the client file descriptor in the poll fd
- *    array
- *
- */
-void terminate_client(TelemDaemon *daemon, client *cl, nfds_t index);
-
-/**
- * Process record from a client
- *
- * @param daemon Pointer to the daemon
- * @param cl Pointer to the client in the client list
- *
- * @return true on success, false on failure
- */
-void process_record(TelemDaemon *daemon, client *cl);
-
-/**
- * Check if burst limiting is enabled
- *
- * @param burst_limit Burst limit amount. If negative rate limiting
- * is disabled
- *
- * @return true if rate limiting for burst_limit variable is enabled
- */
-bool burst_limit_enabled(int64_t burst_limit);
-
-/**
- * Checks whether record does not exceed the burst threshold
- *
- * @param current_minute The current minute of time the record processed
- * @param burst_limit Amount of records that can be passed within a window
- * @param window_length Window of time burst limit refers to
- * @param array Pointer to array that stores record or byte count
- * @param incValue Amount to increase array slot by
- *
- * @return true if record does not exceed threshold, and false otherwise
- */
-bool rate_limit_check(int current_minute, int64_t burst_limit, int
-                      window_length, size_t *array, size_t incValue);
-
-/**
- * Updates rate limit arrays with passing record information
- *
- * @param current_minute The current minute of time the record processed
- * @param window_length Window of time burst limit refers to
- * @param array Pointer to array that stores record or byte count
- * @param incValue Amount to increase array slot by
- *
- */
-void rate_limit_update(int current_minute, int window_length,
-                       size_t *array, size_t incValue);
-
-/**
- * Checks if the spool strategy is chosen for rate-limiting
- *
- * @param daemon Pointer to the daemon
- *
- * @return true if spool is choosen false if drop is choosen
- */
-bool spool_strategy_selected(TelemDaemon *daemon);
-
-/**
- * Check if daemon should direct-spool after encountering network issues.
- *
- * @param daemon Pointer to the daemon
- * @param time The time as time_t (usually from time())
- *
- * @return true if inside direct spool window, false otherwise.
- */
-bool inside_direct_spool_window(TelemDaemon *daemon, time_t time);
-
-/**
- * Set the start time of the direct spool window. Will use the
- * current time.
- *
- * @param daemon Pointer to the daemon
- *
- * @return no return value.
- */
-void start_network_bypass(TelemDaemon *daemon);
-
-/**
- * Function pointer for posting the records to the server. Replaced by a
- * mock function in the unit tests.
- *
- * @param header_values The header data read from the socket
- * @param body The payload of the record read
- * @param spool Indicate whether the record should be spooled if not
- *    successfully delivered
- *
- * @return true if the record was successfully posted
- */
-extern bool (*post_record_ptr)(char *headers[], char *body, bool spool);
-
-/**
- * Post records on a http connection to server
- *
- * @param header_values The header data read from the socket
- * @param body The payload of the record read
- * @param spool Indicate whether the record should be spooled if not
- *    successfully delivered
- *
- * @return true if the record was successfully posted
- */
-bool post_record_http(char *header_values[], char *body, bool spool);
-
-/**
- * Spool the record in case it cannot be delivered to the server
- *
- * @param headers The headers for the record
- * @param body The payload of the record
- *
- */
-void spool_record(TelemDaemon *daemon, char *headers[], char *body);
-
 /**
  * Get random machine id stored in file
  *
@@ -290,4 +146,5 @@ int update_machine_id(void);
  * or if the file is empty
  */
 char *read_machine_id_override(void);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/telemetry.c

@@ -15,6 +15,7 @@
  */
 
 #define _GNU_SOURCE
+
 #include <string.h>
 #include <stdio.h>
 #include <sys/stat.h>
@@ -30,9 +31,11 @@
 #include <inttypes.h>
 #include <ctype.h>
 
+#include "util.h"
 #include "common.h"
 #include "configuration.h"
 #include "telemetry.h"
+#include "log.h"
 
 /**
  * Return a file descriptor to either site's version file
@@ -74,9 +77,7 @@ static int set_header(char **dest, const char *prefix, char *value, size_t *head
         rc = asprintf(dest, "%s: %s\n", prefix, value);
 
         if (rc < 0) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         } else {
                 *header_size += (size_t)rc;
@@ -137,38 +138,14 @@ static int set_severity_header(struct telem_ref *t_ref, uint32_t severity)
  */
 static int set_classification_header(struct telem_ref *t_ref, char *classification)
 {
-        size_t i, j;
-        int slashes = 0;
-        int x = 0;
 
-        j = strlen(classification);
-
-        if (j > MAX_CLASS_LENGTH) {
-                return -EINVAL;
-        }
-
-        for (i = 0, x = 0; i <= (j - 1); i++, x++) {
-                if (classification[i] == '/') {
-                        slashes++;
-                        x = 0;
-                } else {
-                        if (x > MAX_SUBCAT_LENGTH) {
-                                return -EINVAL;
-                        }
-                }
-        }
-
-        if (slashes != 2) {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Classification string should have two /s.\n");
-#endif
+        if (validate_classification(classification) == 1) {
                 return -EINVAL;
         }
 
         return set_header(&(t_ref->record->headers[TM_CLASSIFICATION]),
                           TM_CLASSIFICATION_STR, classification,
                           &(t_ref->record->header_size));
-
 }
 
 /**
@@ -255,35 +232,49 @@ static int set_system_name_header(struct telem_ref *t_ref)
         int fd;
         char buf[SMALL_LINE_BUF] = { 0 };
         char name[SMALL_LINE_BUF] = { 0 };
+        char variant[SMALL_LINE_BUF] = { 0 };
 
         fd = version_file();
         if (fd == -1) {
-#ifdef DEBUG
-                fprintf(stderr, "WARNING: Cannot find os-release file\n");
-#endif
+                telem_log(LOG_WARNING, "WARNING: Cannot find os-release file\n");
                 sprintf(name, "unknown");
         } else {
                 fs = fdopen(fd, "r");
                 while (fgets(buf, SMALL_LINE_BUF, fs)) {
-                        if (sscanf(buf, "ID=%s", name) < 1) {
-                                continue;
-                        } else {
+                        if (sscanf(buf, "ID=%s", name) > 0) {
+                                break;
+                        }
+                }
+                rewind(fs);
+                while (fgets(buf, SMALL_LINE_BUF, fs)) {
+                        if (sscanf(buf, "VARIANT_ID=%s", variant) > 0) {
                                 break;
                         }
                 }
 
+                fclose(fs);
+
                 if (strlen(name) == 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "WARNING: Cannot find os-release field: ID\n");
-#endif
+                        telem_log(LOG_WARNING, "WARNING: Cannot find os-release field: ID\n");
                         sprintf(name, "unknown");
                 }
 
-                fclose(fs);
+                if (strlen(variant) == 0) {
+                        snprintf(buf, sizeof(buf), "%s", name);
+                } else {
+                        /* We do not want the header to be greater than 80 characters, allow truncation */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-truncation"
+                        if (snprintf(buf, sizeof(buf), "%s_%s", name, variant) > sizeof(buf)) {
+                                telem_log(LOG_WARNING, "WARNING: truncating system name header to 80 characters\n");
+                        }
+                        telem_debug("DEBUG: Found variant OS information: %s\n", variant);
+#pragma GCC diagnostic pop
+                }
         }
 
         return set_header(&(t_ref->record->headers[TM_SYSTEM_NAME]),
-                          TM_SYSTEM_NAME_STR, name,
+                          TM_SYSTEM_NAME_STR, buf,
                           &(t_ref->record->header_size));
 
 }
@@ -307,9 +298,7 @@ static int set_system_build_header(struct telem_ref *t_ref)
 
         fd = version_file();
         if (fd == -1) {
-#ifdef DEBUG
-                fprintf(stderr, "WARNING: Cannot find build version file\n");
-#endif
+                telem_log(LOG_WARNING, "WARNING: Cannot find build version file\n");
                 sprintf(version, "0");
         } else {
                 fs = fdopen(fd, "r");
@@ -322,9 +311,7 @@ static int set_system_build_header(struct telem_ref *t_ref)
                 }
 
                 if (strlen(version) == 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "WARNING: Cannot find build version number\n");
-#endif
+                        telem_log(LOG_WARNING, "WARNING: Cannot find build version number\n");
                         sprintf(version, "0");
                 }
 
@@ -338,7 +325,7 @@ static int set_system_build_header(struct telem_ref *t_ref)
 
 /**
  * Sets the header reporting the machine_id. Note, the machine_id is really
- * managed by the telemetrics daemon (telemd), and populated by the daemon
+ * managed by the telemetrics daemon (telemprobd), and populated by the daemon
  * as it delivers the records. Consequently, the header is simply set to a
  * dummy value of 0xFFFFFFF.
  *
@@ -400,6 +387,60 @@ static int set_timestamp_header(struct telem_ref *t_ref)
         return status;
 }
 
+/**
+ * Sets cpu model for telemetry record. The information from cpu is extracted
+ * from /proc/cpuinfo, specifically "model name" attribute.
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_cpu_model_header(struct telem_ref *t_ref)
+{
+        FILE *fs = NULL;
+        char buf[SMALL_LINE_BUF] = { 0 };
+        char *model_name = NULL;
+        const char *attr_name = "model name";
+        int status = 0;
+        size_t attr_len = strlen(attr_name);
+        size_t model_str_len = 0;
+
+        fs = fopen("/proc/cpuinfo", "r");
+        if (fs != NULL) {
+                while (fgets(buf, SMALL_LINE_BUF, fs)) {
+                        if (strncmp(attr_name, buf, attr_len) == 0) {
+                                model_name = strchr(buf, ':');
+                                break;
+                        }
+                }
+                fclose(fs);
+                if (model_name != NULL) {
+                        model_str_len = strlen(model_name);
+                        if (model_str_len > 2) {
+                                model_name = (char *)model_name + 2 * sizeof(char);
+                                model_name[model_str_len - 3] = '\0';
+                        } else {
+                                model_name = "blank";
+                        }
+                } else {
+                        model_name = "blank";
+                        telem_log(LOG_NOTICE, "NOTICE: Unable to find attribute:%s\n", attr_name);
+                }
+
+                status = set_header(
+                        &(t_ref->record->headers[TM_CPU_MODEL]),
+                        TM_CPU_MODEL_STR, model_name,
+                        &(t_ref->record->header_size));
+
+        } else {
+                telem_log(LOG_NOTICE, "NOTICE: Unable to open /proc/cpuinfo\n");
+                status = -1;
+        }
+
+        return status;
+}
+
 /**
  * A healper function for set_host_type_header that reads the first line out of
  * a file and chomps the newline if necessary. If the file does not exist the
@@ -427,9 +468,7 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
         *buf = (char *)malloc(sizeof(char) * SMALL_LINE_BUF);
 
         if (*buf == NULL) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         }
 
@@ -452,9 +491,7 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
                 if (new_size == 0) {
                         old_value = *buf;
                         if (asprintf(buf, "blank") < 0) {
-#ifdef DEBUG
-                                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                                 ret = -ENOMEM;
                         }
                         free(old_value);
@@ -471,24 +508,18 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
                         if (j == new_size) {
                                 old_value = *buf;
                                 if (asprintf(buf, "blank") < 0) {
-#ifdef DEBUG
-                                        fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                                        telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                                         ret = -ENOMEM;
                                 }
                                 free(old_value);
                         }
                 }
         } else {
-#ifdef DEBUG
-                fprintf(stderr, "NOTICE: Dmi file %s does not exist\n", source);
-#endif
+                telem_debug("DEBUG: Dmi file %s does not exist\n", source);
 
                 old_value = *buf;
                 if (asprintf(buf, "no_%s_file", key) < 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                        telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                         ret = -ENOMEM;
                 }
                 free(old_value);
@@ -497,6 +528,118 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
         return ret;
 }
 
+/**
+ * Sets the board name for telemetry record, this record is a combination of
+ * board name and board vendor. This information is read from dmi filesystem
+ * Board Name (board_name) and Board Vendor (board_vendor).
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_board_name_header(struct telem_ref *t_ref)
+{
+        int status = 0;
+        int rc;
+        char *buf = NULL;
+        char *bn = NULL;
+        char *bv = NULL;
+
+        rc  = get_dmi_value("/sys/class/dmi/id/board_name", "bn", &bn);
+        if (rc < 0) {
+                status = rc;
+                goto cleanup;
+        }
+
+        rc  = get_dmi_value("/sys/class/dmi/id/board_vendor", "bv", &bv);
+        if (rc < 0) {
+                status = rc;
+                goto cleanup;
+        }
+
+        rc = asprintf(&buf, "%s|%s", bn, bv);
+
+        if (rc < 0) {
+                status = -ENOMEM;
+                goto cleanup;
+        } else {
+                status = set_header(
+                        &(t_ref->record->headers[TM_BOARD_NAME]),
+                        TM_BOARD_NAME_STR, buf,
+                        &(t_ref->record->header_size));
+                free(buf);
+        }
+
+cleanup:
+        if (bn != NULL) {
+                free(bn);
+        }
+
+        if (bv != NULL) {
+                free(bv);
+        }
+
+        return status;
+}
+
+/**
+ * Sets BIOS version header for telemetry record, this information is
+ * read from dmi filesystem BIOS Version (bios_version).
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_bios_version_header(struct telem_ref *t_ref)
+{
+        int status = 0;
+        int rc = 0;
+        char *bios_version  = NULL;
+
+        rc  = get_dmi_value("/sys/class/dmi/id/bios_version", "bv", &bios_version);
+        if (rc < 0) {
+                status = rc;
+        } else {
+                status = set_header(
+                        &(t_ref->record->headers[TM_BIOS_VERSION]),
+                        TM_BIOS_VERSION_STR, bios_version,
+                        &(t_ref->record->header_size));
+                free(bios_version);
+        }
+
+        return status;
+}
+
+/**
+ * Sets the event_id header, this id is an identifier that multiple
+ * records can share. This means that one event can lead to multiple
+ * records.
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, -1 on failure
+ *
+ */
+static int set_event_id_header(struct telem_ref *t_ref)
+{
+        int rc = 0;
+        char *buff = NULL;
+
+        rc = get_random_id(&buff);
+
+        if (rc == 0) {
+                rc = set_header(
+                        &(t_ref->record->headers[TM_EVENT_ID]),
+                        TM_EVENT_ID_STR, buff,
+                        &(t_ref->record->header_size));
+        }
+        free(buff);
+
+        return rc;
+}
+
 /**
  * Sets the hosttype header, which is a tuple of three values looked for
  * in the dmi filesystem.  System Vendor (sys_vendor), Product Name
@@ -631,9 +774,9 @@ static int set_payload_format_header(struct telem_ref *t_ref, uint32_t payload_v
         return status;
 }
 
-void tm_set_config_file(char *c_file)
+int tm_set_config_file(const char *c_file)
 {
-        set_config_file(c_file);
+        return set_config_file(c_file);
 }
 
 /**
@@ -730,6 +873,30 @@ int allocate_header(struct telem_ref *t_ref, uint32_t severity,
                 goto free_and_fail;
         }
 
+        i++;
+
+        if ((ret = set_board_name_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_cpu_model_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_bios_version_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_event_id_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
         i++; /* Not necessary, but including for future expansion */
 
         return ret;
@@ -746,28 +913,16 @@ int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
                      char *classification, uint32_t payload_version)
 {
         int ret = 0;
-        int k = 0;
-        struct stat unused;
-
-        k = stat(TM_OPT_OUT_FILE, &unused);
-        if (k == 0) {
-                // Bail early if opt-out is enabled
-                return -ECONNREFUSED;
-        }
 
         *t_ref = (struct telem_ref *)malloc(sizeof(struct telem_ref));
         if (*t_ref == NULL) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         }
 
         (*t_ref)->record = (struct telem_record *)malloc(sizeof(struct telem_record));
         if ((*t_ref)->record == NULL) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 free(*t_ref);
                 return -ENOMEM;
         }
@@ -821,14 +976,6 @@ int tm_set_payload(struct telem_ref *t_ref, char *payload)
 {
         size_t payload_len;
         int ret = 0;
-        int k = 0;
-        struct stat unused;
-
-        k = stat(TM_OPT_OUT_FILE, &unused);
-        if (k == 0) {
-                // Bail early if opt-out is enabled
-                return -ECONNREFUSED;
-        }
 
         payload_len = strlen((char *)payload);
 
@@ -840,26 +987,72 @@ int tm_set_payload(struct telem_ref *t_ref, char *payload)
                 return -EINVAL;
         }
 
-        t_ref->record->payload = (char *)malloc(sizeof(char) * payload_len + 1);
+        t_ref->record->payload = strdup(payload);
 
         if (!t_ref->record->payload) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         }
 
-        memset(t_ref->record->payload, 0, sizeof(char) * payload_len + 1);
-
-        strncpy((char *)(t_ref->record->payload), (char *)payload, payload_len);
-
         t_ref->record->payload_size = payload_len;
 
         return ret;
 }
 
 /**
- * Write nbytes from buf to fd. Used to send records to telemd.
+ * Checks for id to have length = 32 characters
+ * and hexadecimal characters only.
+ *
+ * @param id A pointer to string to be checked
+ *
+ * @return 0 on success, 1 on failure
+ *
+ */
+static int validate_event_id(char *id)
+{
+        int rc = 0;
+        const char alphab[] = EVENT_ID_ALPHAB;
+
+        // make sure is not null
+        if (!id) {
+                return 1;
+        }
+
+        // 32 - 32 = 0
+        if ((rc = (int)(strlen(id) - EVENT_ID_LEN)) != 0) {
+                return 1;
+        }
+
+        // verify alphabet
+        // strspn checks for characters in alphab
+        if (strspn(id, alphab) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        return rc;
+}
+
+int tm_set_event_id(struct telem_ref *t_ref, char *event_id)
+{
+        int rc = -1;
+
+        if (!validate_event_id(event_id)) {
+                if (t_ref && t_ref->record) {
+                        // free default id before overriding
+                        free(t_ref->record->headers[TM_EVENT_ID]);
+                        // set new event_id
+                        if (asprintf(&(t_ref->record->headers[TM_EVENT_ID]),
+                                     "%s: %s\n", TM_EVENT_ID_STR, event_id) > 0) {
+                                rc = 0;
+                        }
+                }
+        }
+
+        return rc;
+}
+
+/**
+ * Write nbytes from buf to fd. Used to send records to telemprobd.
  *
  * @param fd Socket fd obtained from tm_get_socket.
  * @param buf Data to be written to the socket.
@@ -880,10 +1073,7 @@ static int tm_write_socket(int fd, char *buf, size_t nbytes)
 
                 if (b == -1 && errno != EAGAIN) {
                         ret = -errno;
-#ifdef DEBUG
-                        fprintf(stderr, "ERR: Write to daemon socket with"
-                                " system error: %s\n", strerror(errno));
-#endif
+                        telem_perror("Error writing to daemon socket");
                         return ret;
                 } else if (b == -1 &&
                            (errno == EAGAIN || errno == EWOULDBLOCK)) {
@@ -923,10 +1113,7 @@ static int tm_get_socket(void)
 
         if (sfd == -1) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Attempt to allocate socket fd failed:"
-                        " %s\n", strerror(errno));
-#endif
+                telem_perror("Attempt to allocate socket fd failed");
                 return ret;
         }
 
@@ -937,10 +1124,7 @@ static int tm_get_socket(void)
         if (setsockopt(sfd, SOL_SOCKET, SO_RCVTIMEO, &tv,
                        sizeof(tv)) < 0) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to set socket timeout:"
-                        " %s\n", strerror(errno));
-#endif
+                telem_perror("Failed to set socket timeout");
                 goto out1;
         }
 
@@ -996,9 +1180,7 @@ static int tm_get_socket(void)
         // Set non-blocking after the connect() succeeded
         if ((sflags = fcntl(sfd, F_GETFL, NULL)) < 0) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to get socket flags\n");
-#endif
+                telem_log(LOG_ERR, "Failed to get socket flags\n");
                 goto out1;
         }
 
@@ -1006,9 +1188,7 @@ static int tm_get_socket(void)
 
         if (fcntl(sfd, F_SETFL, sflags) < 0) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to set socket flags\n");
-#endif
+                telem_log(LOG_ERR, "Failed to set socket flags\n");
                 goto out1;
         }
 
@@ -1019,6 +1199,17 @@ out1:
         return ret;
 }
 
+int tm_is_opted_in(void)
+{
+        struct stat unused;
+
+        if (stat(TM_OPT_IN_FILE, &unused) == -1) {
+                return 0;
+        }
+
+        return 1;
+}
+
 int tm_send_record(struct telem_ref *t_ref)
 {
         int i;
@@ -1028,55 +1219,77 @@ int tm_send_record(struct telem_ref *t_ref)
         char *data = NULL;
         size_t offset = 0;
         int ret = 0;
-        int k = 0;
-        struct stat unused;
+        size_t cfg_file_name_size = 0;
+        const char *cfg_file_name = NULL;
 
-        k = stat(TM_OPT_OUT_FILE, &unused);
-        if (k == 0) {
-                // Bail early if opt-out is enabled
+        if (tm_is_opted_in() == 0) {
+                // Bail early if opt-in is not existent
                 return -ECONNREFUSED;
         }
 
         sfd = tm_get_socket();
 
         if (sfd < 0) {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to get socket fd: %s\n",
-                        strerror(-sfd));
-#endif
+                telem_log(LOG_ERR, "Failed to get socket fd: %s\n",
+                          strerror(-sfd));
                 return sfd;
         }
 
         total_size = t_ref->record->header_size + t_ref->record->payload_size;
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Header size : %zu\n", t_ref->record->header_size);
-        fprintf(stderr, "DEBUG: Payload size : %zu\n", t_ref->record->payload_size);
 
-        fprintf(stderr, "DEBUG: Total size : %zu\n", total_size);
-#endif
+        /*
+         * The user may want to use a custom (cmd line specified) config file.
+         * If that is the case, we need to include the absolute path in the data packet.
+         * We optionally add the string CFG:<absolute path> as the first string
+         * in the data. If the receiving end sees this in the data, it should
+         * use the config file specified to override any default telemetrics.conf
+         * files.
+         */
+
+        cfg_file_name = get_cmd_line_config_file();
+        if (cfg_file_name != NULL) {
+                cfg_file_name_size = strlen(cfg_file_name) + 1;
+                total_size += (cfg_file_name_size + CFG_PREFIX_LENGTH);
+        }
+
+        if (cfg_file_name != NULL) {
+                telem_debug("DEBUG: CFG field size : %zu\n", cfg_file_name_size + CFG_PREFIX_LENGTH);
+                telem_debug("DEBUG: CFG file name : %s\n", cfg_file_name);
+        }
+
+        telem_debug("DEBUG: Header size : %zu\n", t_ref->record->header_size);
+        telem_debug("DEBUG: Payload size : %zu\n", t_ref->record->payload_size);
+
+        telem_debug("DEBUG: Total size : %zu\n", total_size);
 
         /*
          * Allocating buffer for what we intend to send.  Buffer layout is:
-         * <uint32_t total_size><uint32_t header_size><headers + Payload>
+         * <uint32_t record_size>     : so recv knows how much to read
+         * <custom cfg file field>    : optional
+         * <uint32_t header_size>
+         * <headers + Payload>
          * <null-byte>
          * The additional char at the end ensures null termination
          */
         record_size = (2 * sizeof(uint32_t)) + total_size + 1;
 
-        data = malloc(record_size);
+        data = calloc(sizeof(char), record_size);
         if (!data) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 close(sfd);
                 return -ENOMEM;
         }
 
-        memset(data, 0, record_size);
-
-        memcpy(data, &total_size, sizeof(uint32_t));
+        memcpy(data, &record_size, sizeof(uint32_t));
         offset += sizeof(uint32_t);
 
+        if (cfg_file_name != NULL) {
+                memcpy(data + offset, CFG_PREFIX, CFG_PREFIX_LENGTH);
+                offset += CFG_PREFIX_LENGTH;
+                memcpy(data + offset, cfg_file_name, cfg_file_name_size);
+                offset += cfg_file_name_size;
+        }
+
         memcpy(data + offset, &t_ref->record->header_size, sizeof(uint32_t));
         offset += sizeof(uint32_t);
 
@@ -1089,17 +1302,11 @@ int tm_send_record(struct telem_ref *t_ref)
 
         memcpy(data + offset, t_ref->record->payload, t_ref->record->payload_size);
 
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Data to be sent :\n\n%s\n", data + 2 * sizeof(uint32_t));
-#endif
+        telem_debug("DEBUG: Data to be sent :\n\n%s\n", data + 2 * sizeof(uint32_t));
         if ((ret = tm_write_socket(sfd, data, record_size)) == 0) {
-#ifdef DEBUG
-                fprintf(stderr, "INFO: Successfully sent record over the socket\n");
-#endif
+                telem_log(LOG_INFO, "INFO: Successfully sent record over the socket\n");
         } else {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Error while writing data to socket\n");
-#endif
+                telem_log(LOG_ERR, "Error while writing data to socket\n");
         }
 
         close(sfd);

src/telemetry.h

@@ -31,7 +31,10 @@
  * - a library, libtelemetry, that telemetrics probes use to create telemetrics
  *   records and send them to the daemon for further processing.
  *
- * - a daemon, telemd, that prepares the records to send to a telemetrics server
+ * - telemprobd, a daemon that handles communication with probes and forwards
+ *   records to a second daemon.
+ *
+ * - telempostd, this daemon sends telemetry records to a telemetrics server
  *   (not included in this source tree), or spools the records on disk in case
  *   it's unable to successfully deliver them.
  *
@@ -55,11 +58,11 @@ struct telem_ref {
 /**
  * Set the configuration file name to use
  *
- * @param c_file Absolute file name of the configuration file. If the path name
- * provided is not an absolute path, or the path given cannot be successfuly
- * stat'd, then the effective configuration file is left unchanged.
+ * @param c_file Absolute file name of the configuration file.
+ * @return 0 on success, or a negative errno-style value on error. In case of 
+ *     an error return the effective configuration file is left unchanged.
  */
-void tm_set_config_file(char *c_file);
+int tm_set_config_file(const char *c_file);
 
 /**
  * Create a new telemetrics record
@@ -85,6 +88,19 @@ void tm_set_config_file(char *c_file);
 int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
                      char *classification, uint32_t payload_version);
 
+/**
+ * Sets the event_id to a user defined id
+ *
+ * @param t_ref A pointer to a telem_ref struct that was previously initialized
+ *        this struct will have an event_id already initialized and it will be
+ *        overriden by a user provided event_id
+ * @param event_id A pointer to a string that contains the 32 character length
+ *        event_id
+ *
+ * @return 0 on success, or a negative errno-style value on error
+ */
+int tm_set_event_id(struct telem_ref *t_ref, char *event_id);
+
 /**
  * Set the payload field of a telemetrics record
  *
@@ -104,6 +120,13 @@ int tm_set_payload(struct telem_ref *t_ref, char *payload);
  */
 int tm_send_record(struct telem_ref *t_ref);
 
+/**
+ * Checks if telemetry was opted in
+ *
+ * @return 1 when opt-in, or 0 when opt-out
+ */
+int tm_is_opted_in(void);
+
 /**
  * Release the memory allocated to a telemetrics record.
  *

src/telemetry.sym

@@ -16,3 +16,13 @@ TM_3_0_0 {
     tm_set_payload;
     tm_send_record;
 } TM_2_0_0;
+
+TM_4_0_0 {
+  global:
+    tm_set_event_id;
+} TM_3_0_0;
+
+TM_4_1_0 {
+  global:
+    tm_is_opted_in;
+} TM_4_0_0;

src/telempostdaemon.c

@@ -0,0 +1,789 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <signal.h>
+#include <dirent.h>
+#include <malloc.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+#include <curl/curl.h>
+#include <sys/signalfd.h>
+
+#include "log.h"
+#include "util.h"
+#include "spool.h"
+#include "iorecord.h"
+#include "retention.h"
+#include "telempostdaemon.h"
+
+/* spool window check */
+static bool inside_direct_spool_window(TelemPostDaemon *daemon, time_t current_time)
+{
+        return (current_time < daemon->bypass_http_post_ts + NETWORK_BYPASS_DURATION) ? true : false;
+}
+
+/* set http bypass */
+static void start_network_bypass(TelemPostDaemon *daemon)
+{
+        daemon->bypass_http_post_ts = time(NULL);
+}
+
+/* burst limit check  */
+bool burst_limit_enabled(int64_t burst_limit)
+{
+        return (burst_limit > -1) ? true : false;
+}
+
+/* rate limit check */
+bool rate_limit_check(int current_minute, int64_t burst_limit, int
+                      window_length, size_t *array, size_t incValue)
+{
+        size_t count = 0;
+
+        /* Variable to determine last minute allowed in burst window */
+        int window_start = (TM_RATE_LIMIT_SLOTS + (current_minute - window_length + 1))
+                           % TM_RATE_LIMIT_SLOTS;
+
+        /* The modulo is not placed in the for loop itself, because if
+         * the current minute is less than the window start (from wrapping)
+         * the for loop will never be entered.
+         */
+
+        /* Counts all elements within burst_window in array */
+        for (int i = window_start; i < (window_start + window_length); i++) {
+                count += array[i % TM_RATE_LIMIT_SLOTS];
+
+                if ((array[i % TM_RATE_LIMIT_SLOTS] + incValue) >= SIZE_MAX) {
+                        /* Exceeds maximum size array can hold */
+                        return false;
+                }
+        }
+        /* Include current record being processed to count */
+        count += incValue;
+
+        /* Determines whether the count has exceeded the limit or not */
+        return (count > burst_limit) ? false : true;
+}
+
+/* spool strategy check */
+bool spool_strategy_selected(TelemPostDaemon *daemon)
+{
+        /* Performs action depending on strategy choosen */
+        return (strcmp(daemon->rate_limit_strategy, "spool") == 0) ? true : false;
+}
+
+/* rate limit check  */
+void rate_limit_update(int current_minute, int window_length, size_t *array,
+                       size_t incValue)
+{
+        /* Variable to determine the amount of zero'd out spots needed */
+        int blank_slots = (TM_RATE_LIMIT_SLOTS - window_length);
+
+        /* Update specified array depending on increment value */
+        array[current_minute] += incValue;
+        /* Zero out expired records for record array */
+        for (int i = current_minute + 1; i < (blank_slots + current_minute + 1); i++) {
+                array[i % TM_RATE_LIMIT_SLOTS] = 0;
+        }
+}
+
+static void set_pollfd(TelemPostDaemon *daemon, int fd, enum fdindex i, short events)
+{
+        assert(daemon);
+        assert(fd);
+
+        daemon->pollfds[i].fd = fd;
+        daemon->pollfds[i].events = events;
+        daemon->pollfds[i].revents = 0;
+}
+
+static void initialize_signals(TelemPostDaemon *daemon)
+{
+        int sigfd;
+        sigset_t mask;
+
+        sigemptyset(&mask);
+
+        if (sigaddset(&mask, SIGHUP) != 0) {
+                telem_perror("Error adding signal SIGHUP to mask");
+                exit(EXIT_FAILURE);
+        }
+
+        if (sigaddset(&mask, SIGINT) != 0) {
+                telem_perror("Error adding signal SIGINT to mask");
+                exit(EXIT_FAILURE);
+        }
+
+        if (sigaddset(&mask, SIGTERM) != 0) {
+                telem_perror("Error adding signal SIGTERM to mask");
+                exit(EXIT_FAILURE);
+        }
+
+        if (sigaddset(&mask, SIGPIPE) != 0) {
+                telem_perror("Error adding signal SIGPIPE to mask");
+                exit(EXIT_FAILURE);
+        }
+
+        if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) {
+                telem_perror("Error changing signal mask with SIG_BLOCK");
+                exit(EXIT_FAILURE);
+        }
+
+        sigfd = signalfd(-1, &mask, 0);
+        if (sigfd == -1) {
+                telem_perror("Error creating the signalfd");
+                exit(EXIT_FAILURE);
+        }
+
+        daemon->sfd = sigfd;
+
+        set_pollfd(daemon, sigfd, signlfd, POLLIN);
+}
+
+static void initialize_rate_limit(TelemPostDaemon *daemon)
+{
+        for (int i = 0; i < TM_RATE_LIMIT_SLOTS; i++) {
+                daemon->record_burst_array[i] = 0;
+                daemon->byte_burst_array[i] = 0;
+        }
+        daemon->rate_limit_enabled = rate_limit_enabled_config();
+        daemon->record_burst_limit = record_burst_limit_config();
+        daemon->record_window_length = record_window_length_config();
+        daemon->byte_burst_limit = byte_burst_limit_config();
+        daemon->byte_window_length = byte_window_length_config();
+        daemon->rate_limit_strategy = rate_limit_strategy_config();
+}
+
+static void initialize_record_delivery(TelemPostDaemon *daemon)
+{
+        daemon->record_retention_enabled = record_retention_enabled_config();
+        daemon->record_server_delivery_enabled = record_server_delivery_enabled_config();
+}
+
+void initialize_post_daemon(TelemPostDaemon *daemon)
+{
+        assert(daemon);
+
+        daemon->bypass_http_post_ts = 0;
+        daemon->is_spool_valid = is_spool_valid();
+        daemon->record_journal = open_journal(JOURNAL_PATH);
+        daemon->fd = inotify_init();
+        if (daemon->fd < 0) {
+                telem_perror("Error initializing inotify");
+                exit(EXIT_FAILURE);
+        }
+        daemon->wd = inotify_add_watch(daemon->fd, spool_dir_config(), IN_CLOSE_WRITE);
+
+        initialize_signals(daemon);
+        set_pollfd(daemon, daemon->fd, watchfd, POLLIN);
+
+        initialize_rate_limit(daemon);
+        initialize_record_delivery(daemon);
+        /* Register record retention delete action as a callback to prune entry */
+        if (daemon->record_journal != NULL && daemon->record_retention_enabled) {
+                daemon->record_journal->prune_entry_callback = &delete_record_by_id;
+        }
+        daemon->current_spool_size = 0;
+}
+
+size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
+{
+        telem_log(LOG_DEBUG, "Received data:\n%s\n", ptr);
+        return size * nmemb;
+}
+
+bool post_record_http(char *headers[], char *body, char *cfg)
+{
+        CURL *curl;
+        int res = 0;
+        char *content = "Content-Type: application/text";
+        struct curl_slist *custom_headers = NULL;
+        char errorbuf[CURL_ERROR_SIZE];
+        long http_response = 0;
+        const char *cert_file = get_cainfo_config();
+        const char *tid_header = get_tidheader_config();
+        const char *saved_config_file = NULL;
+
+        if (cfg != NULL) {
+                saved_config_file = get_config_file();
+                if (set_config_file(cfg) != 0) {
+                       telem_log(LOG_ERR, "set-config_file(): Failed to set %s\n", cfg);
+                       // If we fail to load the specified config file, do not send the
+                       // record out. We don't want to send the record out with different
+                       // settings than explicitly requested.
+                       // However, report success so the record gets deleted.
+                       res = 0;
+                       goto Done;
+                }
+                reload_config();
+                telem_debug("DEBUG: override server_addr:%s\n", server_addr_config());
+        }
+
+        // Initialize the libcurl global environment once per POST. This lets us
+        // clean up the environment after each POST so that when the daemon is
+        // sitting idle, it will be consuming as little memory as possible.
+        curl_global_init(CURL_GLOBAL_ALL);
+
+        curl = curl_easy_init();
+        if (!curl) {
+                telem_log(LOG_ERR, "curl_easy_init(): Unable to start libcurl"
+                          " easy session, exiting\n");
+                exit(EXIT_FAILURE);
+                /* TODO: check if memory needs to be released */
+        }
+
+        // Errors for any curl_easy_* functions will store nice error messages
+        // in errorbuf, so send log messages with errorbuf contents
+        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf);
+
+        curl_easy_setopt(curl, CURLOPT_URL, server_addr_config());
+        curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 5L);
+        curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
+        curl_easy_setopt(curl, CURLOPT_POST, 1);
+#ifdef DEBUG
+        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+#endif
+
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+
+        for (int i = 0; i < NUM_HEADERS; i++) {
+                custom_headers = curl_slist_append(custom_headers, headers[i]);
+        }
+        custom_headers = curl_slist_append(custom_headers, tid_header);
+        // This should be set by probes/libtelemetry in the future
+        custom_headers = curl_slist_append(custom_headers, content);
+
+        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, custom_headers);
+        curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body);
+        curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strlen(body));
+        curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);
+
+        if (strlen(cert_file) > 0) {
+                if (access(cert_file, F_OK) != -1) {
+                        curl_easy_setopt(curl, CURLOPT_CAINFO, cert_file);
+                        telem_log(LOG_INFO, "cafile was set to %s\n", cert_file);
+                }
+        }
+
+        telem_log(LOG_DEBUG, "Executing curl operation...\n");
+        errorbuf[0] = 0;
+        res = curl_easy_perform(curl);
+        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_response);
+
+        if (res) {
+                size_t len = strlen(errorbuf);
+                if (len) {
+                        telem_log(LOG_DEBUG, "Failed sending record: %s%s", errorbuf,
+                                  ((errorbuf[len - 1] != '\n') ? "\n" : ""));
+                } else {
+                        telem_log(LOG_DEBUG, "Failed sending record: %s\n",
+                                  curl_easy_strerror(res));
+                }
+        } else if (http_response != 201 && http_response != 200) {
+                /*  201 means the record was  successfully created
+                 *  200 is a generic "ok".
+                 */
+                telem_log(LOG_ERR, "Encountered error %ld on the server\n",
+                          http_response);
+                // We treat HTTP error codes the same as libcurl errors
+                res = 1;
+        } else {
+                telem_log(LOG_INFO, "Record sent successfully\n");
+        }
+
+        curl_slist_free_all(custom_headers);
+        curl_easy_cleanup(curl);
+
+        curl_global_cleanup();
+
+Done:
+        if (saved_config_file != NULL) {
+                if (set_config_file(saved_config_file) != 0) {
+                        telem_log(LOG_ERR, "set-config_file(): Failed to set %s",
+                                  saved_config_file);
+                        res = 1;
+                }
+                reload_config();
+                telem_debug("DEBUG: restored server_addr:%s\n", server_addr_config());
+        }
+
+        return res ? false : true;
+}
+
+static void save_local_copy(TelemPostDaemon *daemon, char *body)
+{
+        int ret = 0;
+        char *tmpbuf = NULL;
+        FILE *tmpfile = NULL;
+
+        if (daemon == NULL || daemon->record_journal == NULL ||
+            daemon->record_journal->latest_record_id == NULL) {
+                return;
+        }
+
+        ret = asprintf(&tmpbuf, "%s/%s", RECORD_RETENTION_DIR,
+                       daemon->record_journal->latest_record_id);
+        if (ret == -1) {
+                telem_log(LOG_ERR, "Failed to allocate memory for record full path, aborting\n");
+                return;
+        }
+
+        tmpfile = fopen(tmpbuf, "w");
+        if (!tmpfile) {
+                telem_perror("Error opening local record copy temp file");
+                goto save_err;
+        }
+
+        // Save body
+        fprintf(tmpfile, "%s\n", body);
+        fclose(tmpfile);
+
+save_err:
+        free(tmpbuf);
+        return;
+}
+
+static void save_entry_to_journal(TelemPostDaemon *daemon, time_t t_stamp, char *headers[])
+{
+        char *classification_value = NULL;
+        char *event_id_value = NULL;
+
+        if (get_header_value(headers[TM_CLASSIFICATION], &classification_value) &&
+            get_header_value(headers[TM_EVENT_ID], &event_id_value)) {
+                if (new_journal_entry(daemon->record_journal, classification_value, t_stamp, event_id_value) != 0) {
+                        telem_log(LOG_INFO, "new_journal_entry in process_record: failed saving record entry\n");
+                }
+        }
+        free(classification_value);
+        free(event_id_value);
+}
+
+/* Check window length conf values */
+static bool windows_length_value_check(TelemPostDaemon *daemon)
+{
+        if (daemon->record_window_length == -1 ||
+            daemon->byte_window_length == -1) {
+                telem_log(LOG_ERR, "Invalid value for window length\n");
+                return false;
+        }
+
+        return true;
+}
+
+/* Rate limiting checks */
+static void rate_limit_checks(TelemPostDaemon *daemon, bool *record_check_passed, bool *byte_check_passed)
+{
+        time_t temp = time(NULL);
+        struct tm *tm_s = localtime(&temp);
+        int current_minute;
+        current_minute = tm_s->tm_min;
+        bool record_burst_enabled = burst_limit_enabled(daemon->record_burst_limit);
+        bool byte_burst_enabled =  burst_limit_enabled(daemon->byte_burst_limit);
+
+        /* Checks if entirety of rate limiting is enabled */
+        if (daemon->rate_limit_enabled) {
+                /* Checks whether record and byte bursts are enabled individually */
+                record_burst_enabled = burst_limit_enabled(daemon->record_burst_limit);
+                byte_burst_enabled = burst_limit_enabled(daemon->byte_burst_limit);
+
+                if (record_burst_enabled) {
+                        *record_check_passed = rate_limit_check(current_minute,
+                                                                daemon->record_burst_limit, daemon->record_window_length,
+                                                                daemon->record_burst_array, TM_RECORD_COUNTER);
+                }
+                if (byte_burst_enabled) {
+                        *byte_check_passed = rate_limit_check(current_minute,
+                                                              daemon->byte_burst_limit, daemon->byte_window_length,
+                                                              daemon->byte_burst_array, RECORD_SIZE_LEN);
+                }
+                /* If both record and byte burst disabled, rate limiting disabled */
+                if (!record_burst_enabled && !byte_burst_enabled) {
+                        daemon->rate_limit_enabled = false;
+                }
+        }
+
+}
+
+/* Wrapper for save local copy */
+static void apply_retention_policies(TelemPostDaemon *daemon, char *body)
+{
+        if (daemon->record_retention_enabled) {
+                save_local_copy(daemon, body);
+        }
+}
+
+/* Deliver record to backend if rate limiting policies are met otherwise
+ * spool record for future delivery */
+static bool deliver_record(TelemPostDaemon *daemon, char *headers[], char *body, char* cfg_file)
+{
+
+        bool ret = false;
+        int current_minute;
+        bool do_spool = false;
+        bool record_sent = false;
+        time_t temp = time(NULL);
+        struct tm *tm_s = localtime(&temp);
+        current_minute = tm_s->tm_min;
+        /* Checks flags */
+        bool record_check_passed = true;
+        bool byte_check_passed = true;
+        bool record_burst_enabled = burst_limit_enabled(daemon->record_burst_limit);
+        bool byte_burst_enabled =  burst_limit_enabled(daemon->byte_burst_limit);
+
+        /* Perform record and byte rate limiting checks */
+        rate_limit_checks(daemon, &record_check_passed, &byte_check_passed);
+
+        /* Sends record if rate limiting is disabled, or all checks passed */
+        if (!daemon->rate_limit_enabled || (record_check_passed && byte_check_passed)) {
+                /* Send the record as https post */
+                record_sent = post_record_ptr(headers, body, cfg_file);
+                /**
+                 * This is the only point where an error condition could be returned
+                 * if the record was not sent
+                 * */
+                ret = record_sent;
+        }
+        // Get rate-limit strategy
+        do_spool = spool_strategy_selected(daemon);
+
+        // Drop record
+        if (!record_sent && !do_spool) {
+                // Not an error condition
+                ret = true;
+        }
+        // Spool Record
+        else if (!record_sent && do_spool) {
+                start_network_bypass(daemon);
+                telem_log(LOG_INFO, "process_record: initializing direct-spool window\n");
+                // False will keep record around
+                ret = false;
+        } else {
+                /* Updates rate limiting arrays if record sent */
+                if (record_burst_enabled) {
+                        rate_limit_update(current_minute, daemon->record_window_length,
+                                          daemon->record_burst_array, TM_RECORD_COUNTER);
+                }
+                if (byte_burst_enabled) {
+                        rate_limit_update(current_minute, daemon->byte_window_length,
+                                          daemon->byte_burst_array, RECORD_SIZE_LEN);
+                }
+        }
+
+        return ret;
+}
+
+bool process_staged_record(char *filename, bool is_retry, TelemPostDaemon *daemon)
+{
+        int k;
+        bool ret = false;
+        char *headers[NUM_HEADERS];
+        char *body = NULL;
+        struct stat buf = { 0 };
+        time_t current_time = time(NULL);
+        int64_t max_spool_size = 0;
+        char *cfg_file = NULL;
+
+        for (k = 0; k < NUM_HEADERS; k++) {
+                headers[k] = NULL;
+        }
+
+        /** Load record **/
+        if ((ret = read_record(filename, headers, &body, &cfg_file)) == false) {
+                telem_log(LOG_WARNING, "unable to read record\n");
+                ret = true; // Record corrupted? true will remove record
+                goto end_processing_file;
+        }
+
+        /** Get file information  **/
+        if (stat(filename, &buf) == -1) {
+                telem_perror("Processing staged file unable to stat record in spool");
+                ret = true; // true to remove it
+                goto end_processing_file;
+        }
+
+        /** Update spool directory size **/
+        daemon->current_spool_size += (buf.st_blocks * 512);
+
+        /** Check that record is not expired **/
+        if (!S_ISREG(buf.st_mode) ||
+            (current_time - buf.st_mtime > (record_expiry_config() * 60)) ||
+            (buf.st_uid  != getuid())) {
+                ret = true; // Expired, true to remove it
+                goto end_processing_file;
+        }
+
+        /* Retries should not be recorded */
+        if (is_retry == false) {
+                /** Journal entry **/
+                save_entry_to_journal(daemon, current_time, headers);
+                /** Record retention **/
+                apply_retention_policies(daemon, body);
+        }
+
+        /** Record delivery **/
+        if (!daemon->record_server_delivery_enabled) {
+                telem_log(LOG_INFO, "record server delivery disabled\n");
+                // Not an error condition
+                ret = true;
+                goto end_processing_file;
+        }
+
+        /** Spool policies **/
+        if (inside_direct_spool_window(daemon, time(NULL))) {
+                telem_log(LOG_INFO, "process_record: delivering directly to spool\n");
+                /* Check spool max size conf */
+                max_spool_size = spool_max_size_config();
+                if (max_spool_size != -1 &&
+                    daemon->current_spool_size >= (max_spool_size * 1024)) {
+                        // Drop record
+                        telem_log(LOG_INFO, "Spool dir full, dropping record\n");
+                        ret = true;
+                } else {
+                        // Keep record, non error condition
+                        ret = false;
+                }
+                goto end_processing_file;
+        }
+
+        /** Check window_length **/
+        if (windows_length_value_check(daemon) == false) {
+                exit(EXIT_FAILURE);
+        }
+
+        /** Deliver or spool **/
+        ret = deliver_record(daemon, headers, body, cfg_file);
+
+end_processing_file:
+        /** Update spool size if record will be removed **/
+        if (ret) {
+                daemon->current_spool_size -= (buf.st_blocks * 512);
+        }
+        telem_log(LOG_DEBUG, "spool_size: %ld\n", daemon->current_spool_size);
+        free(body);
+
+        for (k = 0; k < NUM_HEADERS; k++) {
+                free(headers[k]);
+        }
+
+        if (cfg_file != NULL) {
+                free(cfg_file);
+        }
+        return ret;
+}
+
+static int directory_dot_filter(const struct dirent *entry)
+{
+        if ((strcmp(entry->d_name, ".") == 0) || (strcmp(entry->d_name, "..") == 0)) {
+                return 0;
+        } else {
+                return 1;
+        }
+}
+
+int staging_records_loop(TelemPostDaemon *daemon)
+{
+        int ret;
+        int processed;
+        int numentries;
+        struct dirent **namelist;
+
+        numentries = scandir(spool_dir_config(), &namelist, directory_dot_filter, NULL);
+        processed = 0;
+
+        if (numentries == 0) {
+                telem_log(LOG_DEBUG, "No entries in staging\n");
+                return numentries;
+        } else if (numentries < 0) {
+                telem_perror("Error while scanning staging");
+                return numentries;
+        }
+
+        for (int i = 0; i < numentries; i++) {
+                char *record_path;
+                telem_log(LOG_DEBUG, "Processing staged record: %s\n",
+                          namelist[i]->d_name);
+                ret = asprintf(&record_path, "%s/%s", spool_dir_config(), namelist[i]->d_name);
+                if (ret == -1) {
+                        telem_log(LOG_ERR, "Failed to allocate memory for staging record full path\n");
+                        exit(EXIT_FAILURE);
+                }
+                if (process_staged_record(record_path, true, daemon)) {
+                        unlink(record_path);
+                        processed++;
+                }
+                free(record_path);
+        }
+
+        for (int i = 0; i < numentries; i++) {
+                free(namelist[i]);
+        }
+        free(namelist);
+
+        return numentries - processed;
+}
+
+void run_daemon(TelemPostDaemon *daemon)
+{
+        int ret;
+        /* retry_attempt of zero indicates we don't need to retry */
+        int retry_attempt = 0;
+        int spool_process_time = spool_process_time_config();
+        bool daemon_recycling_enabled = daemon_recycling_enabled_config();
+        time_t last_spool_run_time = time(NULL);
+        time_t last_record_received = time(NULL);
+
+        assert(daemon);
+        assert(daemon->pollfds);
+        assert(daemon->pollfds[signlfd].fd);
+        assert(daemon->pollfds[watchfd].fd);
+
+        /* If we failed to send spooled records, indicate we need to retry */
+        if (daemon->bypass_http_post_ts != 0) {
+                retry_attempt = 1;
+        }
+
+        while (1) {
+                int retry_delay = spool_process_time;
+                malloc_trim(0);
+
+                /* check if we need to retry sending spooled records */
+                if (retry_attempt > 0) {
+                        retry_delay = retry_attempt * retry_attempt;
+                        daemon->bypass_http_post_ts = 0;
+                        telem_log(LOG_INFO, "Record delivery failed will retry in %d seconds",
+                                  retry_delay);
+                }
+
+                ret = poll(daemon->pollfds, NFDS, retry_delay * 1000);
+                if (ret == -1) {
+                        telem_perror("Failed to poll daemon file descriptors");
+                        break;
+                } else if (ret != 0) {
+
+                        if (daemon->pollfds[signlfd].revents != 0) {
+                                struct signalfd_siginfo fdsi;
+                                ssize_t s;
+                                s = read(daemon->sfd, &fdsi, sizeof(struct signalfd_siginfo));
+                                if (s != sizeof(struct signalfd_siginfo)) {
+                                        telem_perror("Error while reading from the signal"
+                                                     "file descriptor");
+                                        exit(EXIT_FAILURE);
+                                }
+
+                                if (fdsi.ssi_signo == SIGTERM || fdsi.ssi_signo == SIGINT) {
+                                        telem_log(LOG_INFO, "Received either a \
+                                                                     SIGINT/SIGTERM signal\n");
+                                        break;
+                                }
+                        } else if (daemon->pollfds[watchfd].revents != 0) {
+
+                                int ret = 0;
+                                ssize_t i = 0;
+                                ssize_t length = 0;
+                                char buffer[BUFFER_LEN];
+
+                                length = read(daemon->fd, buffer, BUFFER_LEN);
+                                if (length < 0) {
+                                        telem_perror("Error while reading from inotify"
+                                                     "watcher");
+                                        exit(EXIT_FAILURE);
+                                }
+
+                                while (i < length) {
+                                        struct inotify_event *event = (struct inotify_event *)&buffer[i];
+
+                                        if (event->len) {
+                                                if (event->mask & IN_CLOSE_WRITE && !(event->mask & IN_ISDIR)) {
+                                                        char *record_name = NULL;
+
+                                                        /* Retrieve foldername from watch id?  */
+                                                        ret = asprintf(&record_name, "%s/%s", spool_dir_config(), event->name);
+                                                        if (ret == -1) {
+                                                                telem_log(LOG_ERR, "Failed to allocate memory for record full path, aborting\n");
+                                                                exit(EXIT_FAILURE);
+                                                        }
+                                                        /* Process inotify event */
+                                                        if (process_staged_record(record_name, false, daemon)) {
+                                                                unlink(record_name);
+                                                        }
+                                                        free(record_name);
+                                                        last_record_received = time(NULL);
+                                                }
+                                        }
+
+                                        i += (ssize_t)EVENT_SIZE + event->len;
+                                }
+                        }
+                } else {
+                        time_t now = time(NULL);
+                        /* time to recycle the daemon has elapsed*/
+                        if (daemon_recycling_enabled &&
+                            difftime(now, last_record_received) >= TM_DAEMON_EXIT_TIME) {
+                                /* Exit */
+                                telem_log(LOG_INFO, "Telemetry post daemon exiting for recycling\n");
+                                break;
+                        }
+
+                        /* Check if this was a retry attempt */
+                        if (retry_attempt > 0) {
+                                /* Stop attempting retries if successful, increment counter if not */
+                                if (staging_records_loop(daemon) == 0) {
+                                        retry_attempt = 0;
+                                } else {
+                                        retry_attempt++;
+                                }
+                                /* Give up if counter reaches MAX_RETRY_ATTEMPTS */
+                                if (retry_attempt == MAX_RETRY_ATTEMPTS) {
+                                        telem_log(LOG_ERR, "Record deliver failed after %d attempts",
+                                                  MAX_RETRY_ATTEMPTS);
+                                        retry_attempt = 0;
+                                }
+                        }
+
+                        /* Check spool  */
+                        if (difftime(now, last_spool_run_time) >= spool_process_time) {
+                                spool_records_loop(&(daemon->current_spool_size));
+                                last_spool_run_time = time(NULL);
+                        }
+                }
+
+                /* Check journal records and prune if needed */
+                ret = prune_journal(daemon->record_journal, JOURNAL_TMPDIR);
+                if (ret != 0) {
+                        telem_log(LOG_WARNING, "Unable to prune journal\n");
+                }
+        }
+}
+
+void close_daemon(TelemPostDaemon *daemon)
+{
+
+        if (daemon->fd) {
+                if (daemon->wd) {
+                        inotify_rm_watch(daemon->fd, daemon->wd);
+                }
+                close(daemon->fd);
+        }
+
+        close_journal(daemon->record_journal);
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/telempostdaemon.h

@@ -0,0 +1,137 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define EVENT_SIZE sizeof(struct inotify_event)
+#define BUFFER_LEN 1024 * (EVENT_SIZE + 16)
+#define NFDS 2
+#define TM_RATE_LIMIT_SLOTS (1 /*h*/ * 60 /*m*/)
+#define TM_RECORD_COUNTER (1)
+#define MAX_RETRY_ATTEMPTS 8
+#define NETWORK_BYPASS_DURATION TM_DAEMON_EXIT_TIME
+
+#include <poll.h>
+#include <stdbool.h>
+#include <sys/inotify.h>
+
+#include "common.h"
+#include "journal/journal.h"
+#include "configuration.h"
+
+enum fdindex {signlfd, watchfd};
+
+typedef struct TelemPostDaemon {
+        int fd;
+        int wd;
+        int sfd;
+        char event_buffer[BUFFER_LEN];
+        struct pollfd pollfds[NFDS];
+        /* Telemetry Journal*/
+        TelemJournal *record_journal;
+        /* Time of last failed post */
+        time_t bypass_http_post_ts;
+        /* Rate limit record and byte arrays */
+        size_t record_burst_array[TM_RATE_LIMIT_SLOTS];
+        size_t byte_burst_array[TM_RATE_LIMIT_SLOTS];
+        /* Rate Limit Configurations */
+        bool rate_limit_enabled;
+        int64_t record_burst_limit;
+        int record_window_length;
+        int64_t byte_burst_limit;
+        int byte_window_length;
+        const char *rate_limit_strategy;
+        /* Spool configuration */
+        bool is_spool_valid;
+        long current_spool_size;
+        /* Record local copy and delivery  */
+        bool record_retention_enabled;
+        bool record_server_delivery_enabled;
+} TelemPostDaemon;
+
+/**
+ * Initializes daemon
+ *
+ * @param daemon a pointer to telemetry post daemon
+ */
+void initialize_post_daemon(TelemPostDaemon *daemon);
+
+/**
+ * Starts daemon
+ *
+ * @param daemon a pointer to telemetry post daemon
+ */
+void run_daemon(TelemPostDaemon *daemon);
+
+/**
+ * Cleans up inotify descriptors
+ *
+ * @param daemon a pointer to telemetry post daemon
+ */
+void close_daemon(TelemPostDaemon *daemon);
+
+/**
+ * Processed record written on disk
+ *
+ * @param filename a pointor to record on disk
+ * @param is_retry a boolean value that indicates if
+ *        the record has been previously processed.
+ * @param daemon post to telemetry post daemon
+ */
+bool process_staged_record(char *filename, bool is_retry, TelemPostDaemon *daemon);
+
+/**
+ * Scans staging directory to process files that were
+ * missed by file watcher
+ *
+ * @param daemon a pointer to telemetry post daemon
+ * @return the number of records that were removed from spool
+ */
+int staging_records_loop(TelemPostDaemon *daemon);
+
+/**
+ * Posts a record to backend
+ *
+ * @param headers a pointer to an array with keys and values
+ * @param body a pointer to the payload
+ * @param cfg_file a pointer to a non-default configuration
+ *        file to be used.
+ */
+bool post_record_http(char *headers[], char *body, char *cfg_file);
+
+/**
+ * Pointer to function to isolate backend call during
+ * unit testing.
+ *
+ * @param headers pointer to array of keys
+ * @param body a pinter to payload
+ * */
+extern bool (*post_record_ptr)(char *headers[], char *body, char *cfg_file);
+
+/** Helper functions **/
+/* rate limit check */
+bool rate_limit_check(int current_minute, int64_t burst_limit,
+                      int window_length, size_t *array, size_t incValue);
+
+/* burst limit check  */
+bool burst_limit_enabled(int64_t burst_limit);
+
+/* rate limit check  */
+void rate_limit_update(int current_minute, int window_length, size_t *array,
+                       size_t incValue);
+
+/* spool strategy check */
+bool spool_strategy_selected(TelemPostDaemon *daemon);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/util.c

@@ -16,20 +16,29 @@
 
 #define _GNU_SOURCE
 
+#define RANDOM_ID_LEN 32
+
 #include <stdio.h>
 #include <string.h>
 #include <stdbool.h>
 #include <assert.h>
 #include <stdlib.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <dirent.h>
 #include <errno.h>
 
+#include "common.h"
 #include "util.h"
+#include "log.h"
 
-bool get_header(const char *haystack, const char *needle, char **line, size_t len)
+bool get_header(const char *haystack, const char *needle, char **line)
 {
+        size_t len = strlen(needle);
+
         if (haystack && !strncmp(haystack, needle, len)) {
                 *line = strdup(haystack);
                 return true;
@@ -37,6 +46,28 @@ bool get_header(const char *haystack, const char *needle, char **line, size_t le
         return false;
 }
 
+bool get_header_value(const char *header, char **value)
+{
+        char *sep = NULL;
+
+        *value = NULL;
+
+        if (header == NULL) {
+                return false;
+        }
+
+        if ((sep = strchr(header, ':')) != NULL) {
+                sep++;
+                // skip space after colon if there's one
+                if (*sep == ' ') {
+                        sep++;
+                }
+                *value = strdup(sep);
+        }
+
+        return (bool)(*value != NULL);
+}
+
 void *reallocate(void **addr, size_t *allocated, size_t requested)
 {
         void *newaddr;
@@ -70,10 +101,7 @@ long get_directory_size(const char *dir_path)
         dir = opendir(dir_path);
         if (!dir) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Error opening spool dir: %s\n",
-                        strerror(errno));
-#endif
+                telem_perror("Error opening spool dir");
                 return ret;
         }
 
@@ -83,19 +111,15 @@ long get_directory_size(const char *dir_path)
                 }
                 ret = asprintf(&file_path, "%s/%s", dir_path, de->d_name);
                 if (ret < 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "CRIT: Cannot allocate memory\n");
-#endif
+                        telem_log(LOG_CRIT, "CRIT: Cannot allocate memory\n");
                         closedir(dir);
                         return -ENOMEM;
                 }
 
                 ret = lstat(file_path, &buf);
                 if (ret < 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "ERR: Could not stat file %s: %s\n",
+                        telem_log(LOG_ERR, "Could not stat file %s: %s\n",
                                 file_path, strerror(errno));
-#endif
                 } else {
                         total_size += (buf.st_blocks * 512);
                 }
@@ -107,4 +131,78 @@ long get_directory_size(const char *dir_path)
         return total_size;
 }
 
+/**
+ * Generates a 32 characters random id
+ *
+ * @param buffer pointer to allocate and copy data
+ *
+ */
+int get_random_id(char **buff)
+{
+        int result = -1;
+        int frandom = -1;
+        uint64_t random_id[2] = { '\0' };
+
+        frandom = open("/dev/urandom", O_RDONLY);
+        if (frandom < 0) {
+                return -1;
+        }
+
+        if (read(frandom, &random_id, sizeof(random_id)) == sizeof(random_id)) {
+                if (asprintf(buff, "%.16" PRIx64 "%.16" PRIx64, random_id[0], random_id[1]) == RANDOM_ID_LEN) {
+                        result = 0;
+                }
+        }
+
+        close(frandom);
+
+        return result;
+}
+
+/**
+ * Validate classification value. A valid classification
+ * is a string with 2 slashes, with max length of
+ * MAX_CLASS_LENGTH and strings betweeb slashes should
+ * have a max length of MAX_SUBCAT_LENGTH.
+ *
+ * @param classification A pointer to classification value
+ *        to be validated.
+ *
+ * @return 0 on sucess and 1 on failure
+ */
+int validate_classification(char *classification)
+{
+        size_t i, j;
+        int slashes = 0;
+        int x = 0;
+
+        if (classification == NULL) {
+                return 1;
+        }
+
+        j = strlen(classification);
+
+        if (j > MAX_CLASS_LENGTH) {
+                return 1;
+        }
+
+        for (i = 0, x = 0; i <= (j - 1); i++, x++) {
+                if (classification[i] == '/') {
+                        slashes++;
+                        x = 0;
+                } else {
+                        if (x > MAX_SUBCAT_LENGTH) {
+                                return 1;
+                        }
+                }
+        }
+
+        if (slashes != 2) {
+                telem_log(LOG_ERR, "Classification string should have two /s.\n");
+                return 1;
+        }
+
+        return 0;
+}
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/util.h

@@ -22,9 +22,18 @@
 void *reallocate(void **addr, size_t *allocated, size_t requested);
 
 /* Check if haystacks begins with needle and return a copy of haystack */
-bool get_header(const char *haystack, const char *needle, char **line, size_t len);
+bool get_header(const char *haystack, const char *needle, char **line);
+
+/* Get the value of a header */
+bool get_header_value(const char *header, char **value);
 
 /* Get the size of the directory */
 long get_directory_size(const char *sdir);
 
+/* Initialize buff and copy generated id */
+int get_random_id(char **buff);
+
+/* Validates classification value */
+int validate_classification(char *classification);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

tests/check_config.c

@@ -40,7 +40,7 @@ START_TEST(check_read_valid_config)
         ck_assert_str_eq(config.strValues[CONF_SPOOL_DIR], "/tmp/spool");
         ck_assert_int_eq(config.intValues[CONF_RECORD_EXPIRY], 1200);
         ck_assert_int_eq(config.intValues[CONF_SPOOL_MAX_SIZE], 1024);
-        ck_assert_int_eq(config.intValues[CONF_SPOOL_PROCESS_TIME], 900);
+        ck_assert_int_eq(config.intValues[CONF_SPOOL_PROCESS_TIME], 180);
         ck_assert_int_eq(config.intValues[CONF_RECORD_BURST_LIMIT], 100);
         ck_assert_int_eq(config.intValues[CONF_RECORD_WINDOW_LENGTH], 15);
         ck_assert_int_eq(config.intValues[CONF_BYTE_BURST_LIMIT], 1000);
@@ -55,6 +55,77 @@ START_TEST(check_read_valid_config)
 }
 END_TEST
 
+START_TEST(check_default_config)
+{
+        configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
+        int ret = set_default_config_values(&config);
+        ck_assert(ret == true);
+
+        ck_assert_str_eq(config.strValues[CONF_SERVER_ADDR], DEFAULT_SERVER_ADDR);
+        ck_assert_str_eq(config.strValues[CONF_SOCKET_PATH], DEFAULT_SOCKET_PATH);
+        ck_assert_str_eq(config.strValues[CONF_SPOOL_DIR], DEFAULT_SPOOL_DIR);
+        ck_assert_str_eq(config.strValues[CONF_RATE_LIMIT_STRATEGY], DEFAULT_RATE_LIMIT_STRATEGY);
+        ck_assert_str_eq(config.strValues[CONF_CAINFO], DEFAULT_CAINFO);
+        ck_assert_str_eq(config.strValues[CONF_TIDHEADER], DEFAULT_TIDHEADER);
+
+        ck_assert_int_eq(config.intValues[CONF_RECORD_EXPIRY], DEFAULT_RECORD_EXPIRY);
+        ck_assert_int_eq(config.intValues[CONF_SPOOL_MAX_SIZE], DEFAULT_SPOOL_MAX_SIZE);
+        ck_assert_int_eq(config.intValues[CONF_SPOOL_PROCESS_TIME], DEFAULT_SPOOL_PROCESS_TIME);
+        ck_assert_int_eq(config.intValues[CONF_RECORD_WINDOW_LENGTH], DEFAULT_RECORD_WINDOW_LENGTH);
+        ck_assert_int_eq(config.intValues[CONF_BYTE_WINDOW_LENGTH], DEFAULT_BYTE_WINDOW_LENGTH);
+        ck_assert_int_eq(config.intValues[CONF_RECORD_BURST_LIMIT], DEFAULT_RECORD_BURST_LIMIT);
+        ck_assert_int_eq(config.intValues[CONF_BYTE_BURST_LIMIT], DEFAULT_BYTE_BURST_LIMIT);
+
+        ck_assert(config.boolValues[CONF_RATE_LIMIT_ENABLED] == DEFAULT_RATE_LIMIT_ENABLED);
+        ck_assert(config.boolValues[CONF_DAEMON_RECYCLING_ENABLED] == DEFAULT_DAEMON_RECYCLING_ENABLED);
+        ck_assert(config.boolValues[CONF_RECORD_RETENTION_ENABLED] == DEFAULT_RECORD_RETENTION_ENABLED);
+        ck_assert(config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED] == DEFAULT_RECORD_SERVER_DELIVERY_ENABLED);
+}
+END_TEST
+
+START_TEST(check_layered_config)
+{
+        char *config_file = TOPSRCDIR "/src/data/example.2.conf";
+        configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
+
+        int ret = read_config_from_file(config_file, &config);
+        ck_assert(ret == true);
+
+        ck_assert_str_eq(config.strValues[CONF_SERVER_ADDR], "http://127.0.0.1");
+        ck_assert_str_eq(config.strValues[CONF_SOCKET_PATH], DEFAULT_SOCKET_PATH);
+        ck_assert_str_eq(config.strValues[CONF_SPOOL_DIR], DEFAULT_SPOOL_DIR);
+        ck_assert_str_eq(config.strValues[CONF_RATE_LIMIT_STRATEGY], DEFAULT_RATE_LIMIT_STRATEGY);
+        ck_assert_str_eq(config.strValues[CONF_CAINFO], "/tmp/cacert.crt");
+        ck_assert_str_eq(config.strValues[CONF_TIDHEADER], DEFAULT_TIDHEADER);
+
+        ck_assert_int_eq(config.intValues[CONF_RECORD_EXPIRY], DEFAULT_RECORD_EXPIRY);
+        ck_assert_int_eq(config.intValues[CONF_SPOOL_MAX_SIZE], DEFAULT_SPOOL_MAX_SIZE);
+        ck_assert_int_eq(config.intValues[CONF_SPOOL_PROCESS_TIME], DEFAULT_SPOOL_PROCESS_TIME);
+        ck_assert_int_eq(config.intValues[CONF_RECORD_WINDOW_LENGTH], DEFAULT_RECORD_WINDOW_LENGTH);
+        ck_assert_int_eq(config.intValues[CONF_BYTE_WINDOW_LENGTH], DEFAULT_BYTE_WINDOW_LENGTH);
+        ck_assert_int_eq(config.intValues[CONF_RECORD_BURST_LIMIT], DEFAULT_RECORD_BURST_LIMIT);
+        ck_assert_int_eq(config.intValues[CONF_BYTE_BURST_LIMIT], DEFAULT_BYTE_BURST_LIMIT);
+
+        ck_assert(config.boolValues[CONF_RATE_LIMIT_ENABLED] == DEFAULT_RATE_LIMIT_ENABLED);
+        ck_assert(config.boolValues[CONF_DAEMON_RECYCLING_ENABLED] == DEFAULT_DAEMON_RECYCLING_ENABLED);
+        ck_assert(config.boolValues[CONF_RECORD_RETENTION_ENABLED] == DEFAULT_RECORD_RETENTION_ENABLED);
+        ck_assert(config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED] == DEFAULT_RECORD_SERVER_DELIVERY_ENABLED);
+}
+END_TEST
+
+START_TEST(check_read_valid_config_record_retention_delivery)
+{
+        char *config_file = TOPSRCDIR "/src/data/example.1.conf";
+        configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
+
+        int ret = read_config_from_file(config_file, &config);
+        ck_assert(ret == true);
+
+        ck_assert(config.boolValues[CONF_RECORD_RETENTION_ENABLED] == true);
+        ck_assert(config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED] == false);
+}
+END_TEST
+
 START_TEST(check_config_initialised)
 {
         char *config_file = ABSTOPSRCDIR "/src/data/example.conf";
@@ -66,7 +137,7 @@ START_TEST(check_config_initialised)
         ck_assert_str_eq(spool_dir_config(), "/tmp/spool");
         ck_assert_int_eq(record_expiry_config(), 1200);
         ck_assert_int_eq(spool_max_size_config(), 1024);
-        ck_assert_int_eq(spool_process_time_config(), 900);
+        ck_assert_int_eq(spool_process_time_config(), 180);
         ck_assert_int_eq(record_burst_limit_config(), 100);
         ck_assert_int_eq(record_window_length_config(), 15);
         ck_assert_int_eq(byte_burst_limit_config(), 1000);
@@ -89,6 +160,9 @@ Suite *config_suite(void)
         TCase *t = tcase_create("config");
         tcase_add_test(t, check_read_config_for_invalid_file);
         tcase_add_test(t, check_read_valid_config);
+        tcase_add_test(t, check_default_config);
+        tcase_add_test(t, check_layered_config);
+        tcase_add_test(t, check_read_valid_config_record_retention_delivery);
         tcase_add_test(t, check_config_initialised);
 
         // add more TCases here

tests/check_journal.c

@@ -0,0 +1,261 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#include <check.h>
+#include <errno.h>
+#include <stdio.h>
+#include "common.h"
+#include "journal/journal.h"
+
+static char *journal_file = "journal.txt";
+static char *journal_print_file = "journal.print.txt";
+static struct TelemJournal *journal = NULL;
+static char *eid = "00007766547776eb7fc478eb0eb43e43";
+static int K = 20;
+
+void teardown(void)
+{
+        remove(journal_file);
+}
+
+START_TEST(check_open_journal)
+{
+        struct TelemJournal *j = open_journal(journal_file);
+        ck_assert_ptr_nonnull(j);
+        ck_assert_ptr_nonnull(j->fptr);
+        ck_assert_ptr_nonnull(j->boot_id);
+        ck_assert_int_eq(j->record_count, 0);
+        close_journal(j);
+}
+END_TEST
+
+START_TEST(check_unsuccessful_open_journal)
+{
+        char *bad_journal_file = "/sys/firmware/acpi/tables/journal";
+        struct TelemJournal *j = open_journal(bad_journal_file);
+        ck_assert_ptr_null(j);
+}
+END_TEST
+
+void new_entry_setup(void)
+{
+        if (journal) {
+                return;
+        }
+
+        journal = open_journal(journal_file);
+}
+
+START_TEST(check_new_entry)
+{
+        int ret;
+        ret = new_journal_entry(journal, "t/t/t", 1520054957,
+                                "3bc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 0);
+        close_journal(journal);
+}
+END_TEST
+
+START_TEST(check_new_entry_bad_class)
+{
+        int ret = 0;
+        /* has an invalid class */
+        ret = new_journal_entry(journal, "t/t", 1520054957,
+                                "3bc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+START_TEST(check_new_entry_bad_id)
+{
+        int ret = 0;
+        /* event_id is invalid */
+        ret = new_journal_entry(journal, "t/t/t", 1520054957,
+                                "Xbc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+START_TEST(check_new_entry_null_class)
+{
+        int ret = 0;
+        /* param is NULL */
+        ret = new_journal_entry(journal, NULL, 1520054957,
+                                "3bc17766547776eb7fc478eb0eb43e43");
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+START_TEST(check_new_entry_null_id)
+{
+        int ret = 0;
+        /* null id */
+        ret = new_journal_entry(journal, "t/t/t", 1520054957, NULL);
+        ck_assert_int_eq(ret, 1);
+}
+END_TEST
+
+void new_entry_teardown(void)
+{
+        if (journal) {
+                close_journal(journal);
+                journal = NULL;
+        }
+        remove(journal_file);
+}
+
+void insert_n_records(int n, struct TelemJournal *j)
+{
+        for (int i = 0; i < n; i++) {
+                new_journal_entry(j, "t/t/t", 1520054957 + i,
+                                  "3bc17766547776eb7fc478eb0eb43e43");
+        }
+}
+
+/*
+   failing in travis-ci
+ */
+START_TEST(check_journal_file_prune)
+{
+        int rc = 0;
+        struct TelemJournal *j = open_journal(journal_file);
+
+        insert_n_records(j->record_count_limit * 2, j);
+        ck_assert_int_gt(j->record_count, j->record_count_limit);
+        rc = prune_journal(j, "./");
+        ck_assert(rc == 0);
+        // Record count should always be below the limit + hysteresis
+        ck_assert_int_lt(j->record_count, j->record_count_limit + DEVIATION);
+        close_journal(j);
+}
+END_TEST
+
+void journal_entry_setup(void)
+{
+        int result = 0;
+
+        if (journal) {
+                close_journal(journal);
+        }
+
+        journal = open_journal(journal_print_file);
+
+        insert_n_records(K, journal);
+        /* Insert recors with specific values */
+        result = new_journal_entry(journal, "a/b/c", 1520054957, eid);
+        ck_assert(result == 0);
+        result = new_journal_entry(journal, "a/b/d", 1520054957, eid);
+        ck_assert(result == 0);
+
+}
+
+START_TEST(check_journal_print)
+{
+        int count = 0;
+        count = print_journal(journal, NULL, NULL, NULL, NULL, 0);
+        ck_assert_int_eq(count, K + 2);
+}
+END_TEST
+
+START_TEST(check_journal_filter_by_class)
+{
+        int result = print_journal(journal, "a/b/c", NULL, NULL, NULL, 0);
+        ck_assert_int_eq(result, 1);
+}
+END_TEST
+
+START_TEST(check_journal_filter_by_class_prefix)
+{
+        int result = print_journal(journal, "a/b/*", NULL, NULL, NULL, 0);
+        ck_assert_int_eq(result, 2);
+}
+END_TEST
+
+START_TEST(check_journal_filter_by_event_id)
+{
+        int result = print_journal(journal, NULL, NULL, eid, NULL, 0);
+        ck_assert(result == 2);
+}
+END_TEST
+
+void journal_entry_teardown(void)
+{
+        close_journal(journal);
+        remove(journal_print_file);
+}
+
+Suite *config_suite(void)
+{
+        // A suite is comprised of test cases, defined below
+        Suite *s = suite_create("journal_feature");
+
+        // Individual unit tests are added to "test cases"
+        TCase *t = tcase_create("journal");
+        tcase_add_unchecked_fixture(t, NULL, teardown);
+        tcase_add_test(t, check_open_journal);
+        tcase_add_test(t, check_unsuccessful_open_journal);
+        suite_add_tcase(s, t);
+
+        t = tcase_create("new entry");
+        tcase_add_unchecked_fixture(t, new_entry_setup, new_entry_teardown);
+        tcase_add_test(t, check_new_entry);
+        tcase_add_test(t, check_new_entry_bad_class);
+        tcase_add_test(t, check_new_entry_bad_id);
+        tcase_add_test(t, check_new_entry_null_class);
+        tcase_add_test(t, check_new_entry_null_id);
+        suite_add_tcase(s, t);
+
+        t = tcase_create("prunning journal");
+        tcase_add_unchecked_fixture(t, NULL, teardown);
+        tcase_add_test(t, check_journal_file_prune);
+        suite_add_tcase(s, t);
+
+        t = tcase_create("print journal");
+        tcase_add_unchecked_fixture(t, journal_entry_setup,
+                                    journal_entry_teardown);
+        tcase_add_test(t, check_journal_print);
+        tcase_add_test(t, check_journal_filter_by_class);
+        tcase_add_test(t, check_journal_filter_by_class_prefix);
+        tcase_add_test(t, check_journal_filter_by_event_id);
+        suite_add_tcase(s, t);
+
+        return s;
+}
+
+int main(void)
+{
+        Suite *s;
+        SRunner *sr;
+
+        s = config_suite();
+        sr = srunner_create(s);
+
+        // Use the TAP driver for now, so that each
+        // unit test will PASS/FAIL in the log output.
+        srunner_set_log(sr, NULL);
+        srunner_set_tap(sr, "-");
+
+        srunner_run_all(sr, CK_SILENT);
+        // failed = srunner_ntests_failed(sr);
+        srunner_free(sr);
+
+        // if you want the TAP driver to report a hard error based
+        // on certain conditions (e.g. number of failed tests, etc.),
+        // return non-zero here instead.
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

tests/check_libtelemetry.c

@@ -24,6 +24,7 @@
 #include "telemetry.h"
 
 static struct telem_ref *ref = NULL;
+static char *original_event_id = NULL;
 
 void create_setup(void)
 {
@@ -40,7 +41,7 @@ void create_setup(void)
          * with the libcheck API...
          */
         ck_assert_msg(ret != -ECONNREFUSED,
-                      "Opt-out enabled. Opt in to run these tests");
+                      "First time opt-in required to run test");
 }
 
 START_TEST(record_create_non_null)
@@ -97,12 +98,22 @@ void create_teardown(void)
         }
 }
 
+START_TEST(is_opt_in)
+{
+        int ret;
+        ret = tm_is_opted_in();
+        /* Smoke testing function */
+        ck_assert_msg(ret == 0 || ret == 1,
+                      "Something wrong with opt-in check");
+}
+END_TEST
+
 START_TEST(record_create_invalid_class1)
 {
         int ret;
         ret = tm_create_record(&ref, 1, "t/t", 2000);
         ck_assert_msg(ret != -ECONNREFUSED,
-                      "Opt-out enabled. Opt in to run this test");
+                      "First time opt-in required to run test");
         ck_assert(ret == -EINVAL);
 }
 END_TEST
@@ -112,7 +123,7 @@ START_TEST(record_create_invalid_class2)
         int ret;
         ret = tm_create_record(&ref, 1, "t/t/t/t", 2000);
         ck_assert_msg(ret != -ECONNREFUSED,
-                      "Opt-out enabled. Opt in to run this test");
+                      "First time opt-in required to run test");
         ck_assert(ret == -EINVAL);
 }
 END_TEST
@@ -125,7 +136,7 @@ START_TEST(record_create_severity_underflow)
         // Severity of 0 is too low; raise it to 1, the minimum
         ret = tm_create_record(&ref, 0, "a/a/a", 2000);
         ck_assert_msg(ret != -ECONNREFUSED,
-                      "Opt-out enabled. Opt in to run this test");
+                      "First time opt-in required to run test");
 
         if (asprintf(&result, "%s: %u\n", TM_SEVERITY_STR, 1) < 0) {
                 return;
@@ -145,7 +156,7 @@ START_TEST(record_create_severity_overflow)
         // Severity of 5 is too high; lower it to 4, the maximum
         ret = tm_create_record(&ref, 5, "b/b/b", 2000);
         ck_assert_msg(ret != -ECONNREFUSED,
-                      "Opt-out enabled. Opt in to run this test");
+                      "First time opt-in required to run test");
 
         if (asprintf(&result, "%s: %u\n", TM_SEVERITY_STR, 4) < 0) {
                 return;
@@ -157,6 +168,87 @@ START_TEST(record_create_severity_overflow)
 }
 END_TEST
 
+void event_id_setup(void)
+{
+        int ret;
+
+        if (ref) {
+                return;
+        }
+
+        ret = tm_create_record(&ref, 1, "t/t/t", 2000);
+        original_event_id = strdup(ref->record->headers[TM_EVENT_ID]);
+        if (!original_event_id) {
+                return;
+        }
+        ck_assert_msg(ret != -ECONNREFUSED,
+                      "First time opt-in required to run test");
+}
+
+START_TEST(record_set_event_id)
+{
+        int ret;
+        char *event_id = "aaaaaa00000033333344444466666622";
+        char *result;
+        if (asprintf(&result, "%s: %s\n", TM_EVENT_ID_STR, event_id) < 0) {
+                return;
+        }
+        ck_assert_str_ne(ref->record->headers[TM_EVENT_ID], result);
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, 0);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], result);
+        free(result);
+}
+END_TEST
+
+START_TEST(record_set_event_id_invalid_chars)
+{
+        int ret;
+        char *event_id = "aaaaaa000000333333444444666666ZZ";
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+START_TEST(record_set_event_id_null)
+{
+        int ret;
+        char *event_id = NULL;
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+START_TEST(record_set_event_id_short)
+{
+        int ret;
+        char *event_id = "aaaa";
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+START_TEST(record_set_event_id_long)
+{
+        int ret;
+        char *event_id = "0000000000000000000000000000000000000000000";
+        ret = tm_set_event_id(ref, event_id);
+        ck_assert_int_eq(ret, -1);
+        ck_assert_str_eq(ref->record->headers[TM_EVENT_ID], original_event_id);
+}
+END_TEST
+
+void event_id_teardown(void)
+{
+        if (ref) {
+                free(ref);
+        }
+        free(original_event_id);
+}
+
 Suite *lib_suite(void)
 {
         Suite *s = suite_create("libtelemetry");
@@ -170,6 +262,10 @@ Suite *lib_suite(void)
         tcase_add_test(t, record_create_version);
         suite_add_tcase(s, t);
 
+        t = tcase_create("Opt-in");
+        tcase_add_test(t, is_opt_in);
+        suite_add_tcase(s, t);
+
         t = tcase_create("invalid classification");
         tcase_add_test(t, record_create_invalid_class1);
         tcase_add_test(t, record_create_invalid_class2);
@@ -180,6 +276,15 @@ Suite *lib_suite(void)
         tcase_add_test(t, record_create_severity_overflow);
         suite_add_tcase(s, t);
 
+        t = tcase_create("event id");
+        tcase_add_unchecked_fixture(t, event_id_setup, event_id_teardown);
+        tcase_add_test(t, record_set_event_id);
+        tcase_add_test(t, record_set_event_id_invalid_chars);
+        tcase_add_test(t, record_set_event_id_null);
+        tcase_add_test(t, record_set_event_id_short);
+        tcase_add_test(t, record_set_event_id_long);
+        suite_add_tcase(s, t);
+
         return s;
 }
 

tests/check_ncb64.c

@@ -0,0 +1,192 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2017 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define OUT_MAX_LEN 100
+
+#include <stdio.h>
+#include <check.h>
+#include "nica/b64enc.h"
+
+START_TEST(check_nc_b64_no_overflow)
+{
+        size_t n = 2;
+        char out[OUT_MAX_LEN];
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foobar";
+
+        ck_assert_msg(!nc_b64enc_filename(filename, &out[0], n), "Should quit since buffer smaller than content");
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_empty)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "\0";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/empty";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_fo)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm8=";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/fo";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_foob)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm9vYg==";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foob";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_foobar)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm9vYmFyZm9vYmFy";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foobar";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], n), "Error opening file");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_file_long_text)
+{
+        size_t out_len = 8000;
+        char out[out_len];
+        const char *expected_out = \
+                "TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIG5vbnVtbXkgdml0YWUsIGluIHZpdmFtdXMgc3Vz\n"
+                "cGVuZGlzc2UgYWMuIFNlbXBlciBzZWQgcGhhcmV0cmEgc2NlbGVyaXNxdWUuIEVnZXQgZWxlaWZl\n"
+                "bmQgYW1ldCB2ZWwgbnVuYyB2b2x1dHBhdCBjdXJzdXMsIGF1Y3RvciBwbGF0ZWEgcHJldGl1bSwg\n"
+                "bWF0dGlzIHJpc3VzIGZhY2lsaXNpcyBmYXVjaWJ1cywgc3VzcGVuZGlzc2UgZGlhbSwgbGVjdHVz\n"
+                "IG1ldHVzIG51bGxhLiBOdW5jIGp1c3RvIGZhY2lsaXNpIG5hbSBmZWxpcyB2ZWwgbGFvcmVldCwg\n"
+                "bmliaCBsZW8gbWFzc2Egc3VzcGVuZGlzc2UgYWNjdW1zYW4gY29udmFsbGlzLCBzZWQgcXVpcyBw\n"
+                "ZWxsZW50ZXNxdWUgZWdlc3RhcywgbG9yZW0gYW50ZSBtb3JiaSBtYXR0aXMgdml0YWUuIFByYWVz\n"
+                "ZW50IGluIHJ1dHJ1bSBlZ2V0IGFjLCB2aXZhbXVzIHBlZGUgc3VzcGVuZGlzc2UgbGVjdHVzIG51\n"
+                "bGxhbSwgZXQgcGVkZSBldSBvZGlvIHBlZGUsIGV0aWFtIGxlbyBlZ2VzdGFzIGluLCBub251bW15\n"
+                "IHNlbXBlci4gQSBhY2N1bXNhbiBkdWksIHZlc3RpYnVsdW0gcmlkaWN1bHVzIGV0IGluLCBqdXN0\n"
+                "byBldCBwbGFjZXJhdCBkdWlzIHV0IHZpdmFtdXMgbGliZXJvLiBWZXN0aWJ1bHVtIGVnZXQgdXQg\n"
+                "bW9sbGlzLCBudW5jIGEgc29sbGljaXR1ZGluLCBmcmluZ2lsbGEgZXJvcyBwb3N1ZXJlIG1vbGxp\n"
+                "cyBhYyBuYXRvcXVlIHBlZGUsIHJpc3VzIG51bGxhIHJ1dHJ1bSB0dXJwaXMgYSB2aXRhZSBpZC4g\n"
+                "SW5jZXB0b3MgdGluY2lkdW50IHF1aWRlbSBhcmN1IHR1cnBpcyBudW5jIHNvbGxpY2l0dWRpbi4g\n"
+                "UGhhc2VsbHVzIG1ldHVzIGVyYXQgcGxhY2VyYXQsIGluIGVyYXQgbG9yZW0gbWF1cmlzIG51bGxh\n"
+                "LCBhdCBsaWJlcm8gZXJhdCBzdXNwZW5kaXNzZSBzYXBpZW4gbW9udGVzIHJob25jdXMsIG1hZ25h\n"
+                "IHF1YW0uIEp1c3RvIHByYWVzZW50IGxhY3VzIG1hc3NhLCByZXByZWhlbmRlcml0IG51bmMsIHJp\n"
+                "c3VzIGRpZ25pc3NpbSBmZWxpcyBub24sIGRpZ25pc3NpbSBvZGlvIHRlbGx1cyBhbGlxdWFtLCBl\n"
+                "bmltIHNvZGFsZXMgdGVtcG9yIHF1aXNxdWUgc29kYWxlcyBtYWduaXMgcG9ydHRpdG9yLiBMaWd1\n"
+                "bGEgc2VkIHJpc3VzLCBldSBlc3QgcXVpc3F1ZSwgY2xhc3Mgbm9uIGF1dGUgZXUsIHBsYXRlYSBz\n"
+                "Y2VsZXJpc3F1ZSBpZCBpbnRlZ2VyIGRpZ25pc3NpbW9zIGludGVnZXIgc3VzY2lwaXQuClZlaGlj\n"
+                "dWxhIGVpdXMgbmVjLCBjb252YWxsaXMgbWFlY2VuYXMgbGVjdHVzIHB1cnVzIHF1aXNxdWUgYWxp\n"
+                "cXVhbSwgZXQgZXQsIGxhY2luaWEgZGlzIGRpZ25pc3NpbSBsb3JlbS4gVm9sdXB0YXRlbSB1dCwg\n"
+                "dml2ZXJyYSBhZW5lYW4gcGhhc2VsbHVzIGZlbGlzLCBmZXJtZW50dW0gbGlndWxhIGVnZXQgdWxs\n"
+                "YW1jb3JwZXIgYW1ldCwgbWF1cmlzIGxlbyBsdWN0dXMuIEV0IGV0aWFtIHNlbXBlciBwaGFyZXRy\n"
+                "YSBuaWJoIG1pIG1hdXJpcy4gSGVuZHJlcml0IHNlZCBhbGlxdWFtLCBlZ2V0IGlkIHRlbGx1cyBt\n"
+                "YWduYSwgc2VkIGxhb3JlZXQgdWx0cmljZXMgZW5pbSwgbW9sbGlzIHN1c3BlbmRpc3NlIGluLiBF\n"
+                "dSBudWxsYW0gbG9yZW0gc2l0IGV0aWFtLCBvcmNpIGluIGxpYmVyby4gTnVuYyBpcHN1bSBtYXVy\n"
+                "aXMgZXQgc2VtIGhhYy4gRGljdHVtIGZhdWNpYnVzIGRpcyB2aXRhZSBpbiB2b2x1dHBhdCBtb3Ji\n"
+                "aSwgYSBkaWN0dW0sIHF1aXNxdWUgbWF1cmlzLCBhdWN0b3IgbmVjIHZlbCBwZWxsZW50ZXNxdWUg\n"
+                "dWxsYW1jb3JwZXIgZXRpYW0uIEN1bSBsaWJlcm8gd2lzaSBhY2N1bXNhbiBhbGlxdWFtIGNvbnNl\n"
+                "Y3RldHVlciB0ZWxsdXMsIHBoYXNlbGx1cyBjdXJhZSwgYXQgaWxsdW0gYW50ZSBwcmV0aXVtIG5p\n"
+                "YmggbW9yYmkuIFNlZCBibGFuZGl0IHB1bHZpbmFyIHB1bHZpbmFyLiBEaWN0dW0gZXN0IG51bmMg\n"
+                "ZWxlaWZlbmQsIHZlbGl0IHZlbGl0IHRlbXBvciBhY2N1bXNhbiBsb2JvcnRpcyBsYW9yZWV0IGNv\n"
+                "bmd1ZS4gU2l0IGFtZXQsIHNlZCBuaWJoIHBvcnJvIG5lcXVlIGF1Y3RvciBoeW1lbmFlb3MgcG9z\n"
+                "dWVyZSwgb2RpbyBudWxsYSBibGFuZGl0IGNvbmd1ZSBlbGl0LiBJZCBmYXVjaWJ1cyBldCB0ZW1w\n"
+                "dXMgbWFsZXN1YWRhIHBsYXRlYS4=";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/long_text";
+
+        ck_assert_msg(nc_b64enc_filename(filename, &out[0], out_len), "Error opening file");
+        ck_assert_str_eq(out, expected_out);
+}
+END_TEST
+
+START_TEST(check_nc_b64_enc_n_filehandler_foobar)
+{
+        size_t n = OUT_MAX_LEN;
+        char out[OUT_MAX_LEN];
+        char *result = "Zm9vYmFyZm9vYmFy";
+        const char *filename = TOPSRCDIR "/tests/nc_b64enc_test_files/foobar";
+        FILE *fh = NULL;
+
+        fh = fopen(filename, "rb");
+
+        if (fh == NULL) {
+                ck_abort_msg("Unable to open foobar file");
+        }
+
+        ck_assert_msg(nc_b64enc_file(fh, &out[0], n), "Result does not match");
+        ck_assert_str_eq(out, result);
+}
+END_TEST
+
+Suite *config_suite(void)
+{
+        // A suite is comprised of test cases, defined below
+        Suite *s = suite_create("nc_b64enc");
+
+        // Individual unit tests are added to "test cases"
+        TCase *t = tcase_create("nc_b64enc");
+        tcase_add_test(t, check_nc_b64_no_overflow);
+        tcase_add_test(t, check_nc_b64_enc_n_file_empty);
+        tcase_add_test(t, check_nc_b64_enc_n_file_fo);
+        tcase_add_test(t, check_nc_b64_enc_n_file_foob);
+        tcase_add_test(t, check_nc_b64_enc_n_file_foobar);
+        tcase_add_test(t, check_nc_b64_enc_n_file_long_text);
+        tcase_add_test(t, check_nc_b64_enc_n_filehandler_foobar);
+
+        suite_add_tcase(s, t);
+
+        return s;
+}
+
+int main(void)
+{
+        Suite *s;
+        SRunner *sr;
+
+        s = config_suite();
+        sr = srunner_create(s);
+
+        // Use the TAP driver for now, so that each
+        // unit test will PASS/FAIL in the log output.
+        srunner_set_log(sr, NULL);
+        srunner_set_tap(sr, "-");
+
+        srunner_run_all(sr, CK_SILENT);
+        // failed = srunner_ntests_failed(sr);
+        srunner_free(sr);
+
+        // if you want the TAP driver to report a hard error based
+        // on certain conditions (e.g. number of failed tests, etc.),
+        // return non-zero here instead.
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

tests/check_postd.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -22,60 +22,30 @@
 #include <unistd.h>
 
 #include "configuration.h"
-#include "telemdaemon.h"
+#include "telempostdaemon.h"
 #include "common.h"
 
-TelemDaemon tdaemon;
-static bool post_was_called = false;
+TelemPostDaemon tdaemon;
 
-char *get_serialized_record(char *headers, char *post_body, size_t *record_size)
+bool dummy_post(char *headers[], char *body, char *cfg_file)
 {
-        size_t totalsize, headersize, payloadsize, offset;
-        char *data;
-
-        offset = 0;
-        headersize = strlen(headers);
-        payloadsize = strlen(post_body);
-        totalsize = headersize + payloadsize;
-
-        *record_size = 2 * sizeof(uint32_t) + totalsize + 1;
-        data = malloc(*record_size);
-        memset(data, 0, *record_size);
-        memcpy(data, &totalsize, sizeof(uint32_t));
-        offset += sizeof(uint32_t);
-
-        memcpy(data + offset, &headersize, sizeof(uint32_t));
-        offset += sizeof(uint32_t);
-
-        memcpy(data + offset, headers, headersize);
-        offset += headersize;
-
-        memcpy(data + offset, post_body, payloadsize + 1);
-        return data;
+        return true;
 }
 
-bool dummy_post(char *headers[], char *body, bool spool)
-{
-        post_was_called = true;
-        return post_was_called;
-}
-
-bool (*post_record_ptr)(char *[], char *, bool) = dummy_post;
+bool (*post_record_ptr)(char *headers[], char *body, char *cfg_file) = dummy_post;
 
 void setup(void)
 {
         char *config_file = ABSTOPSRCDIR "/src/data/example.conf";
         set_config_file(config_file);
 
-        initialize_daemon(&tdaemon);
+        initialize_post_daemon(&tdaemon);
 }
 
 START_TEST(check_daemon_is_initialized)
 {
         setup();
 
-        ck_assert(tdaemon.nfds == 0);
-        ck_assert(tdaemon.pollfds == NULL);
         ck_assert(tdaemon.rate_limit_enabled == true);
         ck_assert(tdaemon.record_burst_limit == 100);
         ck_assert(tdaemon.record_window_length == 15);
@@ -85,107 +55,16 @@ START_TEST(check_daemon_is_initialized)
 }
 END_TEST
 
-int get_poll_fd(TelemDaemon *tdaemon, nfds_t i)
-{
-        if (i >= 0 && i < tdaemon->nfds) {
-                return tdaemon->pollfds[i].fd;
-        }
-        return -1;
-}
-
-START_TEST(check_add_del_poll_fd)
-{
-        setup();
-
-        int fd = 1;
-        int short events = 1;
-        add_pollfd(&tdaemon, fd, events);
-        add_pollfd(&tdaemon, ++fd, events);
-        add_pollfd(&tdaemon, ++fd, events);
-        ck_assert_msg(tdaemon.nfds == 3, "Failed to add pollfd");
-
-        fd = get_poll_fd(&tdaemon, 0);
-        ck_assert(fd == 1);
-        fd = get_poll_fd(&tdaemon, 1);
-        ck_assert(fd == 2);
-        fd = get_poll_fd(&tdaemon, 2);
-        ck_assert(fd == 3);
-
-        del_pollfd(&tdaemon, 1);
-        ck_assert_msg(tdaemon.nfds == 2, "Failed to delete pollfd");
-        fd = get_poll_fd(&tdaemon, 1);
-        ck_assert(fd == 3);
-
-        del_pollfd(&tdaemon, 1);
-        ck_assert_msg(tdaemon.nfds == 1, "Failed to delete pollfd");
-        fd = get_poll_fd(&tdaemon, 1);
-        ck_assert(fd == -1);
-
-        del_pollfd(&tdaemon, 0);
-        ck_assert_msg(tdaemon.nfds == 0, "Failed to delete pollfd");
-        fd = get_poll_fd(&tdaemon, 0);
-        ck_assert(fd == -1);
-}
-END_TEST
-
-START_TEST(check_add_remove_client)
-{
-        setup();
-
-        client *cl, *cl1, *cl2, *cl3;
-        cl1 = add_client(&(tdaemon.client_head), 1);
-        cl2 = add_client(&(tdaemon.client_head), 2);
-        cl3 = add_client(&(tdaemon.client_head), 3);
-
-        int i = 3;
-        LIST_FOREACH(cl, &(tdaemon.client_head), client_ptrs) {
-                ck_assert_msg((cl->fd == i), "Actual value is %d\n", cl->fd);
-                i--;
-        }
-
-        remove_client(&(tdaemon.client_head), cl1);
-        remove_client(&(tdaemon.client_head), cl2);
-        remove_client(&(tdaemon.client_head), cl3);
-        //ck_assert(tdaemon.client_head.lh_first == NULL);
-        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove clients\n");
-}
-END_TEST
-
-void set_up_socket_pair(int *client, int *server)
-{
-        int sv[2];
-        int ret;
-
-        ret = socketpair(AF_UNIX, SOCK_STREAM, 0, sv);
-        ck_assert_msg(ret == 0, "Failed to create socket pair\n");
-        ck_assert_msg((fcntl(sv[0], F_SETFL, O_NONBLOCK) == 0),
-                      "Failed to set socket to non-blocking\n");
-        ck_assert_msg((fcntl(sv[1], F_SETFL, O_NONBLOCK) == 0),
-                      "Failed to set socket to non-blocking\n");
-        *client = sv[0];
-        *server = sv[1];
-}
-
 START_TEST(check_handle_client_with_no_data)
 {
         setup();
 
-        client *cl;
+        bool success;
+        char *filename = ABSTOPSRCDIR "/tests/telempostd/empty_message";
 
-        int server_fd;
-        int client_fd;
-        bool processed;
-
-        set_up_socket_pair(&client_fd, &server_fd);
-        cl = add_client(&(tdaemon.client_head), client_fd);
-        ck_assert_msg(cl != NULL, "failed to malloc client");
-        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
-
-        processed = handle_client(&tdaemon, 0, cl);
-        ck_assert(processed == false);
-        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
-        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
-        close(server_fd);
+        success = process_staged_record(filename, false, &tdaemon);
+        // Return true to remove corrupted record
+        ck_assert(success == true);
 }
 END_TEST
 
@@ -193,92 +72,12 @@ START_TEST(check_handle_client_with_incorrect_data)
 {
         setup();
 
-        client *cl;
-        char *buf = "test";
-        int server_fd;
-        int client_fd;
-        bool processed;
+        bool success;
+        char *filename = ABSTOPSRCDIR "/tests/telempostd/incorrect_message";
 
-        set_up_socket_pair(&client_fd, &server_fd);
-        cl = add_client(&(tdaemon.client_head), client_fd);
-        ck_assert_msg(cl != NULL, "failed to malloc client");
-        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
-
-        ssize_t ret = write(server_fd, buf, 2);
-        ck_assert(ret == 2);
-
-        processed = handle_client(&tdaemon, 0, cl);
-        ck_assert(processed == false);
-        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
-        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
-        close(server_fd);
-}
-END_TEST
-
-START_TEST(check_handle_client_with_incorrect_size)
-{
-        setup();
-
-        client *cl;
-        int server_fd;
-        int client_fd;
-        bool processed;
-        char *data = "test";
-        char buf[4096];
-
-        set_up_socket_pair(&client_fd, &server_fd);
-        cl = add_client(&(tdaemon.client_head), client_fd);
-        ck_assert_msg(cl != NULL, "failed to malloc client");
-        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
-
-        int size = 10;
-        memset(buf, 0, 4096);
-        memcpy(buf, &size, RECORD_SIZE_LEN);
-        memcpy(buf + RECORD_SIZE_LEN, data, sizeof(uint32_t));
-        ssize_t ret = write(server_fd, buf, 2);
-        ck_assert(ret == 2);
-
-        processed = handle_client(&tdaemon, 0, cl);
-        ck_assert(processed == false);
-        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
-        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
-        close(server_fd);
-}
-END_TEST
-
-START_TEST(check_handle_client_with_correct_size)
-{
-        setup();
-
-        client *cl;
-        int server_fd;
-        int client_fd;
-        bool processed;
-        char *data = "testing hello world message for handle client check";
-        char buf[256];
-
-        set_up_socket_pair(&client_fd, &server_fd);
-        cl = add_client(&(tdaemon.client_head), client_fd);
-        ck_assert_msg(cl != NULL, "failed to malloc client");
-        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
-
-        size_t size = strlen(data);
-        memset(buf, 0, 256);
-        memcpy(buf, &size, RECORD_SIZE_LEN);
-        memcpy(buf + RECORD_SIZE_LEN, &size, sizeof(uint32_t));
-        memcpy(buf + 2 * sizeof(uint32_t), data, strlen(data) + 1);
-        ck_assert(strcmp(data, buf + 2 * sizeof(int32_t)) == 0);
-
-        post_was_called = false;
-        ssize_t ret = write(server_fd, buf, 2 * sizeof(uint32_t) + size + 1);
-        ck_assert(ret != -1);
-        processed = handle_client(&tdaemon, 0, cl);
-        ck_assert(processed == true);
-        ck_assert(post_was_called == false);
-
-        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
-        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
-        close(server_fd);
+        success = process_staged_record(filename, false, &tdaemon);
+        // Return true to remove corrupted record
+        ck_assert(success == true);
 }
 END_TEST
 
@@ -286,35 +85,11 @@ START_TEST(check_process_record_with_correct_size_and_data)
 {
         setup();
 
-        client *cl;
-        int server_fd, client_fd;
-        bool processed;
-        char *record;
-        size_t record_size;
-        char *headers = "record_format_version: 1\nclassification: crash/kernel/bug\nseverity: 0\n"
-                        "machine_id: 1234\ncreation_timestamp: 1418672344\narch:x86_64\n"
-                        "host_type: macbookpro\nbuild: 200\nkernel_version: 3.15\n"
-                        "payload_format_version: 1\n"
-                        "system_name: clear-linux-os\n";
-        char *post_body = "test message";
+        bool success;
+        char *filename = ABSTOPSRCDIR "/tests/telempostd/correct_message";
 
-        set_up_socket_pair(&client_fd, &server_fd);
-        cl = add_client(&(tdaemon.client_head), client_fd);
-        ck_assert_msg(cl != NULL, "failed to malloc client");
-        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
-
-        post_was_called = false;
-        record = get_serialized_record(headers, post_body, &record_size);
-        ssize_t ret = write(server_fd, record, record_size);
-        ck_assert(ret == record_size);
-
-        processed = handle_client(&tdaemon, 0, cl);
-        ck_assert(processed == true);
-        ck_assert(post_was_called == true);
-        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with correct data\n");
-        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with correct data\n");
-        close(server_fd);
-        free(record);
+        success = process_staged_record(filename, false, &tdaemon);
+        ck_assert(success == true);
 }
 END_TEST
 
@@ -322,33 +97,12 @@ START_TEST(check_process_record_with_incorrect_headers)
 {
         setup();
 
-        client *cl;
-        int server_fd, client_fd;
-        bool processed;
-        char *record;
-        size_t record_size;
-        char *headers = "record_format_version: 1\nclassificatioooon: crash/kernel/bug\nseverity: 0\n"
-                        "machine_id: 1234\ncreation_timestamp: 1418672344\narch:x86_64\n"
-                        "host_type: macbookpro\nbuild: 200\nkernel_version: 3.15\n";
-        char *post_body = "test message";
+        bool success;
+        char *filename = ABSTOPSRCDIR "/tests/telempostd/incorrect_headers";
 
-        set_up_socket_pair(&client_fd, &server_fd);
-        cl = add_client(&(tdaemon.client_head), client_fd);
-        ck_assert_msg(cl != NULL, "failed to malloc client");
-        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
-
-        post_was_called = false;
-        record = get_serialized_record(headers, post_body, &record_size);
-        ssize_t ret = write(server_fd, record, record_size);
-        ck_assert(ret == record_size);
-
-        processed = handle_client(&tdaemon, 0, cl);
-        ck_assert(processed == true);
-        ck_assert(post_was_called == false);
-        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with incorrect headers\n");
-        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with incorrect headers\n");
-        close(server_fd);
-        free(record);
+        success = process_staged_record(filename, false, &tdaemon);
+        // Return true to remove corrupted record
+        ck_assert(success == true);
 }
 END_TEST
 
@@ -638,17 +392,13 @@ END_TEST
 Suite *config_suite(void)
 {
         // A suite is comprised of test cases, defined below
-        Suite *s = suite_create("daemon");
+        Suite *s = suite_create("postd");
 
         // Individual unit tests are added to "test cases"
-        TCase *t = tcase_create("daemon");
-        tcase_add_test(t, check_add_del_poll_fd);
+        TCase *t = tcase_create("postd");
         tcase_add_test(t, check_daemon_is_initialized);
-        tcase_add_test(t, check_add_remove_client);
         tcase_add_test(t, check_handle_client_with_no_data);
         tcase_add_test(t, check_handle_client_with_incorrect_data);
-        tcase_add_test(t, check_handle_client_with_incorrect_size);
-        tcase_add_test(t, check_handle_client_with_correct_size);
         tcase_add_test(t, check_process_record_with_correct_size_and_data);
         tcase_add_test(t, check_process_record_with_incorrect_headers);
         tcase_add_test(t, check_rate_limit_enabled_functions);

tests/check_probd.c

@@ -0,0 +1,383 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2015 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#include <check.h>
+#include <sys/socket.h>
+#include <sys/fcntl.h>
+#include <stdlib.h>
+#include <sys/queue.h>
+#include <unistd.h>
+
+#include "configuration.h"
+#include "telemdaemon.h"
+#include "common.h"
+
+TelemDaemon tdaemon;
+
+char *get_serialized_record(char *headers, char *post_body, size_t *record_size)
+{
+        size_t totalsize, headersize, payloadsize, offset;
+        char *data;
+
+        offset = 0;
+        headersize = strlen(headers);
+        payloadsize = strlen(post_body);
+        totalsize = headersize + payloadsize;
+
+        *record_size = 2 * sizeof(uint32_t) + totalsize + 1;
+        data = malloc(*record_size);
+        memset(data, 0, *record_size);
+        memcpy(data, &totalsize, sizeof(uint32_t));
+        offset += sizeof(uint32_t);
+
+        memcpy(data + offset, &headersize, sizeof(uint32_t));
+        offset += sizeof(uint32_t);
+
+        memcpy(data + offset, headers, headersize);
+        offset += headersize;
+
+        memcpy(data + offset, post_body, payloadsize + 1);
+        return data;
+}
+
+void setup(void)
+{
+        char *config_file = ABSTOPSRCDIR "/src/data/example.conf";
+        set_config_file(config_file);
+
+        initialize_probe_daemon(&tdaemon);
+}
+
+START_TEST(check_daemon_is_initialized)
+{
+        setup();
+
+        ck_assert(tdaemon.nfds == 0);
+        ck_assert(tdaemon.pollfds == NULL);
+}
+END_TEST
+
+int get_poll_fd(TelemDaemon *tdaemon, nfds_t i)
+{
+        if (i >= 0 && i < tdaemon->nfds) {
+                return tdaemon->pollfds[i].fd;
+        }
+        return -1;
+}
+
+START_TEST(check_add_del_poll_fd)
+{
+        setup();
+
+        int fd = 1;
+        int short events = 1;
+        add_pollfd(&tdaemon, fd, events);
+        add_pollfd(&tdaemon, ++fd, events);
+        add_pollfd(&tdaemon, ++fd, events);
+        ck_assert_msg(tdaemon.nfds == 3, "Failed to add pollfd");
+
+        fd = get_poll_fd(&tdaemon, 0);
+        ck_assert(fd == 1);
+        fd = get_poll_fd(&tdaemon, 1);
+        ck_assert(fd == 2);
+        fd = get_poll_fd(&tdaemon, 2);
+        ck_assert(fd == 3);
+
+        del_pollfd(&tdaemon, 1);
+        ck_assert_msg(tdaemon.nfds == 2, "Failed to delete pollfd");
+        fd = get_poll_fd(&tdaemon, 1);
+        ck_assert(fd == 3);
+
+        del_pollfd(&tdaemon, 1);
+        ck_assert_msg(tdaemon.nfds == 1, "Failed to delete pollfd");
+        fd = get_poll_fd(&tdaemon, 1);
+        ck_assert(fd == -1);
+
+        del_pollfd(&tdaemon, 0);
+        ck_assert_msg(tdaemon.nfds == 0, "Failed to delete pollfd");
+        fd = get_poll_fd(&tdaemon, 0);
+        ck_assert(fd == -1);
+}
+END_TEST
+
+START_TEST(check_add_remove_client)
+{
+        setup();
+
+        client *cl, *cl1, *cl2, *cl3;
+        cl1 = add_client(&(tdaemon.client_head), 1);
+        cl2 = add_client(&(tdaemon.client_head), 2);
+        cl3 = add_client(&(tdaemon.client_head), 3);
+
+        int i = 3;
+        LIST_FOREACH(cl, &(tdaemon.client_head), client_ptrs) {
+                ck_assert_msg((cl->fd == i), "Actual value is %d\n", cl->fd);
+                i--;
+        }
+
+        remove_client(&(tdaemon.client_head), cl1);
+        remove_client(&(tdaemon.client_head), cl2);
+        remove_client(&(tdaemon.client_head), cl3);
+        //ck_assert(tdaemon.client_head.lh_first == NULL);
+        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove clients\n");
+}
+END_TEST
+
+void set_up_socket_pair(int *client, int *server)
+{
+        int sv[2];
+        int ret;
+
+        ret = socketpair(AF_UNIX, SOCK_STREAM, 0, sv);
+        ck_assert_msg(ret == 0, "Failed to create socket pair\n");
+        ck_assert_msg((fcntl(sv[0], F_SETFL, O_NONBLOCK) == 0),
+                      "Failed to set socket to non-blocking\n");
+        ck_assert_msg((fcntl(sv[1], F_SETFL, O_NONBLOCK) == 0),
+                      "Failed to set socket to non-blocking\n");
+        *client = sv[0];
+        *server = sv[1];
+}
+
+START_TEST(check_handle_client_with_no_data)
+{
+        setup();
+
+        client *cl;
+
+        int server_fd;
+        int client_fd;
+        bool processed;
+
+        set_up_socket_pair(&client_fd, &server_fd);
+        cl = add_client(&(tdaemon.client_head), client_fd);
+        ck_assert_msg(cl != NULL, "failed to malloc client");
+        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
+
+        processed = handle_client(&tdaemon, 0, cl);
+        ck_assert(processed == false);
+        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
+        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
+        close(server_fd);
+}
+END_TEST
+
+START_TEST(check_handle_client_with_incorrect_data)
+{
+        setup();
+
+        client *cl;
+        char *buf = "test";
+        int server_fd;
+        int client_fd;
+        bool processed;
+
+        set_up_socket_pair(&client_fd, &server_fd);
+        cl = add_client(&(tdaemon.client_head), client_fd);
+        ck_assert_msg(cl != NULL, "failed to malloc client");
+        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
+
+        ssize_t ret = write(server_fd, buf, 2);
+        ck_assert(ret == 2);
+
+        processed = handle_client(&tdaemon, 0, cl);
+        ck_assert(processed == false);
+        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
+        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
+        close(server_fd);
+}
+END_TEST
+
+START_TEST(check_handle_client_with_incorrect_size)
+{
+        setup();
+
+        client *cl;
+        int server_fd;
+        int client_fd;
+        bool processed;
+        char *data = "test";
+        char buf[4096];
+
+        set_up_socket_pair(&client_fd, &server_fd);
+        cl = add_client(&(tdaemon.client_head), client_fd);
+        ck_assert_msg(cl != NULL, "failed to malloc client");
+        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
+
+        int size = 10;
+        memset(buf, 0, 4096);
+        memcpy(buf, &size, RECORD_SIZE_LEN);
+        memcpy(buf + RECORD_SIZE_LEN, data, sizeof(uint32_t));
+        ssize_t ret = write(server_fd, buf, 2);
+        ck_assert(ret == 2);
+
+        processed = handle_client(&tdaemon, 0, cl);
+        ck_assert(processed == false);
+        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
+        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
+        close(server_fd);
+}
+END_TEST
+
+START_TEST(check_handle_client_with_correct_size)
+{
+        setup();
+
+        client *cl;
+        int server_fd;
+        int client_fd;
+        bool processed;
+        char *data = "testing hello world message for handle client check";
+        char buf[256];
+
+        set_up_socket_pair(&client_fd, &server_fd);
+        cl = add_client(&(tdaemon.client_head), client_fd);
+        ck_assert_msg(cl != NULL, "failed to malloc client");
+        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
+
+        size_t size = strlen(data);
+        memset(buf, 0, 256);
+        memcpy(buf, &size, RECORD_SIZE_LEN);
+        memcpy(buf + RECORD_SIZE_LEN, &size, sizeof(uint32_t));
+        memcpy(buf + 2 * sizeof(uint32_t), data, strlen(data) + 1);
+        ck_assert(strcmp(data, buf + 2 * sizeof(int32_t)) == 0);
+
+        ssize_t ret = write(server_fd, buf, 2 * sizeof(uint32_t) + size + 1);
+        ck_assert(ret != -1);
+        processed = handle_client(&tdaemon, 0, cl);
+        ck_assert(processed == true);
+
+        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with no data\n");
+        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with n data\n");
+        close(server_fd);
+}
+END_TEST
+
+START_TEST(check_process_record_with_correct_size_and_data)
+{
+        setup();
+
+        client *cl;
+        int server_fd, client_fd;
+        bool processed;
+        char *record;
+        size_t record_size;
+        char *headers = "record_format_version: 1\nclassification: crash/kernel/bug\nseverity: 0\n"
+                        "machine_id: 1234\ncreation_timestamp: 1418672344\narch:x86_64\n"
+                        "host_type: macbookpro\nbuild: 200\nkernel_version: 3.15\n"
+                        "payload_format_version: 1\n"
+                        "system_name: clear-linux-os\n"
+                        "board_name: Qemu|Intel\n"
+                        "cpu_model: Intel(R) Core(TM) i7-5650U CPU @ 2.20GHz\n"
+                        "bios_version: Qemu\n"
+                        "event_id: 3a2d799826edc6266d72824d2aac6763\n";
+        char *post_body = "test message";
+
+        set_up_socket_pair(&client_fd, &server_fd);
+        cl = add_client(&(tdaemon.client_head), client_fd);
+        ck_assert_msg(cl != NULL, "failed to malloc client");
+        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
+
+        record = get_serialized_record(headers, post_body, &record_size);
+        ssize_t ret = write(server_fd, record, record_size);
+        ck_assert(ret == record_size);
+
+        processed = handle_client(&tdaemon, 0, cl);
+        ck_assert(processed == true);
+        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with correct data\n");
+        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with correct data\n");
+        close(server_fd);
+        free(record);
+}
+END_TEST
+
+START_TEST(check_process_record_with_incorrect_headers)
+{
+        setup();
+
+        client *cl;
+        int server_fd, client_fd;
+        bool processed;
+        char *record;
+        size_t record_size;
+        char *headers = "record_format_version: 1\nclassificatioooon: crash/kernel/bug\nseverity: 0\n"
+                        "machine_id: 1234\ncreation_timestamp: 1418672344\narch:x86_64\n"
+                        "host_type: macbookpro\nbuild: 200\nkernel_version: 3.15\n";
+        char *post_body = "test message";
+
+        set_up_socket_pair(&client_fd, &server_fd);
+        cl = add_client(&(tdaemon.client_head), client_fd);
+        ck_assert_msg(cl != NULL, "failed to malloc client");
+        add_pollfd(&tdaemon, client_fd, POLLIN | POLLPRI);
+
+        record = get_serialized_record(headers, post_body, &record_size);
+        ssize_t ret = write(server_fd, record, record_size);
+        ck_assert(ret == record_size);
+
+        processed = handle_client(&tdaemon, 0, cl);
+        ck_assert(processed == true);
+        ck_assert_msg(is_client_list_empty(&(tdaemon.client_head)), "Failed to remove client with incorrect headers\n");
+        ck_assert_msg(tdaemon.nfds == 0, "Failed to remove poll fd for client with incorrect headers\n");
+        close(server_fd);
+        free(record);
+}
+END_TEST
+
+Suite *config_suite(void)
+{
+        // A suite is comprised of test cases, defined below
+        Suite *s = suite_create("probd");
+
+        // Individual unit tests are added to "test cases"
+        TCase *t = tcase_create("probd");
+        tcase_add_test(t, check_add_del_poll_fd);
+        tcase_add_test(t, check_daemon_is_initialized);
+        tcase_add_test(t, check_add_remove_client);
+        tcase_add_test(t, check_handle_client_with_no_data);
+        tcase_add_test(t, check_handle_client_with_incorrect_data);
+        tcase_add_test(t, check_handle_client_with_incorrect_size);
+        tcase_add_test(t, check_handle_client_with_correct_size);
+        tcase_add_test(t, check_process_record_with_correct_size_and_data);
+        tcase_add_test(t, check_process_record_with_incorrect_headers);
+
+        suite_add_tcase(s, t);
+
+        return s;
+}
+
+int main(void)
+{
+        Suite *s;
+        SRunner *sr;
+
+        s = config_suite();
+        sr = srunner_create(s);
+
+        // Use the TAP driver for now, so that each
+        // unit test will PASS/FAIL in the log output.
+        srunner_set_log(sr, NULL);
+        srunner_set_tap(sr, "-");
+
+        srunner_run_all(sr, CK_SILENT);
+        // failed = srunner_ntests_failed(sr);
+        srunner_free(sr);
+
+        // if you want the TAP driver to report a hard error based
+        // on certain conditions (e.g. number of failed tests, etc.),
+        // return non-zero here instead.
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

tests/check_probes.c

@@ -29,36 +29,18 @@
 #include "src/probes/klog_scanner.h"
 #include "src/probes/oops_parser.h"
 
-char reason[1024];
-nc_string *bt;
-nc_string *pl;
+static char reason[1024];
+static nc_string *pl;
 
 void callback_func(struct oops_log_msg *msg)
 {
-        strncpy(reason, msg->lines[0], strlen(msg->lines[0]) + 1);
-
-        pl = parse_payload(msg);
-}
-
-void setup(char *oopsfile)
-{
-        char *buf = NULL;
-        int buflen = 0;
-
-        oops_parser_cleanup();
-
-        buf = readfile(oopsfile);
-        buflen = (int)getbuflen();
-
-        oops_parser_init(write_oops_to_file);
-
-        // Allocate space to write the oops file to
-        if (allocate_filespace() < 0) {
-                telem_log(LOG_ERR, "%s\n", strerror(errno));
-                exit(EXIT_FAILURE);
+        size_t len = strlen(msg->lines[0]) + 1;
+        if (len > sizeof(reason)) {
+                len = sizeof(reason);
         }
-
-        split_buf_by_line(buf, buflen);
+        memcpy(reason, msg->lines[0], len);
+        reason[sizeof(reason) - 1] = 0;
+        pl = parse_payload(msg);
 }
 
 void setup_payload(char *oopsfile)

tests/local.mk

@@ -12,8 +12,11 @@ TESTS = $(check_PROGRAMS) $(dist_check_SCRIPTS)
 
 check_PROGRAMS = \
 	%D%/check_config \
-	%D%/check_daemon \
+	%D%/check_probd \
+	%D%/check_postd \
 	%D%/check_probes \
+	%D%/check_ncb64  \
+	%D%/check_journal \
 	%D%/check_libtelemetry
 
 dist_check_SCRIPTS = \
@@ -30,25 +33,70 @@ dist_check_SCRIPTS = \
 	@CHECK_LIBS@ \
 	$(top_builddir)/src/libtelem-shared.la
 
-%C%_check_daemon_SOURCES = \
-	%D%/check_daemon.c \
-	src/telemdaemon.c \
-	src/telemdaemon.h
+if HAVE_SYSTEMD_JOURNAL
+if LOG_SYSTEMD
+%C%_check_config_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_check_config_LDADD += $(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
 
-%C%_check_daemon_CFLAGS = \
+%C%_check_probd_SOURCES = \
+	%D%/check_probd.c \
+	src/telemdaemon.c \
+	src/telemdaemon.h \
+	src/iorecord.h \
+	src/iorecord.c \
+	src/journal/journal.c \
+	src/journal/journal.h
+
+%C%_check_probd_CFLAGS = \
 	$(AM_CFLAGS) \
 	@CHECK_CFLAGS@ \
 	@CURL_CFLAGS@
-%C%_check_daemon_LDADD = \
+%C%_check_probd_LDADD = \
 	@CHECK_LIBS@ \
 	@CURL_LIBS@ \
 	$(top_builddir)/src/libtelem-shared.la
 
 if LOG_SYSTEMD
 if HAVE_SYSTEMD_JOURNAL
-%C%_check_daemon_CFLAGS += \
+%C%_check_probd_CFLAGS += \
         $(SYSTEMD_JOURNAL_CFLAGS)
-%C%_check_daemon_LDADD += \
+%C%_check_probd_LDADD += \
+        $(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
+%C%_check_postd_SOURCES = \
+	%D%/check_postd.c \
+	src/spool.c \
+	src/iorecord.c \
+	src/retention.c \
+        src/telempostdaemon.c \
+        src/telempostdaemon.h \
+        src/journal/journal.c \
+        src/journal/journal.h
+
+EXTRA_DIST += \
+	%D%/telempostd/correct_message \
+	%D%/telempostd/incorrect_headers \
+	%D%/telempostd/empty_message \
+	%D%/telempostd/incorrect_message
+
+%C%_check_postd_CFLAGS = \
+        $(AM_CFLAGS) \
+        @CHECK_CFLAGS@ \
+        @CURL_CFLAGS@
+%C%_check_postd_LDADD = \
+        @CHECK_LIBS@ \
+        @CURL_LIBS@ \
+        $(top_builddir)/src/libtelem-shared.la
+
+if LOG_SYSTEMD
+if HAVE_SYSTEMD_JOURNAL
+%C%_check_postd_CFLAGS += \
+        $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_check_postd_LDADD += \
         $(SYSTEMD_JOURNAL_LIBS)
 endif
 endif
@@ -95,6 +143,7 @@ EXTRA_DIST += \
         @CHECK_CFLAGS@
 %C%_check_probes_LDADD = \
         @CHECK_LIBS@ \
+        $(top_builddir)/src/libtelemetry.la \
         $(top_builddir)/src/libtelem-shared.la
 
 if LOG_SYSTEMD
@@ -106,9 +155,51 @@ if HAVE_SYSTEMD_JOURNAL
 endif
 endif
 
+%C%_check_ncb64_SOURCES = \
+	%D%/check_ncb64.c \
+	src/nica/b64enc.h \
+	src/nica/b64enc.c
+
+%C%_check_ncb64_CFLAGS = \
+	$(AM_CFLAGS) \
+	@CHECK_CFLAGS@
+
+%C%_check_ncb64_LDADD = \
+	@CHECK_LIBS@
+
+EXTRA_DIST += \
+	%D%/nc_b64enc_test_files/empty \
+	%D%/nc_b64enc_test_files/fo \
+	%D%/nc_b64enc_test_files/foob \
+	%D%/nc_b64enc_test_files/foobar \
+	%D%/nc_b64enc_test_files/long_text
+
+%C%_check_journal_SOURCES = \
+	%D%/check_journal.c \
+	src/journal/journal.c \
+	src/util.h \
+	src/util.c
+
+%C%_check_journal_CFLAGS = \
+	$(AM_CFLAGS) \
+	@CHECK_CFLAGS@
+
+%C%_check_journal_LDADD = \
+	@CHECK_LIBS@
+
+if HAVE_SYSTEMD_JOURNAL
+if LOG_SYSTEMD
+%C%_check_journal_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_check_journal_LDADD += $(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
 %C%_check_libtelemetry_SOURCES = \
 	%D%/check_libtelemetry.c \
-	src/telemetry.c
+	src/configuration.h \
+	src/util.h \
+	src/nica/hashmap.h \
+	src/nica/inifile.h
 
 %C%_check_libtelemetry_CFLAGS = \
 	$(AM_CFLAGS) \
@@ -118,4 +209,11 @@ endif
 	$(top_builddir)/src/libtelemetry.la \
 	$(top_builddir)/src/libtelem-shared.la
 
+if HAVE_SYSTEMD_JOURNAL
+if LOG_SYSTEMD
+%C%_check_libtelemetry_CFLAGS += $(SYSTEMD_JOURNAL_CFLAGS)
+%C%_check_libtelemetry_LDADD += $(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
 # vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

tests/nc_b64enc_test_files/fo

@@ -0,0 +1 @@
+fo

tests/nc_b64enc_test_files/foob

@@ -0,0 +1 @@
+foob

tests/nc_b64enc_test_files/foobar

@@ -0,0 +1 @@
+foobarfoobar