configure.ac: bump version to 2.2.0

Version bump from 2.1.1 to 2.2.0

Signed-off-by: Juro Bystricky <juro.bystricky@intel.com>

.gitignore

@@ -33,18 +33,21 @@ build-aux/m4/ltversion.m4
 *.trs
 /cscope.*
 /tags
-telemd
+telemprobd
+telempostd
 autoscan.log
 configure.scan
 tprobe
 hprobe
+bertprobe
 crashprobe
 journalprobe
+pythonprobe
 telem-record-gen
 klogscanner
-oopsprobe
 pstoreclean
 pstoreprobe
+telem_journal
 make.out
 types_c.taghl
 src/recomp.sh
@@ -53,7 +56,11 @@ src/gcc.out
 tests/demo
 tests/check_config
 tests/check_daemon
+tests/check_journal
 tests/check_libtelemetry
+tests/check_ncb64
+tests/check_postd
+tests/check_probd
 tests/check_probes
 src/data/*.path
 src/data/*.service

.travis.yml

@@ -0,0 +1,35 @@
+dist: trusty
+
+compiler:
+    - gcc
+
+os:
+    - linux
+
+before_script:
+    ./autogen.sh
+
+language: c
+
+addons:
+    apt:
+        sources:
+            - ubuntu-toolchain-r-test
+        packages:
+            - libcurl4-gnutls-dev
+            - valgrind
+            - autoconf 
+            - automake
+            - check
+            - libelf-dev
+            - libdw-dev
+
+install:
+        - wget https://github.com/libcheck/check/releases/download/0.12.0/check-0.12.0.tar.gz
+        - tar -xvf check-0.12.0.tar.gz
+        - pushd check-0.12.0 && ./configure --prefix=/usr --libdir=/usr/lib/x86_64-linux-gnu && make -j48 && sudo make install && popd
+
+script:
+    - ./configure && make && make check
+
+after_failure: cat test-suite.log

AUTHORS

@@ -3,3 +3,6 @@ Patrick McCarty <patrick.mccarty@intel.com>
 Rob Nesius <robert.a.nesius@intel.com>
 Gabrielle N. Beyer <gabrielle.n.beyer@intel.com>
 Pam Leonard <pam.leonard@intel.com>
+Alex V. Jaramillo <alex.v.jaramillo@intel.com>
+California Sullivan <california.l.sullivan@intel.com>
+Juro Bystricky <juro.bystricky@intel.com>

CODING-STYLE

@@ -62,7 +62,7 @@ Coding Style
 - To make type casts stand out more, include no space between the cast and the
   variable, etc. to cast:
 
-        GString **foo = (GString **)bar;
+        nc_string **foo = (nc_string **)bar;
 
 - Whenever arithmetic operators are used, always include one space before and
   after each operator:

Makefile.am

@@ -14,10 +14,13 @@ DISTCHECK_CONFIGURE_FLAGS = \
 
 include $(top_srcdir)/build-aux/make/cflags.make
 include $(top_srcdir)/build-aux/make/ldflags.make
+include $(top_srcdir)/docs/local.mk
 include $(top_srcdir)/scripts/local.mk
 include $(top_srcdir)/src/local.mk
 include $(top_srcdir)/src/data/local.mk
+include $(top_srcdir)/src/nica/local.mk
 include $(top_srcdir)/src/probes/local.mk
+include $(top_srcdir)/src/journal/local.mk
 include $(top_srcdir)/tests/local.mk
 
 release:

README

@@ -1,38 +0,0 @@
-telemetrics-client
-==================
-
-This package provides the front end component of a complete telemetrics
-solution for Linux-based operating systems. Specifically, the front end
-component includes:
-
-- telemetrics probes that collect specific types of data from the operating
-  system.
-
-- a library, libtelemetry, that telemetrics probes use to create telemetrics
-  records and send them to the daemon for further processing.
-
-- a daemon, telemd, that prepares the records to send to a telemetrics server
-  (not included in this source tree), or spools the records on disk in case
-  it's unable to successfully deliver them.
-
-
-Build dependencies
-==================
-
-- libcheck
-
-- libcurl
-
-- glib
-
-- elfutils, which provides libelf and libdwfl libraries..
-
-- (optional) libsystemd, for syslog-style logging to the systemd journal, and
-  socket/path activation of telemd by systemd.
-
-
-Generating HTML documentation
-=============================
-
-At toplevel, run 'doxygen docs/Doxyfile'. The documentation is generated in
-the docs/html/ directory.

README.md

@@ -0,0 +1,330 @@
+[![Build Status](https://travis-ci.org/clearlinux/telemetrics-client.svg?branch=master)](https://travis-ci.org/clearlinux/telemetrics-client)
+
+Telemetrics-client
+==================
+
+This package provides the front end component of a complete telemetrics
+solution for Linux-based operating systems. Specifically, the front end
+component includes:
+
+- telemetrics probes that collect specific types of data from the operating
+  system. For more info on probes go [here](./src/probes).
+
+- a library, libtelemetry, that telemetrics probes use to create telemetrics
+  records and send them to the daemon for further processing.
+
+- telemprobd, a daemon that handles communication with probes and forwards records
+  to a second daemon.
+
+- telempostd, this daemon sends telemetry records to a telemetrics server
+  (not included in this source tree), or spools the records on disk in case
+  it's unable to successfully deliver them.
+
+A telemetrics server implementation that works with this component is available
+from
+[clearlinux/telemetrics-backend](https://github.com/clearlinux/telemetrics-backend).
+
+
+Build dependencies
+---------------------
+
+- libcheck
+
+- libcurl
+
+- elfutils, which provides libelf and libdwfl libraries..
+
+- (optional) libsystemd, for syslog-style logging to the systemd journal, and
+  socket/path activation of telemprobd and telempostd by systemd.
+
+
+Build and installation
+---------------------
+
+```{r, engine='bash', count_lines}
+$ ./autogen.sh
+$ ./configure
+$ make
+```
+
+Set up
+---------------------
+
+There is a config file installed at
+
+```{r, engine='bash', count_lines}
+/usr/share/defaults/telemetrics/telemetrics.conf
+```
+
+To make modifications, copy that file to
+
+```{r, engine='bash', count_lines}
+/etc/telemetrics/telemetrics.conf
+```
+and modify the /etc version.
+
+Descriptions of config options are listed below in the Usage section.
+
+Starting the client
+---------------------
+
+If the client was compiled with systemd support the respective activation units
+should be already in place (after a ```make install``` invocation). In this case
+the client wil start automatically when data is made available to it. i.e. when
+executing an ```/usr/bin/hprobe``` command.
+
+Method 1 (recommended):
+
+```{r, engine='bash', count_lines}
+ telemctl start
+```
+
+Note: the above invocation technically readies the service for both socket and
+path activation, so you may not see an "active" status.
+
+Method 2:
+
+```{r, engine='bash', count_lines}
+systemctl start telemprobd.service
+systemctl start telempostd.service
+```
+
+Method 3:
+
+```{r, engine='bash', count_lines}
+telemprobd &
+telempostd &
+```
+
+Configure the client to autostart at boot
+---------------------
+
+Method 1 (recommended):
+
+Enable the socket-activated service and path unit:
+
+```{r, engine='bash', count_lines}
+systemctl enable telemprobd.socket telempostd.path
+```
+
+Method 2:
+
+Enable services, which automatically enables the socket and path
+units as well:
+
+```{r, engine='bash', count_lines}
+systemctl enable telemprobd.service
+systemctl enable telempostd.service
+```
+
+Usage
+---------------------
+
+Once the client is running, the telemetrics probes will be ready to use.
+
+Available probes:
+
+* hprobe: A test program that utilizes libtelemetry to ensure
+that telemetrics-client works. It sends a "hello world" record to the server.
+
+* crash probe: A handler for core files that sends the corresponding backtraces
+to the server.
+
+The client uses the following configuration options from the configuration file:
+
+* server: This specifies the web server that the telempostd sends the telemetry records to
+* socket_path: This specifies the path of the unix domain socket that the
+  telemprobd listens on for connections from the probes
+* spool_dir: This config option is related to spooling. If the daemon is not
+  able to send the telemetry records to the backend server due to reasons such
+  as the network availability, then it stores the records in a spool directory.
+  This option specifies that path of the spool directory. This directory should
+  be owned by the same user that the daemon is running as.
+
+```{r, engine='bash', count_lines}
+    mkdir -p /var/spool/telemetry
+    chown -R telemetry:telemetry /var/spool/telemetry
+    systemctl restart telemprobd.service
+```
+
+* record_expiry: This is the time in minutes after which the records in the
+  spool directory are deleted by the daemon.
+* spool_max_size: This specifies the maximum size of the spool directory. When
+  the size of the spool directory reaches this limit, new telemetry records are
+  dropped by the daemon.
+* spool_process_time: This specifies the time interval in seconds that the
+  daemon waits for before checking the spool directory for records. The daemon
+  picks up the records in the order of modification date and tries to send the
+  record to the server. It sends a maximum of 10 records at a time. If it was
+  able to send a record successfully, it deletes the record from the spool.  If
+  the daemon finds a record older than the "record_expiry" time, then it
+  deletes that record. The daemon looks at a maximum of 20 records in a single
+  spool run loop.
+* rate_limit_enabled: This determines whether rate-limiting is enabled or
+  disabled. When enabled, there is a threshold on both records sent within a
+  window of time, and record bytes sent within a window a time.
+* record_burst_limit: This is the maximum amount of records allowed to be
+  passed by the daemon within the record_window_length of time. If set to -1,
+  the rate-limiting for record bursts is disabled.
+* record_window_length: This is the time, in minutes (0-59), that establishes
+  the window length for the record_burst_limit. EX: if record_burst_window=1000
+  and record_window_length=15, then no more than 1000 records can be passed
+  within any given fifteen minute window.
+* byte_burst_limit: This is the maximum amount of bytes that can be passed by
+  the daemon within the byte_window_length of time. If set to -1, the
+  rate-limiting for byte bursts is disabled.
+* byte_window_length: This is the time, in minutes (0-59), that establishes the
+  window length for the byte_burst_limit.
+* rate_limit_strategy: This is the strategy chosen once the rate-limiting
+  threshold has been reached. Currently the options are 'drop' or 'spool', with
+  spool being the default. If spool is chosen, records will be spooled and sent
+  at a later time.
+* record_retention_enabled: When this key is enabled (true) the daemon saves a
+  copy of the payload on disk from all valid records. To avoid the excessive use
+  of disk space only the latest 100 records are kept. The default value for this
+  configuration key is false.
+* record_server_delivery_enabled: This key controls the delivery of records to
+  ```server```, when enabled (default value) the record will be posted to the
+  address in the configuration file. If this configuration key is disabled (false)
+  records will not be spooled or posted to backend. This configuration key can
+  be used in combination with ```record_retention_enabled``` to keep copies of
+  telemetry records locally only.
+
+
+Data reported
+---------------------
+
+The data reported by the telemetry client could be understood as two main sets:
+metadata and a payload.
+
+The metadata is used to report details of a machine's architecture. The
+following are the metadata values currently collected (Record Format Version 4):
+
+* record_format_version: version of the record, currently is 'Version 4'. This
+  value changes when new metadata is added.
+* classification: this field is used to identify the type of record sent by a
+  specific client probe; classifications use the format DOMAIN/PROBE/REST, where
+  DOMAIN is the vendor of the probe, PROBE is the probe name, and REST is a
+  probe-defined field to classify what is contained in the payload.
+* severity: this is an integer value between 1 and 4 where 1 is "low" and 4 is
+  "critical"
+* machine_id: a machine identifier that is rotate every 3 days for privacy reasons.
+* creation_timestamp: timestamp when the record was collected.
+* arch: a string describing machine architecture i.e. 'x86_64'.
+* host_type: a string with the combination of 'System Vendor', 'Product Name', and
+  'Product Version' read from dmi file system.
+* build: OS build number.
+* kernel_version: Kernel version.
+* payload_format_version: version of the payload, currently is 'Version 1'.
+* system_name: the value after 'ID=' from '/etc/os-release' (or distribution
+  provided folder)
+* board_name: a string read from dmi file system that combines 'Board Name' and
+  'Board Vendor'.
+* cpu_model: cpu model name extracted from '/proc/cpuinfo'.
+* bios_version: BIOS version.
+* event_id: an id to group multiple records if these were generated by a single
+  event occurrence.
+
+The payload as mentioned above is reported by probes. The telemetry library adds
+the metadata to the payload (done programatically when using library API) for
+more information about probes go [here](./src/probes).
+
+
+Machine id
+---------------------
+
+The machine id reported by the telemetry client is rotated every 3 days for
+privacy reasons. If you wish to have a static machine id for testing purposes,
+you can opt in by creating a static machine id file named
+"opt-in-static-machine-id" under the directory  "/etc/telemetrics/". Where
+"unique machine ID" is your desired static machine ID:
+
+```{r, engine='bash', count_lines}
+# mkdir -p /etc/telemetrics
+# echo "unique machine ID" > /etc/telemetrics/opt-in-static-machine-id
+```
+
+The telemetry client reads, at most, the first 32 characters from this file
+uses it for the machine id. You can put a string like 'my-machine-name' in this
+file to easily identify your machine.  Restart telemprobd for the machine id
+changes to take effect by running:
+
+```{r, engine='bash', count_lines}
+# systemctl restart telemprobd.service
+```
+
+You can switch back to the rotating machine id by deleting the override file
+and restarting the client.  You can do a quick test to check that your
+machine-id has changed by running "hprobe" and verifying that a record has
+landed on your backend telemetrics server, with the specified machine id.
+
+Event Id
+----------------------
+
+This is a 32 character lowercase hexadecimal string i.e. '5de9de8d5f3c6a7d445d75ba01cc3322'.
+This header is used to group multiple records by an event id. Before this header
+every single record could have been thought of an event, however this is not always
+the case. There are "events" that trigger the creation of multiple records (i.e.
+updates). The ```event_id``` header was added for probes with the capability to
+detect events and group records based on such events. This header was added to
+```telem-record-gen``` and can be specified using the ```-e``` (```--event-id```
+long form) switch.
+
+```{r, engine=bash, count_lines}
+  -e, --event-id        Event id to use in the record
+```
+
+Debugging locally with telemetrics-client
+-----------------------------------------------
+
+The function of the telemetrics-client is to handle the transport of information
+reported by a probe to a backend (see ```server``` key in configuration). This
+information is helpful for developers to debug and fix reported crashes, however
+developers not always have access to the backend in these case users can leverage
+features added for local debugging. The following is a list of steps to enable
+local debug:
+
+* *Enabling record retention*: this step configures telempostd to keep
+copies of telemetry records locally. To enable record retention set the value of
+```record_retention_enabled``` from ```false``` to ```true```. Optionally set
+```record_server_delivery_enabled`` to ```false``` to keep records local only.
+Remember to restart the client after configuration values are updated
+(```telemctl restart```).
+
+* *Creating a record*: run ```hprobe``` command to create a record for the purposes
+of this step by step guide. Once we have the record or records that you need to
+capture locally you can display the data.
+
+* *Displaying record metadata*: telempostd keeps metadata of any valid record,
+to display this data a new option to telemctl was added ```telemctl journal```.
+Assuming that the last record created was the record from previous step `hprobe` we
+can use `tail -n 1` to print the last created record only, i.e.
+
+```
+$ sudo telemctl journal | tail -n 1
+$ org.clearlinux/hello/world     Mon 2018-04-02 17:48:01 UTC a19a0d41ba16788881e274b19b8a1be4 5de9de8d5f3c6a7d445d75ba01cc3322 60c014cd-4693-40f1-b334-548cd932949b
+```
+
+The headers for the metadata (along with other information) can be printed using the
+```-V``` switch with ```telemctl journal``` command, i.e.
+
+```
+$ sudo telemctl journal -V | head -n 1
+$ Classification             Time stamp              Record ID                    Event ID                     Boot ID
+```
+
+* *Displaying record payload*: to print the content of a record payload you can use
+the ```-i``` (```--include_record``` long format) option to ```telemctl journal```
+command. To print the specific record you created you can use the option ```-r```
+(```--record_id``` long format) with the Record Id of the generated record, i.e.
+
+```
+$ sudo telemctl journal --record_id a19a0d41ba16788881e274b19b8a1be4 --include_record
+$ org.clearlinux/hello/world     Mon 2018-04-02 17:48:01 UTC a19a0d41ba16788881e274b19b8a1be4 5de9de8d5f3c6a7d445d75ba01cc3322 60c014cd-4693-40f1-b334-548cd932949b
+$ hello
+```
+
+## Security Disclosures
+
+To report a security issue or receive security advisories please follow procedures
+in this [link](https://01.org/security).

TODO

@@ -1,24 +0,0 @@
-Build
-=====
-- make sure our AM_CFLAGS do not break older compilers, e.g. compilers
-  that do not support -fstack-protector-strong
-
-Probes
-======
-- add pstore probe (Archana)
-- journal probe: detect kernel oopses (Patrick)
-- crash probe: add function/register value info to crash payloads (Patrick)
-- crash probe: if some symbols are missing from backtraces, attempt the
-  debuginfo lookup again after a certain timeout (Patrick)
-- crash probe: for non-debug builds, ensure that we are invoked by the kernel,
-  and write tests to verify that attempts to run the crash probe for non-debug
-  builds fail. (Patrick)
-
-Library
-=======
-- add API for structured payloads
-
-Other
-=====
-- set fixed size for system-wide core file ulimit, and afterward notify QA team
-  to update the BAT test that checks for the ulimit value.

build-aux/make/cflags.make

@@ -2,7 +2,6 @@ AM_CFLAGS = \
 	-std=gnu99 \
 	-pedantic \
 	-Wall \
-	-fstack-protector \
 	-Wformat \
 	-Wformat-security \
 	-Wimplicit-function-declaration \
@@ -12,9 +11,11 @@ AM_CFLAGS = \
 	-Wconversion \
 	-Wunreachable-code \
 	-funsigned-char \
-	-fstack-protector-strong \
 	-fPIE \
-	-fPIC
+	-fPIC \
+	-fno-strict-overflow \
+	-fno-delete-null-pointer-checks \
+	-fwrapv
 
 AM_CPPFLAGS = \
 	-D_FORTIFY_SOURCE=2 \
@@ -26,8 +27,8 @@ AM_CPPFLAGS = \
 	-DABSTOPSRCDIR=\"$(abs_top_srcdir)\" \
 	-DDATADIR=\"$(datadir)\" \
 	-DLOCALSTATEDIR=\"$(localstatedir)\" \
-	-DKERNELOOPSDIR=\"$(localstatedir)/cache/telemetry/oops\" \
 	-DPSTOREDIR=\"$(localstatedir)/cache/telemetry/pstore\" \
-	-DTESTOOPSDIR=\"$(top_srcdir)/tests/oops_test_files\"
+	-DTESTOOPSDIR=\"$(top_srcdir)/tests/oops_test_files\" \
+	-DBACKEND_ADDR=\"$(BACKEND_ADDR)\"
 
 # vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

configure.ac

@@ -2,13 +2,13 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([telemetrics-client], [1.9.0], [https://clearlinux.org/])
+AC_INIT([telemetrics-client], [2.2.0], [https://clearlinux.org/])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([1.14 -Wall -Werror -Wno-extra-portability foreign subdir-objects])
 AM_SILENT_RULES([yes])
 LT_INIT([disable-static])
 AC_CONFIG_MACRO_DIRS([build-aux/m4])
-AC_CONFIG_SRCDIR([src/main.c])
+AC_CONFIG_SRCDIR([src/probe.c])
 AC_CONFIG_HEADERS([config.h])
 
 # Checks for programs.
@@ -21,9 +21,10 @@ AC_PROG_LN_S
 AC_PROG_MAKE_SET
 
 # Checks for libraries.
-PKG_CHECK_MODULES([CHECK], [check >= 0.9.12])
+
+# check >= 0.9.12 is required for TAP output
+PKG_CHECK_MODULES([CHECK], [check >= 0.12])
 PKG_CHECK_MODULES([CURL], [libcurl])
-PKG_CHECK_MODULES([GLIB], [glib-2.0])
 AC_CHECK_LIB([elf], [elf_begin], [have_elflib=yes], [AC_MSG_ERROR([Unable to find libelf from elfutils])])
 AC_CHECK_LIB([dw], [dwfl_begin], [have_dwlib=yes], [AC_MSG_ERROR([Unable to find libdw from elfutils])])
 AS_IF([test "x$have_elflib" = "xyes" -a "x$have_dwlib" = "xyes"],
@@ -97,6 +98,11 @@ AC_ARG_WITH([systemdsysctldir], AS_HELP_STRING([--with-systemdsysctldir=DIR],
 test -z "${sysctlpath}" && sysctlpath=/usr/lib/sysctl.d
 AC_SUBST(SYSTEMD_SYSCTLDIR, [${sysctlpath}])
 
+AC_ARG_WITH([systemdsystemctldir], AS_HELP_STRING([--with-systemdsystemctldir=DIR],
+            [path to systemd systemctl dir @<:@default=/usr/bin@:>@]), [systemctlpath=${withval}])
+test -z "${systemctlpath}" && systemctlpath=/usr/bin
+AC_SUBST(SYSTEMD_SYSTEMCTLDIR, [${systemctlpath}])
+
 # systemd.pc gives the /etc conf dir, so we can't use PKG_CONFIG for this one
 AC_ARG_WITH([systemdsystemconfdir], AS_HELP_STRING([--with-systemdsystemconfdir=DIR],
 	    [path to systemd system conf dir @<:@default=/usr/lib/systemd/system.conf.d@:>@]), [confpath=${withval}],
@@ -113,6 +119,11 @@ AC_ARG_WITH([loglevel], AS_HELP_STRING([--with-loglevel=NUM],
 	    [loglevel=5])
 AC_DEFINE_UNQUOTED([MAX_LOG_LEVEL], [${loglevel}], [Maximum log level for binaries])
 
+AC_ARG_WITH([backendserveraddr], AS_HELP_STRING([--with-backendserveraddr=URI],
+	    [uri to telemetrics backend server @<:@default=https://clr.telemetry.intel.com/v2/collector@:>@]), [backendaddr=${withval}])
+test -z "${backendaddr}" && backendaddr=https://clr.telemetry.intel.com/v2/collector
+AC_SUBST(BACKEND_ADDR, [${backendaddr}])
+
 AC_ARG_ENABLE([logtype], AS_HELP_STRING([--enable-logtype],
               [Vector for logging: stderr (default), syslog, systemd]),
 			  [case ${enableval} in
@@ -137,3 +148,27 @@ AC_CONFIG_COMMANDS([config dir],[mkdir -p src/data; touch src/data/.dirstamp])
 AC_CONFIG_FILES([Makefile])
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_OUTPUT
+
+AC_MSG_NOTICE([
+
+telemetrics-client $VERSION
+===========================
+
+prefix:                 $prefix
+exec_prefix:            $exec_prefix
+libdir:                 $libdir
+bindir:                 $bindir
+datarootdir:            $datarootdir
+
+compiler:               $CC
+cflags:                 $CFLAGS
+ldflags:                $LDFLAGS
+
+unitdir:                $unitpath
+tmpfilesdir:            $tmpfilespath
+sysctldir:              $sysctlpath
+systemconfdir:          $confpath
+socketdir:              $socketpath
+loglevel:               $loglevel
+logtype:                $logtype
+])

docs/README.md

@@ -0,0 +1,13 @@
+
+## Notes on manual pages
+
+The manual pages are generated using `rst2man.py`. To recreate them,
+run `make manpages` in the toplevel folder. If you want to edit
+the documentation for upstream changes, make sure to edit the `*.rst`
+files and not the shipped nroff output files.
+
+
+## Generating HTML documentation
+
+At toplevel, run 'doxygen docs/Doxyfile'. The documentation is
+generated in the docs/html/ directory.

docs/local.mk

@@ -0,0 +1,24 @@
+MANPAGES = \
+	docs/man/telemctl.1 \
+	docs/man/telemprobd.1 \
+	docs/man/telempostd.1 \
+	docs/man/telem-record-gen.1 \
+	docs/man/telemetry.3 \
+	docs/man/telemetrics.conf.5
+
+MANLINKS = \
+	docs/man/tm_create_record.3 \
+	docs/man/tm_send_record.3 \
+	docs/man/tm_set_config_file.3 \
+	docs/man/tm_set_payload.3
+
+manpages:
+	for MANPAGE in $(MANPAGES); do \
+		rst2man.py $${MANPAGE}.rst > $${MANPAGE}; \
+		rst2html.py $${MANPAGE}.rst > $${MANPAGE}.html; \
+	done
+
+dist_man_MANS = \
+	$(MANPAGES) $(MANLINKS)
+
+# vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

docs/man/.gitignore

@@ -0,0 +1 @@
+*.html

docs/man/telem-record-gen.1

@@ -0,0 +1,101 @@
+.\" Man page generated from reStructuredText.
+.
+.TH TELEM-RECORD-GEN 1 "" "" ""
+.SH NAME
+telem-record-gen \- Create custom telemetry record
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelem\-record\-gen\fP <options>
+.SH DESCRIPTION
+.sp
+Generate and attempt to send a telemetry record to \fBtelemprobd\fP(1). This
+program can be used to create a telemetry record from a shell script
+or from another program using e.g. \fBsystem()\fP\&. The program connects
+to a local \fBtelemprobd\fP(1) daemon to spool the record for delivery to
+the actual collection service.
+.sp
+The payload can be provided as standard input, as a file with the
+\fB\-\-payload\-file\fP option, or as a string argument with the \fB\-\-payload\fP
+option. If neither the \fB\-\-payload\fP or \fB\-\-payload\-file\fP is provided,
+the payload is assumed to be on <stdin>.
+.sp
+Each record is expected to have a class identifier, a record version
+number, and a severity level. A class identifier is required to be
+provided.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fB\-h\fP, \fB\-\-help\fP:
+Show help options.
+.IP \(bu 2
+\fB\-V\fP, \fB\-\-version\fP:
+Print the program version.
+.IP \(bu 2
+\fB\-f\fP, \fB\-\-config\-file\fP <file>:
+Path to configuration file (not implemented yet).
+.IP \(bu 2
+\fB\-s\fP, \fB\-\-severity\fP:
+Severity level (1\-4) \- (default 1).
+.IP \(bu 2
+\fB\-c\fP, \fB\-\-class\fP:
+Classification \fBlevel_1\fP/\fBlevel_2\fP/\fBlevel_3\fP\&.
+.IP \(bu 2
+\fB\-p\fP, \fB\-\-payload\fP <payload>:
+Record body (max size = 8k).
+.IP \(bu 2
+\fB\-P\fP, \fB\-\-payload\-file\fP <file>:
+File to read payload from.
+.IP \(bu 2
+\fB\-R\fP, \fB\-\-record\-version\fP <version>:
+Version number for format of payload (default 1).
+.IP \(bu 2
+\fB\-e\fP, \fB\-\-event\-id\fP <version>:
+Event id to use in the record. If not provided a randomly generated id will be assigned to record.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH RETURN VALUES
+.sp
+0 on success. A non\-zero exit code indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
+.

docs/man/telem-record-gen.1.rst

@@ -0,0 +1,81 @@
+================
+telem-record-gen
+================
+
+------------------------------
+Create custom telemetry record
+------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telem-record-gen`` <options>
+
+
+DESCRIPTION
+===========
+
+Generate and attempt to send a telemetry record to ``telemprobd``\(1). This
+program can be used to create a telemetry record from a shell script
+or from another program using e.g. ``system()``. The program connects
+to a local ``telemprobd``\(1) daemon to spool the record for delivery to
+the actual collection service.
+
+The payload can be provided as standard input, as a file with the
+``--payload-file`` option, or as a string argument with the ``--payload``
+option. If neither the ``--payload`` or ``--payload-file`` is provided,
+the payload is assumed to be on <stdin>.
+
+Each record is expected to have a class identifier, a record version
+number, and a severity level. A class identifier is required to be
+provided.
+
+
+OPTIONS
+=======
+
+ * ``-h``, ``--help``:
+   Show help options.
+
+ * ``-V``, ``--version``:
+   Print the program version.
+
+ * ``-f``, ``--config-file`` <file>:
+   Path to configuration file (not implemented yet).
+
+ * ``-s``, ``--severity``:
+   Severity level (1-4) - (default 1).
+
+ * ``-c``, ``--class``:
+   Classification ``level_1``/``level_2``/``level_3``.
+
+ * ``-p``, ``--payload`` <payload>:
+   Record body (max size = 8k).
+
+ * ``-P``, ``--payload-file`` <file>:
+   File to read payload from.
+
+ * ``-R``, ``--record-version`` <version>:
+   Version number for format of payload (default 1).
+
+ * ``-e``, ``--event-id`` <version>:
+   Event id to use in the record. If not provided a randomly generated id will be assigned to record.
+
+
+
+RETURN VALUES
+=============
+
+0 on success. A non-zero exit code indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/

docs/man/telemctl.1

@@ -0,0 +1,78 @@
+.\" Man page generated from reStructuredText.
+.
+.TH TELEMCTL 1 "" "" ""
+.SH NAME
+telemctl \- Telemetry service administration tool
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelemctl\fP
+.sp
+\fB/etc/telemetrics/opt\-out\fP
+.SH DESCRIPTION
+.sp
+Control actions for telemetry services. The command can be used to start,
+restart, or stop \fBtelemprobd\fP(1) and \fBtelempostd\fP(1), or to opt\-in or opt\-out of telemetry delivery of records to a central telemetry service.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fBstart\(ga\(ga|\(ga\(gastop\(ga\(ga|\(ga\(garestart\fP:
+Starts, stops or restarts all running telemetry services.
+.IP \(bu 2
+\fBopt\-in\fP:
+Opts in to telemetry, and starts telemetry services. The opt\-out file
+\fB/etc/telemetrics/opt\-out\fP is removed.
+.IP \(bu 2
+\fBopt\-out\fP:
+Opts out of telemetry, and stops telemetry services. The opt\-out file
+\fB/etc/telemetrics/opt\-out\fP is created.
+.IP \(bu 2
+\fBis\-active\fP:
+Checks if telemetry client daemons are active (telemprobd and telempostd).
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH RETURN VALUES
+.sp
+0 on success. A non\-zero exit code indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
+.

docs/man/telemctl.1.rst

@@ -0,0 +1,57 @@
+========
+telemctl
+========
+
+-------------------------------------
+Telemetry service administration tool
+-------------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telemctl``
+
+``/etc/telemetrics/opt-out``
+
+
+DESCRIPTION
+===========
+
+Control actions for telemetry services. The command can be used to start,
+restart, or stop ``telemprobd``\(1) and ``telempostd``\(1), or to opt-in or opt-out of telemetry delivery of records to a central telemetry service.
+
+
+OPTIONS
+=======
+
+ * ``start``|``stop``|``restart``:
+   Starts, stops or restarts all running telemetry services.
+
+ * ``opt-in``:
+   Opts in to telemetry, and starts telemetry services. The opt-out file
+   ``/etc/telemetrics/opt-out`` is removed.
+
+ * ``opt-out``:
+   Opts out of telemetry, and stops telemetry services. The opt-out file
+   ``/etc/telemetrics/opt-out`` is created.
+
+ * ``is-active``:
+   Checks if telemetry client daemons are active (telemprobd and telempostd).
+
+
+RETURN VALUES
+=============
+
+0 on success. A non-zero exit code indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/

docs/man/telemetrics.conf.5

@@ -0,0 +1,127 @@
+.\" Man page generated from reStructuredText.
+.
+.TH TELEMETRICS.CONF 5 "" "" ""
+.SH NAME
+telemetrics.conf \- Telemetry service configuration file
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fB/etc/telemetrics/telemetrics.conf\fP
+.sp
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.SH DESCRIPTION
+.sp
+This file contains configuration parameters for the \fBtelemprobd\fP(1) and \fBtelempostd\fP(1) telemetry service daemons. The daemon reads this file at startup if it exists.
+.SH SYNTAX
+.sp
+The configuration file contains \fBkey=value\fP pairs, formatted as plain
+text, one option per line. Comments can be added by preceding them with the
+\fB#\fP character. All configuration options should be in a section marked
+with \fB[settings]\fP\&.
+.SH OPTIONS
+.INDENT 0.0
+.IP \(bu 2
+\fBserver=<url>\fP
+.sp
+Server URL including protocol designator.
+.IP \(bu 2
+\fBsocket_path=<path>\fP
+.sp
+Path to the socket that \fItelemprobd\fP will listen on.
+.IP \(bu 2
+\fBcainfo=<path>\fP
+.sp
+Certificate file to use for validation of SSL endpoint.
+.IP \(bu 2
+\fBtidheader=<header>\fP
+.sp
+Telemetry ID in header format, usually \fBtidheader=X\-Telemetry\-TID:<uuid>\fP
+post header used to group records in ingestion service, which may be
+ingesting for more than one set of clients. Can be set to any string.
+.IP \(bu 2
+\fBrecord_expiry=<minutes>\fP
+.sp
+Record expiry time in minutes.
+.IP \(bu 2
+\fBspool_dir=<dir>\fP
+.sp
+Local spool directory used to store records being processed.
+.IP \(bu 2
+\fBspool_max_size=<kB>\fP
+.sp
+maximum size of the spool directory in kB. A value of \fB\-1\fP indicates
+no limit. The block size of the files in this directory is considered,
+and not the actual size of the record itself.
+.IP \(bu 2
+\fBspool_process_time=<seconds>\fP
+.sp
+Time in seconds for processing spool. Valid range: 120..300. Values
+outside this range are clamped.
+.IP \(bu 2
+\fBrate_limit_enabled=<true|false>\fP
+.sp
+Enable rate limiting. If this is set to false then all rate\-limiting
+disabled. It is possible to disable each rate\-limit individually below.
+.IP \(bu 2
+\fBrecord_burst_limit=<limit>\fP
+.sp
+Rate limiting record burst limit. Valid Range:  0..\(ga\(gaINT_MAX\(ga\(ga, \-1 = disabled.
+.IP \(bu 2
+\fBrecord_window_length=<minutes>\fP
+.sp
+Rate limiting record window length in minutes. Valid Range: 0..59.
+.IP \(bu 2
+\fBbyte_burst_limit=<limit>\fP
+.sp
+Rate limiting byte burst limit. Valid Range:  0..\(gaINT_MAX\(ga, \-1 = disabled.
+.IP \(bu 2
+\fBbyte_window_length=<minutes>\fP
+.sp
+Rate limiting byte window length in minutes. Valid Range: 0..59.
+.IP \(bu 2
+\fBrate_limit_strategy=<strategy>\fP
+.sp
+Rate limit strategy \- what to do with record if rate\-limiting prevents
+delivery over network. Valid stategies: \fBspool\fP, \fBdrop\fP\&.
+.UNINDENT
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemprobd\fP(1)
+.IP \(bu 2
+\fBtelempostd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
+.

docs/man/telemetrics.conf.5.rst

@@ -0,0 +1,109 @@
+================
+telemetrics.conf
+================
+
+------------------------------------
+Telemetry service configuration file
+------------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 5
+
+
+SYNOPSIS
+========
+
+``/etc/telemetrics/telemetrics.conf``
+
+``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+
+DESCRIPTION
+===========
+
+This file contains configuration parameters for the ``telemprobd``\(1) and ``telempostd``\(1) telemetry service daemons. The daemon reads this file at startup if it exists.
+
+
+SYNTAX
+======
+
+The configuration file contains ``key=value`` pairs, formatted as plain
+text, one option per line. Comments can be added by preceding them with the
+``#`` character. All configuration options should be in a section marked
+with ``[settings]``.
+
+
+OPTIONS
+=======
+
+-  ``server=<url>``
+
+   Server URL including protocol designator.
+
+-  ``socket_path=<path>``
+
+   Path to the socket that `telemprobd` will listen on.
+
+-  ``cainfo=<path>``
+
+   Certificate file to use for validation of SSL endpoint.
+
+-  ``tidheader=<header>``
+
+   Telemetry ID in header format, usually ``tidheader=X-Telemetry-TID:<uuid>``
+   post header used to group records in ingestion service, which may be
+   ingesting for more than one set of clients. Can be set to any string.
+
+-  ``record_expiry=<minutes>``
+
+   Record expiry time in minutes.
+
+-  ``spool_dir=<dir>``
+
+   Local spool directory used to store records being processed.
+
+-  ``spool_max_size=<kB>``
+
+   maximum size of the spool directory in kB. A value of ``-1`` indicates
+   no limit. The block size of the files in this directory is considered,
+   and not the actual size of the record itself.
+
+-  ``spool_process_time=<seconds>``
+
+   Time in seconds for processing spool. Valid range: 120..300. Values
+   outside this range are clamped.
+
+-  ``rate_limit_enabled=<true|false>``
+
+   Enable rate limiting. If this is set to false then all rate-limiting
+   disabled. It is possible to disable each rate-limit individually below.
+
+-  ``record_burst_limit=<limit>``
+
+   Rate limiting record burst limit. Valid Range:  0..``INT_MAX``, -1 = disabled.
+
+-  ``record_window_length=<minutes>``
+
+   Rate limiting record window length in minutes. Valid Range: 0..59.
+
+-  ``byte_burst_limit=<limit>``
+
+   Rate limiting byte burst limit. Valid Range:  0..`INT_MAX`, -1 = disabled.
+
+-  ``byte_window_length=<minutes>``
+
+   Rate limiting byte window length in minutes. Valid Range: 0..59.
+
+-  ``rate_limit_strategy=<strategy>``
+
+   Rate limit strategy - what to do with record if rate-limiting prevents
+   delivery over network. Valid stategies: ``spool``, ``drop``.
+
+
+SEE ALSO
+========
+
+* ``telemprobd``\(1)
+* ``telempostd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/

docs/man/telemetry.3

@@ -1,15 +1,84 @@
-.TH TELEMETRY 3 "April 2015" Telemetrics
+.\" Man page generated from reStructuredText.
+.
+.TH TELEMETRY 3 "" "" ""
 .SH NAME
-telemetry \- Telemetry record creation and sending
+telemetry \- C programming interface for telemetrics-client
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
 .SH SYNOPSIS
-#include <telemetry.h>
 .sp
-tm_create_record()
-.br
-tm_set_payload()
-.br
-tm_send_record()
-.br
-tm_set_config_File()
-
-
+\fB#include "telemetry.h"\fP
+.sp
+\fBstruct telem_ref { struct telem_record *record; };\fP
+.sp
+\fBint tm_create_record(struct telem_ref **t_ref, uint32_t severity, char *classification, uint32_t payload_version)\fP
+.sp
+\fBint tm_set_payload(struct telem_ref *t_ref, char *payload)\fP
+.sp
+\fBint tm_send_record(struct telem_ref *t_ref)\fP
+.sp
+\fBvoid tm_free_record(struct telem_ref *t_ref)\fP
+.sp
+\fBint tm_set_config_file(const char *c_file)\fP
+.SH DESCRIPTION
+.sp
+The functions in the telemetry library facilitate the delivery of
+telemetry data to the \fBtelemprobd\fP(1) service.
+.sp
+The function \fBtm_create_record()\fP initializes a telemetry record and
+sets the severity and classification of that record, as well as the
+payload version number. The memory needed to store the telemetry record
+is allocated and should be freed with \fBtm_free_record()\fP when no longer
+needed.
+.sp
+The function \fBtm_set_payload()\fP attaches the provided telemetry record
+data to the telemetry record. The current maximum payload size is 8192b.
+.sp
+The function \fBtm_send_record()\fP delivers the record to the local
+\fBtelemd\fP(1) service.
+.sp
+The function \fBtm_set_config_file()\fP can be used to provide an alternate
+configuration path to the telemetry library.
+.SH RETURN VALUES
+.sp
+All these functions return \fB0\fP on success, or a non\-zero return value
+if an error occurred. The function \fBtm_free_record()\fP does not return
+any value.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemprobd\fP(1)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
+.

docs/man/telemetry.3.rst

@@ -0,0 +1,66 @@
+=========
+telemetry
+=========
+
+----------------------------------------------
+C programming interface for telemetrics-client
+----------------------------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 3
+
+
+SYNOPSIS
+========
+
+``#include "telemetry.h"``
+
+``struct telem_ref { struct telem_record *record; };``
+
+``int tm_create_record(struct telem_ref **t_ref, uint32_t severity, char *classification, uint32_t payload_version)``
+
+``int tm_set_payload(struct telem_ref *t_ref, char *payload)``
+
+``int tm_send_record(struct telem_ref *t_ref)``
+
+``void tm_free_record(struct telem_ref *t_ref)``
+
+``int tm_set_config_file(const char *c_file)``
+
+
+DESCRIPTION
+===========
+
+The functions in the telemetry library facilitate the delivery of
+telemetry data to the ``telemprobd``\(1) service.
+
+The function ``tm_create_record()`` initializes a telemetry record and
+sets the severity and classification of that record, as well as the
+payload version number. The memory needed to store the telemetry record
+is allocated and should be freed with ``tm_free_record()`` when no longer
+needed.
+
+The function ``tm_set_payload()`` attaches the provided telemetry record
+data to the telemetry record. The current maximum payload size is 8192b.
+
+The function ``tm_send_record()`` delivers the record to the local
+``telemd``\(1) service.
+
+The function ``tm_set_config_file()`` can be used to provide an alternate
+configuration path to the telemetry library.
+
+
+RETURN VALUES
+=============
+
+All these functions return ``0`` on success, or a non-zero return value
+if an error occurred. The function ``tm_free_record()`` does not return
+any value.
+
+
+SEE ALSO
+========
+
+* ``telemprobd``\(1)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/

docs/man/telempostd.1

@@ -0,0 +1,97 @@
+.\" Man page generated from reStructuredText.
+.
+.TH TELEMPOSTD 1 "" "" ""
+.SH NAME
+telempostd \- Telemetry client service
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelempostd\fP <flags>
+.sp
+\fB/etc/telemetrics/telemetrics.conf\fP
+.sp
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.sp
+\fB/etc/telemetrics/opt\-in\-static\-machine\-id\fP
+.SH DESCRIPTION
+.sp
+The \fBtelempostd\fP program delivers locally generated telemetry records to a remote
+telemetry service. Telemetry data can be in any format, and is relayed as\-is.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fB\-f\fP, \fB\-\-config_file\fP [<file>]:
+Configuration file. This overrides the other parameters.
+.IP \(bu 2
+\fB\-h\fP, \fB\-\-help\fP:
+Display this help message.
+.IP \(bu 2
+\fB\-V\fP, \fB\-\-version\fP:
+Print the program version.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH FILES
+.INDENT 0.0
+.IP \(bu 2
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+If no custom configuration file is found, \fBtelempostd\fP uses the
+settings in this file.
+.UNINDENT
+.UNINDENT
+.IP \(bu 2
+\fB/etc/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+Custom configuration file that \fBtelempostd\fP reads. See \fBtelemetrics.conf\fP(5).
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH EXIT STATUS
+.sp
+0 when no errors occurred. A non\-zero exit status indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemetry\fP(3)
+.IP \(bu 2
+\fBtelemetrics.conf\fP(5)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2018 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
+.

docs/man/telempostd.1.rst

@@ -0,0 +1,71 @@
+==========
+telempostd
+==========
+
+------------------------
+Telemetry client service
+------------------------
+
+:Copyright: \(C) 2018 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telempostd`` \<flags\>
+
+``/etc/telemetrics/telemetrics.conf``
+
+``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+``/etc/telemetrics/opt-in-static-machine-id``
+
+
+DESCRIPTION
+===========
+
+The ``telempostd`` program delivers locally generated telemetry records to a remote
+telemetry service. Telemetry data can be in any format, and is relayed as-is.
+
+
+OPTIONS
+=======
+
+  * ``-f``, ``--config_file`` \[\<file\>\]:
+    Configuration file. This overrides the other parameters.
+
+  * ``-h``, ``--help``:
+    Display this help message.
+
+  * ``-V``, ``--version``:
+    Print the program version.
+
+
+FILES
+=====
+
+* ``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+    If no custom configuration file is found, ``telempostd`` uses the
+    settings in this file.
+
+* ``/etc/telemetrics/telemetrics.conf``
+
+    Custom configuration file that ``telempostd`` reads. See ``telemetrics.conf``\(5).
+
+
+EXIT STATUS
+===========
+
+0 when no errors occurred. A non-zero exit status indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemetry``\(3)
+* ``telemetrics.conf``\(5)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

docs/man/telemprobd.1

@@ -0,0 +1,106 @@
+.\" Man page generated from reStructuredText.
+.
+.TH TELEMPROBD 1 "" "" ""
+.SH NAME
+telemprobd \- Telemetry probes service
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.SH SYNOPSIS
+.sp
+\fBtelemprobd\fP <flags>
+.sp
+\fB/etc/telemetrics/telemetrics.conf\fP
+.sp
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.sp
+\fB/etc/telemetrics/opt\-in\-static\-machine\-id\fP
+.SH DESCRIPTION
+.sp
+The \fBtelemprobd\fP program handles communication between telemetry client and telemetry
+probes.
+.SH OPTIONS
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.IP \(bu 2
+\fB\-f\fP, \fB\-\-config_file\fP [<file>]:
+Configuration file. This overides the other parameters.
+.IP \(bu 2
+\fB\-h\fP, \fB\-\-help\fP:
+Display this help message.
+.IP \(bu 2
+\fB\-V\fP, \fB\-\-version\fP:
+Print the program version.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH FILES
+.INDENT 0.0
+.IP \(bu 2
+\fB/usr/share/defaults/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+If no custom configuration file is found, \fBtelemprobd\fP uses the
+settings in this file.
+.UNINDENT
+.UNINDENT
+.IP \(bu 2
+\fB/etc/telemetrics/telemetrics.conf\fP
+.INDENT 2.0
+.INDENT 3.5
+Custom configuration file that \fBtelemprobd\fP reads. See \fBtelemetrics.conf\fP(5).
+.UNINDENT
+.UNINDENT
+.IP \(bu 2
+\fB/etc/telemetrics/opt\-in\-static\-machine\-id\fP
+.INDENT 2.0
+.INDENT 3.5
+If this file exists, the first line of this file will be sent to
+the server as the machine ID. Otherwise, the machine ID will be
+random and changed regularly.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.SH EXIT STATUS
+.sp
+0 when no errors occurred. A non\-zero exit status indicates a failure occurred.
+.SH SEE ALSO
+.INDENT 0.0
+.IP \(bu 2
+\fBtelemetry\fP(3)
+.IP \(bu 2
+\fBtelemetrics.conf\fP(5)
+.IP \(bu 2
+\fI\%https://github.com/clearlinux/telemetrics\-client\fP
+.IP \(bu 2
+\fI\%https://clearlinux.org/documentation/\fP
+.UNINDENT
+.SH COPYRIGHT
+(C) 2017 Intel Corporation, CC-BY-SA-3.0
+.\" Generated by docutils manpage writer.
+.

docs/man/telemprobd.1.rst

@@ -0,0 +1,78 @@
+==========
+telemprobd
+==========
+
+------------------------
+Telemetry probes service
+------------------------
+
+:Copyright: \(C) 2017 Intel Corporation, CC-BY-SA-3.0
+:Manual section: 1
+
+
+SYNOPSIS
+========
+
+``telemprobd`` \<flags\>
+
+``/etc/telemetrics/telemetrics.conf``
+
+``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+``/etc/telemetrics/opt-in-static-machine-id``
+
+
+DESCRIPTION
+===========
+
+The ``telemprobd`` program handles communication between telemetry client and telemetry
+probes.
+
+
+OPTIONS
+=======
+
+  * ``-f``, ``--config_file`` \[\<file\>\]:
+    Configuration file. This overides the other parameters.
+
+  * ``-h``, ``--help``:
+    Display this help message.
+
+  * ``-V``, ``--version``:
+    Print the program version.
+
+
+FILES
+=====
+
+* ``/usr/share/defaults/telemetrics/telemetrics.conf``
+
+    If no custom configuration file is found, ``telemprobd`` uses the
+    settings in this file.
+
+* ``/etc/telemetrics/telemetrics.conf``
+
+    Custom configuration file that ``telemprobd`` reads. See ``telemetrics.conf``\(5).
+
+
+* ``/etc/telemetrics/opt-in-static-machine-id``
+
+    If this file exists, the first line of this file will be sent to
+    the server as the machine ID. Otherwise, the machine ID will be
+    random and changed regularly.
+
+
+EXIT STATUS
+===========
+
+0 when no errors occurred. A non-zero exit status indicates a failure occurred.
+
+
+SEE ALSO
+========
+
+* ``telemetry``\(3)
+* ``telemetrics.conf``\(5)
+* https://github.com/clearlinux/telemetrics-client
+* https://clearlinux.org/documentation/
+

docs/man/tm_create_record.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

docs/man/tm_send_record.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

docs/man/tm_set_config_file.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

docs/man/tm_set_payload.3

@@ -0,0 +1 @@
+.so man3/telemetry.3

scripts/source-indent.sh

@@ -14,6 +14,6 @@ if [ "$?" -ne 0 ]; then
   exit 1
 fi
 
-find .. -path "../*/*.[ch]" -exec uncrustify {} -c tm.cfg --no-backup \;
+find .. -path "../*/*.[ch]" ! -path "../src/nica/*" -exec uncrustify {} -c tm.cfg --no-backup \;
 
 # vi: ts=8 sw=2 sts=2 et tw=80

scripts/telemctl

@@ -2,22 +2,35 @@
 
 declare -a SPECIAL_UNITS=(
   hprobe.timer
-  telemd.socket
-  telemd.path
+  telemprobd.socket
+  telempostd.path
+  klogscanner.service
+  journal-probe-tail.service
+  python-probe.path
 )
 
 declare -a SERVICES=(
   hprobe.service
-  journal-probe.service
-  oops-probe.service
   pstore-probe.service
-  klogscanner.service
-  telemd.service
+  telemprobd.service
+  telempostd.service
+  journal-probe.service
 )
 
 SCRIPT="$0"
 TELEM_DIR=/etc/telemetrics
 OPT_OUT_FILE=${TELEM_DIR}/opt-out
+TELEM_WRK_DIRS_CONF=/usr/lib/tmpfiles.d/telemetrics-dirs.conf
+
+create_work_dirs() {
+  # Creates dirs if missing, adjust ownership if exists
+  systemd-tmpfiles --create ${TELEM_WRK_DIRS_CONF}
+}
+
+telem_remove_work_dirs() {
+  # Remove dirs
+  awk '/^d/{print $2}' ${TELEM_WRK_DIRS_CONF} | xargs rm -rf;
+}
 
 exit_ok() {
   echo "$1" > /dev/stderr
@@ -50,15 +63,24 @@ telem_stop() {
 
 telem_start() {
   [ -f $OPT_OUT_FILE ] && exit_err "Opt out is enabled. Cannot start services."
+  # trigger systemd-tmpfiles work dirs creation
+  create_work_dirs
   # the units in SERVICES are activated as needed, so no need to start them
   for_each_service "start" ${SPECIAL_UNITS[@]}
 }
 
+telem_is_active() {
+  # check only activation units
+  echo "telemprobd :" $(systemctl is-active telemprobd.socket)
+  echo "telempostd :"  $(systemctl is-active telempostd.path)
+}
+
 telem_opt_out() {
   [ -f $OPT_OUT_FILE ] && exit_ok "Already opted out. Nothing to do."
   mkdir -p $TELEM_DIR || exit_err "Failed to create ${TELEM_DIR}."
   touch $OPT_OUT_FILE || exit_err "Failed to create ${OPT_OUT_FILE}."
   telem_stop
+  telem_remove_work_dirs
 }
 
 telem_opt_in() {
@@ -72,20 +94,29 @@ telem_restart() {
   telem_start
 }
 
+telem_journal_cli() {
+  telem_journal "$@"
+}
+
 usage() {
-  printf "$SCRIPT - Control actions for telemetry services\n"
+  format='  %-10s %s\n'
   printf "\n"
-  printf "%15s   %-65s\n" "stop" "Stops all running telemetry services"
-  printf "%15s   %-65s\n" "start" "Starts all telemetry services"
-  printf "%15s   %-65s\n" "restart" "Restarts all telemetry services"
-  printf "%15s   %-65s\n" "opt-in" "Opts in to telemetry, and starts telemetry services"
-  printf "%15s   %-65s\n" "opt-out" "Opts out of telemetry, and stops telemetry services"
+  printf "%s - Control actions for telemetry services\n" "$SCRIPT"
+  printf "\n"
+  printf "$format" "stop" "Stops all running telemetry services"
+  printf "$format" "start" "Starts all telemetry services"
+  printf "$format" "restart" "Restarts all telemetry services"
+  printf "$format" "is-active" "Checks if telemprobd and telempostd are active"
+  printf "$format" "opt-in" "Opts in to telemetry, and starts telemetry services"
+  printf "$format" "opt-out" "Opts out of telemetry, and stops telemetry services"
+  printf "$format" "journal" "Prints telemtry journal contents. Use -h argument with"
+  printf "$format" ""        "command for more options"
   printf "\n"
   exit 2
 }
 
-if [ $# -ne 1 ]; then
-  usage
+if [ "$1" != "journal" ] && [ $# -ne 1 ]; then
+     usage
 fi
 
 if [ $EUID -ne 0 ]; then
@@ -105,6 +136,10 @@ case $SUBCOMMAND in
     telem_start ;;
   restart)
     telem_restart ;;
+  is-active)
+    telem_is_active ;;
+  journal)
+    telem_journal_cli "$@" ;;
   *)
     notice "Unknown command passed to $SCRIPT"
     usage ;;

src/common.c

@@ -33,7 +33,11 @@ static const char *header_names[] = {
         TM_SYSTEM_BUILD_STR,
         TM_KERNEL_VERSION_STR,
         TM_PAYLOAD_VERSION_STR,
-        TM_SYSTEM_NAME_STR
+        TM_SYSTEM_NAME_STR,
+        TM_BOARD_NAME_STR,
+        TM_CPU_MODEL_STR,
+        TM_BIOS_VERSION_STR,
+        TM_EVENT_ID_STR
 };
 
 const char *get_header_name(int ind)

src/common.h

@@ -35,6 +35,10 @@
 #define TM_KERNEL_VERSION 8
 #define TM_PAYLOAD_VERSION 9
 #define TM_SYSTEM_NAME 10
+#define TM_BOARD_NAME 11
+#define TM_CPU_MODEL 12
+#define TM_BIOS_VERSION 13
+#define TM_EVENT_ID 14
 
 #define TM_RECORD_VERSION_STR "record_format_version"
 #define TM_CLASSIFICATION_STR "classification"
@@ -47,14 +51,26 @@
 #define TM_KERNEL_VERSION_STR "kernel_version"
 #define TM_PAYLOAD_VERSION_STR "payload_format_version"
 #define TM_SYSTEM_NAME_STR "system_name"
+#define TM_BOARD_NAME_STR "board_name"
+#define TM_CPU_MODEL_STR "cpu_model"
+#define TM_BIOS_VERSION_STR "bios_version"
+#define TM_EVENT_ID_STR "event_id"
 
-#define NUM_HEADERS 11
+#define NUM_HEADERS 15
+
+#define EVENT_ID_ALPHAB "0123456789abcdef"
+#define EVENT_ID_LEN 32
+
+/* Journal */
+#define JOURNAL_PATH "/var/log/telemetry/journal"
+#define JOURNAL_TMPDIR  "/var/log/telemetry/"
+#define RECORD_RETENTION_DIR "/var/log/telemetry/records"
 
 /* For internal library usage. Bump the version whenever we change the record
  * structure (e.g. adding or removing a header field). Note that the value
  * should be an unsigned int.
  */
-static const uint32_t RECORD_FORMAT_VERSION = 2;
+static const uint32_t RECORD_FORMAT_VERSION = 4;
 
 #define TM_SITE_VERSION_FILE "/etc/os-release"
 #define TM_DIST_VERSION_FILE "/usr/lib/os-release"
@@ -62,7 +78,7 @@ static const uint32_t RECORD_FORMAT_VERSION = 2;
 #define TM_OPT_OUT_FILE "/etc/telemetrics/opt-out"
 
 /* Currently max supported payload size is 8kb */
-#define MAX_PAYLOAD_SIZE 8192
+#define MAX_PAYLOAD_LENGTH 8192
 
 /* Maximum size for a category string is 122 bytes */
 #define MAX_CLASS_LENGTH 122
@@ -70,12 +86,12 @@ static const uint32_t RECORD_FORMAT_VERSION = 2;
 /* Maximum size for a sub category 40 bytes */
 #define MAX_SUBCAT_LENGTH 40
 
-/* Max Payload size is 8 kibabytes (8192) */
-#define MAX_PAYLOAD_LENGTH 8192
+/* Exit daemon after idle for this duration in seconds */
+#define TM_DAEMON_EXIT_TIME  (5 * 60)
 
 /* SPOOL DEFINES */
-/* Spooling should run every TM_SPOOL_RUN_MAX atleast */
-#define TM_SPOOL_RUN_MAX (60 /*min*/ * 60 /*sec*/)
+/* Spooling should run at least every TM_SPOOL_RUN_MAX */
+#define TM_SPOOL_RUN_MAX TM_DAEMON_EXIT_TIME
 
 /* Spooling should not run more often than TM_SPOOL_RUN_MIN */
 #define TM_SPOOL_RUN_MIN (2 /*min*/ * 60 /*sec*/)
@@ -86,8 +102,10 @@ static const uint32_t RECORD_FORMAT_VERSION = 2;
 /* Maximum records that can be processed in a single spool run loop*/
 #define TM_SPOOL_MAX_PROCESS_RECORDS 60
 
-/* Time to check to exit from  the daemon */
-#define TM_DAEMON_EXIT_TIME  (2 /*h*/ * 60 /*m*/ * 60 /*sec*/)
+/* Definitions for config file override */
+#define CFG_PREFIX        "CFG:"
+#define CFG_PREFIX_LENGTH 4
+#define CFG_PREFIX_32BIT  0x3a474643
 
 /* Very simple structure. Array of header strings and a payload. Calling
  * program is reponsible for passing in the payload as a simple string.
@@ -101,4 +119,5 @@ struct telem_record {
 
 const char *get_header_name(int ind);
 
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/configuration.c

@@ -16,39 +16,73 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <glib.h>
 #include <errno.h>
 #include <limits.h>
 #include <dirent.h>
 #include <string.h>
 #include <ctype.h>
 #include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
 
 #include "configuration.h"
 #include "util.h"
 #include "common.h"
+#include "log.h"
+
+#include "nica/inifile.h"
 
 static char *config_file = NULL;
 static char *default_config_file = DATADIR "/defaults/telemetrics/telemetrics.conf";
 static char *etc_config_file = "/etc/telemetrics/telemetrics.conf";
-static GKeyFile *keyfile = NULL;
+static NcHashmap *keyfile = NULL;
 static bool cmd_line_cfg = false;
 
 /* Conf strings, integers, and booleans expected in the conf file */
-const char *config_key_str[] = { NULL, "server", "socket_path", "spool_dir",
-                                 "rate_limit_strategy", "cainfo",
-                                 "tidheader", NULL };
+static const char *config_key_str[] = { "server",
+                                        "socket_path",
+                                        "spool_dir",
+                                        "rate_limit_strategy",
+                                        "cainfo",
+                                        "tidheader"};
 
-const char *config_key_int[] = { NULL, "record_expiry", "spool_max_size",
-                                 "spool_process_time", "record_window_length",
-                                 "byte_window_length", "record_burst_limit",
-                                 "byte_burst_limit", NULL };
+static const char *config_key_int[] = { "record_expiry",
+                                        "spool_max_size",
+                                        "spool_process_time",
+                                        "record_window_length",
+                                        "byte_window_length",
+                                        "record_burst_limit",
+                                        "byte_burst_limit" };
+
+static const char *config_key_bool[] = { "rate_limit_enabled",
+                                         "daemon_recycling_enabled",
+                                         "record_retention_enabled",
+                                         "record_server_delivery_enabled" };
+
+static const char *config_str_default[] = { DEFAULT_SERVER_ADDR,
+                                            DEFAULT_SOCKET_PATH,
+                                            DEFAULT_SPOOL_DIR,
+                                            DEFAULT_RATE_LIMIT_STRATEGY,
+                                            DEFAULT_CAINFO,
+                                            DEFAULT_TIDHEADER };
+
+static const bool config_bool_default[] = { DEFAULT_RATE_LIMIT_ENABLED,
+                                            DEFAULT_DAEMON_RECYCLING_ENABLED,
+                                            DEFAULT_RECORD_RETENTION_ENABLED,
+                                            DEFAULT_RECORD_SERVER_DELIVERY_ENABLED };
+
+static const int config_int_default[] = { DEFAULT_RECORD_EXPIRY,
+                                          DEFAULT_SPOOL_MAX_SIZE,
+                                          DEFAULT_SPOOL_PROCESS_TIME,
+                                          DEFAULT_RECORD_WINDOW_LENGTH,
+                                          DEFAULT_BYTE_WINDOW_LENGTH,
+                                          DEFAULT_RECORD_BURST_LIMIT,
+                                          DEFAULT_BYTE_BURST_LIMIT };
 
-const char *config_key_bool[] = { NULL, "rate_limit_enabled", NULL };
 
 static struct configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
 
-int validate_config_file(char *f)
+static int validate_config_file(const char *f)
 {
         struct stat sbuf;
 
@@ -56,75 +90,134 @@ int validate_config_file(char *f)
                 return -EINVAL;
         }
 
+        /* check if the file exists and is a regular file */
         if (stat(f, &sbuf) != 0) {
                 return -errno;
         }
 
+        if (!S_ISREG(sbuf.st_mode)) {
+                return -EINVAL;
+        }
+
         return 0;
 
 }
 
-void set_config_file(char *filename)
+const char *get_config_file(void)
 {
+        return config_file;
+}
 
-        /* TODO: This should return a status, but that will require coordinating
-         * an API change.  For now, config-entries referencing a null config
-         * file will fail, so the end result still works. */
+const char *get_cmd_line_config_file(void)
+{
+        if (cmd_line_cfg == true) {
+                return config_file;
+        }
+        return NULL;
+}
 
-        if (validate_config_file(filename) == 0) {
-                config_file = strdup(filename);
-                cmd_line_cfg = true;
+int set_config_file(const char *filename)
+{
+        int ret;
+
+        if ((ret = validate_config_file(filename)) == 0) {
+                char *filename1 = strdup(filename);
+                if (filename1 == NULL) {
+                    ret = -errno;
+                } else {
+                    config_file = filename1;
+                    cmd_line_cfg = true;
+                }
         }
 
+        return ret;
+}
+
+bool set_default_config_values(struct configuration *config)
+{
+        for (int i = 0; i < CONF_STR_MAX; i++) {
+                config->strValues[i] = strdup(config_str_default[i]);
+                        if (config->strValues[i] == NULL) {
+                                telem_log(LOG_ERR, "Could not set config item %s: %s\n",
+                                          config_key_str[i], strerror(errno));
+                                return false;
+                        }
+        }
+        for (int i = 0; i < CONF_INT_MAX; i++) {
+                config->intValues[i] = config_int_default[i];
+        }
+        for (int i = 0; i < CONF_BOOL_MAX; i++) {
+                config->boolValues[i] = config_bool_default[i];
+        }
+        return true;
 }
 
 bool read_config_from_file(char *config_file, struct configuration *config)
 {
-        GError *error = NULL;
-        GKeyFileFlags flags = G_KEY_FILE_NONE;
-
         if (keyfile != NULL) {
-                g_key_file_free(keyfile);
+                nc_hashmap_free(keyfile);
         }
 
-        keyfile = g_key_file_new();
-
-        if (!g_key_file_load_from_file(keyfile, (gchar *)config_file, flags, &error)) {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to read config file: %s\n",
-                        error->message);
-#endif
+        keyfile = nc_ini_file_parse(config_file);
+        if (!keyfile) {
+                telem_log(LOG_ERR, "Failed to read config file\n");
                 return false;
         } else {
-                for (int i = CONF_STR_MIN + 1; i < CONF_STR_MAX; i++) {
-                        config->strValues[i] = g_key_file_get_string(keyfile,
-                                                                     "settings", config_key_str[i], &error);
-                        if (error != NULL) {
-                                return false;
+                for (int i = 0; i < CONF_STR_MAX; i++) {
+                        char *ptr;
+                        ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_str[i]);
+                        if (ptr) {
+                                config->strValues[i] = strdup(ptr);
+                                if (config->strValues[i] == NULL) {
+                                        telem_log(LOG_ERR, "Could not set config item %s: %s\n",
+                                                  config_key_str[i], strerror(errno));
+                                        return false;
+                                }
+                        } else {
+                                config->strValues[i] = strdup(config_str_default[i]);
                         }
+
                 }
-                for (int i = CONF_INT_MIN + 1; i < CONF_INT_MAX; i++) {
-                        config->intValues[i] = g_key_file_get_int64(keyfile,
-                                                                    "settings", config_key_int[i], &error);
-                        if (error != NULL) {
-                                return false;
+
+                for (int i = 0; i < CONF_INT_MAX; i++) {
+                        char *ptr;
+                        ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_int[i]);
+                        if (ptr) {
+                                errno = 0;
+                                config->intValues[i] = strtoll(ptr, NULL, 10);
+                                if (errno != 0) {
+                                        telem_log(LOG_ERR, "Error while parsing value of option %s: %s\n",
+                                                  config_key_int[i], strerror(errno));
+                                        return false;
+                                }
+                        } else {
+                                config->intValues[i] = config_int_default[i];
                         }
                 }
 
-                for (int i = CONF_BOOL_MIN + 1; i < CONF_BOOL_MAX; i++) {
-                        config->boolValues[i] = g_key_file_get_boolean(keyfile,
-                                                                       "settings", config_key_bool[i], &error);
-                        if (error != NULL) {
-                                return false;
+                for (int i = 0; i < CONF_BOOL_MAX; i++) {
+                        char *ptr;
+                        ptr = nc_hashmap_get(nc_hashmap_get(keyfile, "settings"), config_key_bool[i]);
+                        if (ptr) {
+                                if ((strcasecmp(ptr, "TRUE") == 0) || (strcmp(ptr, "1") == 0)) {
+                                        config->boolValues[i] = true;
+                                } else if ((strcasecmp(ptr, "FALSE") == 0) || (strcmp(ptr, "0") == 0)) {
+                                        config->boolValues[i] = false;
+                                } else {
+                                        telem_log(LOG_ERR, "Configuration item '%s' requires a boolean value\n",
+                                                  config_key_bool[i]);
+                                        return false;
+                                }
+                        } else {
+                                config->boolValues[i] = config_bool_default[i];
                         }
                 }
         }
 
-        config->initialized = true;
         return true;
 }
 
-void initialize_config(void)
+static void initialize_config(void)
 {
         if (config.initialized) {
                 return;
@@ -132,29 +225,28 @@ void initialize_config(void)
 
         /* No config file provided on command line */
         if (!config_file) {
-                if (g_file_test(etc_config_file, G_FILE_TEST_IS_REGULAR)) {
+                if (access(etc_config_file, R_OK) == 0) {
                         config_file = etc_config_file;
                 } else {
-                        if (g_file_test(default_config_file,
-                                        G_FILE_TEST_IS_REGULAR)) {
+                        if (access(default_config_file, R_OK) == 0) {
                                 config_file = default_config_file;
-                        } else {
-                                /* If there is no default config, exit with failure */
-#ifdef DEBUG
-                                fprintf(stderr, "ERR: No configuration file"
-                                        " found, exiting\n");
-#endif
-                                exit(EXIT_FAILURE);
                         }
                 }
         }
-        if (!read_config_from_file(config_file, &config)) {
-                /* Error while parsing file  */
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Error while parsing configuration file\n");
-#endif
-                exit(EXIT_FAILURE);
+
+        if (config_file) {
+                if (!read_config_from_file(config_file, &config)) {
+                        /* Error while parsing file  */
+                        exit(EXIT_FAILURE);
+                }
         }
+        else {
+                if (!set_default_config_values(&config)) {
+                        exit(EXIT_FAILURE);
+                }
+        }
+
+        config.initialized = true;
 }
 
 void reload_config(void)
@@ -174,12 +266,10 @@ void free_configuration(void)
                 return;
         }
 
-        for (int i = CONF_STR_MIN + 1; i < CONF_STR_MAX; i++) {
+        for (int i = 0; i < CONF_STR_MAX; i++) {
                 free(config.strValues[i]);
         }
 
-        g_key_file_free(keyfile);
-
         if (cmd_line_cfg) {
                 free(config_file);
         }
@@ -334,4 +424,22 @@ const char *rate_limit_strategy_config()
 
         return val;
 }
+
+bool daemon_recycling_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_DAEMON_RECYCLING_ENABLED];
+}
+
+bool record_retention_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_RETENTION_ENABLED];
+}
+
+bool record_server_delivery_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED];
+}
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/configuration.h

@@ -18,11 +18,31 @@
 #include <stdbool.h>
 #include <stdint.h>
 
+/* Default configuration settings */
+#define DEFAULT_SERVER_ADDR BACKEND_ADDR
+#define DEFAULT_SOCKET_PATH "/run/telem-0"
+#define DEFAULT_SPOOL_DIR LOCALSTATEDIR "/spool/telemetry"
+#define DEFAULT_RATE_LIMIT_STRATEGY "spool"
+#define DEFAULT_CAINFO ""
+#define DEFAULT_TIDHEADER "X-Telemetry-TID: 6907c830-eed9-4ce9-81ae-76daf8d88f0f"
+
+#define DEFAULT_RECORD_EXPIRY 1200
+#define DEFAULT_SPOOL_MAX_SIZE 5120
+#define DEFAULT_SPOOL_PROCESS_TIME 120
+#define DEFAULT_RECORD_WINDOW_LENGTH 15
+#define DEFAULT_BYTE_WINDOW_LENGTH 20
+#define DEFAULT_RECORD_BURST_LIMIT 1000
+#define DEFAULT_BYTE_BURST_LIMIT -1
+
+#define DEFAULT_RATE_LIMIT_ENABLED true
+#define DEFAULT_DAEMON_RECYCLING_ENABLED true
+#define DEFAULT_RECORD_RETENTION_ENABLED false
+#define DEFAULT_RECORD_SERVER_DELIVERY_ENABLED true
+
 #define TM_MAX_WINDOW_LENGTH (1 /*h*/ * 60 /*m*/)
 
 enum config_str_keys {
-        CONF_STR_MIN = 0,
-        CONF_SERVER_ADDR,
+        CONF_SERVER_ADDR = 0,
         CONF_SOCKET_PATH,
         CONF_SPOOL_DIR,
         CONF_RATE_LIMIT_STRATEGY,
@@ -32,8 +52,7 @@ enum config_str_keys {
 };
 
 enum config_int_keys {
-        CONF_INT_MIN = 0,
-        CONF_RECORD_EXPIRY,
+        CONF_RECORD_EXPIRY = 0,
         CONF_SPOOL_MAX_SIZE,
         CONF_SPOOL_PROCESS_TIME,
         CONF_RECORD_WINDOW_LENGTH,
@@ -44,8 +63,10 @@ enum config_int_keys {
 };
 
 enum config_bool_keys {
-        CONF_BOOL_MIN = 0,
-        CONF_RATE_LIMIT_ENABLED,
+        CONF_RATE_LIMIT_ENABLED = 0,
+        CONF_DAEMON_RECYCLING_ENABLED,
+        CONF_RECORD_RETENTION_ENABLED,
+        CONF_RECORD_SERVER_DELIVERY_ENABLED,
         CONF_BOOL_MAX
 };
 
@@ -58,7 +79,16 @@ typedef struct configuration {
 } configuration;
 
 /* Sets the configuration file to be used later */
-void set_config_file(char *filename);
+int set_config_file(const char *filename);
+
+/* Gets the configuration currently in use */
+const char *get_config_file(void);
+
+/* Gets the configuration specified via command line or NULL */
+const char *get_cmd_line_config_file(void);
+
+/* Sets all default configuration values to a given config */
+bool set_default_config_values(struct configuration *config);
 
 /* Parses the ini format config file */
 bool read_config_from_file(char *filename, struct configuration *config);
@@ -113,4 +143,13 @@ const char *get_cainfo_config(void);
 /* Gets tidheader */
 const char *get_tidheader_config(void);
 
+/* Gets whether recycling is enabled */
+bool daemon_recycling_enabled_config(void);
+
+/* Gets whether local retention of records is enabled */
+bool record_retention_enabled_config(void);
+
+/* Gets whether records should be sent to server_addr */
+bool record_server_delivery_enabled_config(void);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/data/bert-probe.service.in

@@ -0,0 +1,13 @@
+[Unit]
+Description=Telemetrics BERT Probe
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=!/etc/telemetrics/opt-out
+ConditionPathExists=/sys/firmware/acpi/tables/data/BERT
+
+[Service]
+ExecStart=@bindir@/bertprobe
+User=telemetry
+
+[Install]
+WantedBy=multi-user.target

src/data/example.1.conf

@@ -0,0 +1,46 @@
+[settings]
+server=http://127.0.0.1
+
+cainfo=/tmp/cacert.crt
+
+tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
+
+socket_path=/tmp/test_telem_socket
+
+#record expiry time in minutes
+record_expiry=1200
+
+spool_dir=/tmp/spool
+
+#maximum size of the spool directory in KB
+spool_max_size=1024
+
+#time in seconds for processing spool
+spool_process_time=900
+
+#rate limiting record burst limit
+record_burst_limit=100
+
+#rate limiting record window length
+record_window_length = 15
+
+#rate limiting byte burst limit
+byte_burst_limit = 1000
+
+#rate limiting byte window length
+byte_window_length=20
+
+#rate limit enabled
+rate_limit_enabled=true
+
+#rate limit strategy if record not send
+rate_limit_strategy=spool
+
+#daemon recycling enabled
+daemon_recycling_enabled=true
+
+#server delivery disable
+record_server_delivery_enabled=false
+
+#local copy enabled
+record_retention_enabled=true

src/data/example.2.conf

@@ -0,0 +1,4 @@
+[settings]
+server=http://127.0.0.1
+
+cainfo=/tmp/cacert.crt

src/data/example.conf

@@ -16,7 +16,7 @@ spool_dir=/tmp/spool
 spool_max_size=1024
 
 #time in seconds for processing spool
-spool_process_time=900
+spool_process_time=180
 
 #rate limiting record burst limit
 record_burst_limit=100
@@ -35,3 +35,6 @@ rate_limit_enabled=true
 
 #rate limit strategy if record not send
 rate_limit_strategy=spool
+
+#daemon recycling enabled
+daemon_recycling_enabled=true

src/data/hprobe.service.in

@@ -1,11 +1,11 @@
 [Unit]
 Description=Telemetrics Heartbeat Probe
-Requires=telemd.socket
-After=telemd.socket
+Requires=telemprobd.socket
+After=telemprobd.socket
 ConditionPathExists=!/etc/telemetrics/opt-out
 
 [Service]
-ExecStart=@bindir@/hprobe -H
+ExecStart=@bindir@/hprobe -Hlu
 User=telemetry
 
 [Install]

src/data/journal-probe-tail.service.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Telemetrics Systemd Journal Probe
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=!/etc/telemetrics/opt-out
+
+[Service]
+ExecStart=@bindir@/journalprobe -t
+User=telemetry
+
+[Install]
+WantedBy=multi-user.target

src/data/journal-probe.service.in

@@ -1,7 +1,7 @@
 [Unit]
 Description=Telemetrics Systemd Journal Probe
-Requires=telemd.socket
-After=telemd.socket
+Requires=telemprobd.socket
+After=telemprobd.socket
 ConditionPathExists=!/etc/telemetrics/opt-out
 
 [Service]

src/data/klogscanner.service.in

@@ -1,7 +1,7 @@
 [Unit]
 Description=Telemetrics Kernel Log Scanner
-Requires=telemd.socket
-After=telemd.socket oops-probe.service
+Requires=telemprobd.socket
+After=telemprobd.socket
 
 [Service]
 ExecStart=@bindir@/klogscanner

src/data/local.mk

@@ -4,23 +4,32 @@ pathfix = @sed \
 	-e 's|@libdir[@]|$(libdir)|g' \
 	-e 's|@localstatedir[@]|$(localstatedir)|g' \
 	-e 's|@PACKAGE_VERSION[@]|$(PACKAGE_VERSION)|g' \
-	-e 's|@SOCKETDIR[@]|$(SOCKETDIR)|g'
+	-e 's|@SOCKETDIR[@]|$(SOCKETDIR)|g' \
+	-e 's|@systemctldir[@]|$(SYSTEMD_SYSTEMCTLDIR)|g' \
+	-e 's|@BACKEND_ADDR[@]|$(BACKEND_ADDR)|g'
 
 EXTRA_DIST += \
 	%D%/40-core-ulimit.conf \
 	%D%/40-crash-probe.conf.in \
 	%D%/example.conf \
+	%D%/example.1.conf \
+	%D%/example.2.conf \
 	%D%/hprobe.service.in \
 	%D%/hprobe.timer \
+	%D%/bert-probe.service.in \
 	%D%/journal-probe.service.in \
-	%D%/oops-probe.service.in \
+	%D%/journal-probe-tail.service.in \
+	%D%/python-probe.service.in \
+	%D%/python-probe.path.in \
 	%D%/pstore-probe.service.in \
 	%D%/klogscanner.service.in \
 	%D%/pstore-clean.service.in \
 	%D%/libtelemetry.pc.in \
-	%D%/telemd.path.in \
-	%D%/telemd.service.in \
-	%D%/telemd.socket.in \
+	%D%/telempostd.service.in \
+	%D%/telempostd.path.in \
+	%D%/telemprobd.service.in \
+	%D%/telemprobd.socket.in \
+	%D%/telemprobd-update-trigger.service.in \
 	%D%/telemetrics-dirs.conf.in \
 	%D%/telemetrics.conf.in
 
@@ -46,22 +55,30 @@ systemdunitdir = @SYSTEMD_UNITDIR@
 systemdunit_DATA = \
 	%D%/hprobe.service \
 	%D%/hprobe.timer \
+	%D%/bert-probe.service \
 	%D%/journal-probe.service \
-	%D%/oops-probe.service \
+	%D%/journal-probe-tail.service \
+	%D%/python-probe.service \
+	%D%/python-probe.path \
 	%D%/pstore-probe.service \
 	%D%/klogscanner.service \
 	%D%/pstore-clean.service \
-	%D%/telemd.service \
-	%D%/telemd.socket \
-	%D%/telemd.path
+	%D%/telemprobd.service \
+	%D%/telemprobd.socket \
+	%D%/telemprobd-update-trigger.service \
+	%D%/telempostd.service \
+	%D%/telempostd.path
 
 %D%/hprobe.service: %D%/hprobe.service.in
 	$(pathfix) < $< > $@
 
+%D%/bert-probe.service: %D%/bert-probe.service.in
+	$(pathfix) < $< > $@
+
 %D%/journal-probe.service: %D%/journal-probe.service.in
 	$(pathfix) < $< > $@
 
-%D%/oops-probe.service: %D%/oops-probe.service.in
+%D%/journal-probe-tail.service: %D%/journal-probe-tail.service.in
 	$(pathfix) < $< > $@
 
 %D%/pstore-probe.service: %D%/pstore-probe.service.in
@@ -73,13 +90,25 @@ systemdunit_DATA = \
 %D%/pstore-clean.service: %D%/pstore-clean.service.in
 	$(pathfix) < $< > $@
 
-%D%/telemd.service: %D%/telemd.service.in
+%D%/telempostd.path: %D%/telempostd.path.in
 	$(pathfix) < $< > $@
 
-%D%/telemd.socket: %D%/telemd.socket.in
+%D%/telempostd.service: %D%/telempostd.service.in
 	$(pathfix) < $< > $@
 
-%D%/telemd.path: %D%/telemd.path.in
+%D%/python-probe.service: %D%/python-probe.service.in
+	$(pathfix) < $< > $@
+
+%D%/python-probe.path: %D%/python-probe.path.in
+	$(pathfix) < $< > $@
+
+%D%/telemprobd.service: %D%/telemprobd.service.in
+	$(pathfix) < $< > $@
+
+%D%/telemprobd.socket: %D%/telemprobd.socket.in
+	$(pathfix) < $< > $@
+
+%D%/telemprobd-update-trigger.service: %D%/telemprobd-update-trigger.service.in
 	$(pathfix) < $< > $@
 
 sysctldir = @SYSTEMD_SYSCTLDIR@
@@ -92,17 +121,22 @@ systemconfdir = @SYSTEMD_SYSTEMCONFDIR@
 systemconf_DATA = %D%/40-core-ulimit.conf
 
 clean-local:
-	-rm -f  %D%/telemd.service \
-		%D%/telemd.socket \
-		%D%/telemd.path \
+	-rm -f  %D%/telemprobd.service \
+		%D%/telemprobd.socket \
+		%D%/telempostd.service \
+		%D%/telempostd.path \
+		%D%/python-probe.service \
+		%D%/python-probe.path \
+		%D%/telemprobd-update-trigger.service \
 		%D%/telemetrics.conf \
 		%D%/telemetrics-dirs.conf \
 		%D%/libtelemetry.pc \
 		%D%/40-crash-probe.conf \
 		%D%/journal-probe.service \
-		%D%/oops-probe.service \
+		%D%/journal-probe-tail.service \
 		%D%/pstore-probe.service \
 		%D%/klogscanner.service \
 		%D%/pstore-clean.service \
 		%D%/hprobe.service \
+		%D%/bert-probe.service \
 		%D%/.dirstamp

src/data/oops-probe.service.in

@@ -1,11 +0,0 @@
-[Unit]
-Description=Telemetrics Kernel Oops Probe
-Requires=telemd.socket
-After=telemd.socket
-
-[Service]
-ExecStart=@bindir@/oopsprobe
-User=telemetry
-
-[Install]
-WantedBy=multi-user.target

src/data/pstore-probe.service.in

@@ -1,7 +1,7 @@
 [Unit]
 Description=Telemetrics Pstore Probe
-Requires=telemd.socket
-After=telemd.socket
+Requires=telemprobd.socket
+After=telemprobd.socket
 
 [Service]
 ExecStart=@bindir@/pstoreprobe

src/data/python-probe.path.in

@@ -0,0 +1,9 @@
+[Unit]
+Description=Python Probe Daemon Staging
+ConditionPathExists=!/etc/telemetrics/opt-out
+
+[Path]
+DirectoryNotEmpty=/var/lib/telemetry/python
+
+[Install]
+WantedBy=multi-user.target

src/data/python-probe.service.in

@@ -0,0 +1,13 @@
+[Unit]
+Description=Telemetrics Python Probe
+Requires=telemprobd.socket
+After=telemprobd.socket
+ConditionPathExists=!/etc/telemetrics/opt-out
+
+[Service]
+User=telemetry
+ExecStart=@bindir@/pythonprobe
+
+[Install]
+WantedBy=multi-user.target
+

src/data/telemetrics-dirs.conf.in

@@ -1,5 +1,8 @@
 d @localstatedir@/lib/telemetry 0755 telemetry telemetry -
 d @localstatedir@/spool/telemetry 0750 telemetry telemetry -
+d @localstatedir@/log/telemetry 0750 telemetry telemetry -
+d @localstatedir@/log/telemetry/records 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry 0750 telemetry telemetry -
-d @localstatedir@/cache/telemetry/oops 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry/pstore 0750 telemetry telemetry -
+d /var/lib/telemetry/python 01777 telemetry telemetry -
+d /var/tmp/telemetry 01777 telemetry telemetry -

src/data/telemetrics.conf.in

@@ -1,53 +1,69 @@
+# values represent their default state
+# copy to /etc/telemetrics/telemetrics.conf and modify for user config
 [settings]
 # ip address of destination server for record delivery
-server=https://clr.telemetry.intel.com/v2/collector
+#server=@BACKEND_ADDR@
 
-socket_path=@SOCKETDIR@/telem-0
+#socket_path=@SOCKETDIR@/telem-0
 
 # certificate file to use to validate ssl endpoint
-cainfo=
+#cainfo=
 
 # Telemetry id - post header used to group records in ingestion service,
 # which may be ingesting for more than one set of clients. Can be set
 # to any string.
-tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
+#tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
 
 # record expiry time in minutes
-record_expiry=1200
+#record_expiry=1200
 
-spool_dir=@localstatedir@/spool/telemetry
+#spool_dir=@localstatedir@/spool/telemetry
 
 # maximum size of the spool directory in KB, -1 = quota disabled.
 # The block size of the files in this directory is considered,
 # and not the actual size of the record itself.
-spool_max_size=5120
+#spool_max_size=5120
 
 # time in seconds for processing spool
-# Valid range: 120..3600. Values outside this range are clamped.
-spool_process_time=900
+# Valid range: 120..300. Values outside this range are clamped.
+#spool_process_time=120
 
 # rate limit enabled - if this is set to false then all rate-limiting disabled.
 # It is possible to disable each rate-limit individually below.
-rate_limit_enabled=true
+#rate_limit_enabled=true
 
 # rate limiting record burst limit
 # Valid Range:  0..INT_MAX, -1 = disabled.
-record_burst_limit=1000
+#record_burst_limit=1000
 
 # rate limiting record window length in minutes
 # Valid Range: 0..59
-record_window_length=15
+#record_window_length=15
 
 # rate limiting byte burst limit
 # Valid Range:  0..INT_MAX, -1 = disabled.
-byte_burst_limit=-1
+#byte_burst_limit=-1
 
 # rate limiting byte window length in minutes
 # Valid Range: 0..59
-byte_window_length=20
+#byte_window_length=20
 
 # rate limit strategy - what to do with record if rate-limiting prevents 
 # delivery over network
 # Valid stategies: spool, drop
-rate_limit_strategy=spool
+#rate_limit_strategy=spool
 
+# daemon recycling enabled - if daemon has been running for a while (2 hours),
+# has not any client nor spool data, then it exits.
+# this is to ensure that latest code runs.
+#daemon_recycling_enabled=true
+
+# record server delivery enabled - when enabled records will be delivered
+# to server otherwise records will be ignored. This configuration can be used
+# with 'record_retention_enable' configuration value to keep records local only.
+#record_server_delivery_enabled=true
+
+# record retention enabled - when enabled a copy of reported telemetry records
+# will be kept locally. This configuration combined with 'record_server_delivery_enabled'
+# value can be used to keep records local only.
+#record_retention_enabled=false

src/data/telempostd.path.in

@@ -1,5 +1,5 @@
 [Unit]
-Description=Telemetry Daemon Spool
+Description=Telemetry Post Daemon staging
 ConditionPathExists=!/etc/telemetrics/opt-out
 
 [Path]

src/data/telempostd.service.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Telemetrics Post Daemon
+ConditionPathExists=!/etc/telemetrics/opt-out
+
+[Service]
+ExecStart=@bindir@/telempostd
+WorkingDirectory=@localstatedir@/spool/telemetry
+User=telemetry
+
+[Install]
+Also=telempostd.path
+WantedBy=multi-user.target

src/data/telemprobd-update-trigger.service.in

@@ -0,0 +1,7 @@
+[Unit]
+Description=Restart Telemetrics Daemon On Update
+BindsTo=update-triggers.target
+
+[Service]
+Type=oneshot
+ExecStart=@systemctldir@/systemctl try-restart telemprobd.service

src/data/telemprobd.service.in

@@ -3,10 +3,10 @@ Description=Telemetrics Daemon
 ConditionPathExists=!/etc/telemetrics/opt-out
 
 [Service]
-ExecStart=@bindir@/telemd
+ExecStart=@bindir@/telemprobd
 WorkingDirectory=@localstatedir@/spool/telemetry
 User=telemetry
 
 [Install]
-Also=telemd.socket telemd.path
+Also=telemprobd.socket
 WantedBy=multi-user.target

src/iorecord.c

@@ -0,0 +1,146 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+
+#include "log.h"
+#include "util.h"
+#include "common.h"
+#include "iorecord.h"
+
+bool _fgets(char *s, int n, FILE *stream)
+{
+        char *i;
+
+        if (!fgets(s, n, stream)) {
+                return false;
+        }
+        if ((i = strchr(s, '\n')) != NULL) {
+                *i = '\0';
+        }
+
+        return true;
+}
+
+bool read_record(char *fullpath, char *headers[], char **body, char **cfg_file)
+{
+        int i, ret = 0;
+        bool result = false;
+        FILE *fp = NULL;
+        long offset;
+#if (LINE_MAX > PATH_MAX)
+        char line[LINE_MAX+1] = { 0 };
+#else
+        char line[PATH_MAX+1] = { 0 };
+#endif
+        struct stat buf;
+        long size;
+        uint32_t cfg_prefix = 0;
+
+        ret = stat(fullpath, &buf);
+        if (ret == -1) {
+                telem_log(LOG_ERR, "Unable to stat record %s in staging\n", fullpath);
+                return false;
+        }
+        size = buf.st_size;
+
+        fp = fopen(fullpath, "r");
+        if (fp == NULL) {
+                telem_log(LOG_ERR, "Unable to open file %s in staging\n", fullpath);
+                return false;
+        }
+
+        // First line may contain configuration file path
+        if (fread(&cfg_prefix, CFG_PREFIX_LENGTH, 1, fp) != 1) {
+                telem_log(LOG_ERR, "Error while parsing staged record configuration info.\n");
+                goto read_error;
+        }
+
+        if (cfg_prefix == CFG_PREFIX_32BIT) {
+                if (!_fgets(line, sizeof(line), fp)) {
+                        telem_log(LOG_ERR, "Error while parsing staged record [%x]\n", cfg_prefix);
+                        goto read_error;
+                }
+
+                size_t pathlen = strlen(line);
+                *cfg_file = malloc(pathlen + 1);
+                if (cfg_file == NULL) {
+                        telem_log(LOG_ERR, "Could not allocate memory for config file path\n");
+                        goto read_error;
+                }
+                strcpy (*cfg_file, line);
+                telem_debug("DEBUG: cfg_file specified: %s\n", *cfg_file);
+        } else {
+                telem_debug("DEBUG: no user cfg file specified, cfg_prefix: %08x\n", cfg_prefix);
+                *cfg_file = NULL;
+                rewind(fp);
+        }
+
+        for (i = 0; i < NUM_HEADERS; i++) {
+                const char *header_name = get_header_name(i);
+                if (!_fgets(line, sizeof(line), fp)) {
+                        telem_log(LOG_ERR, "Error while parsing staged record\n");
+                        fclose(fp);
+                        return false;
+                }
+                //Get rid of trailing newline
+                result = get_header(line, header_name, &headers[i]);
+                if (!result) {
+                        telem_log(LOG_ERR, "read_record: Incorrect"
+                                  " headers in record\n");
+                        goto read_error;
+                }
+        }
+
+        offset = ftell(fp);
+        if (offset == -1) {
+                telem_perror("Failed to get staged file offset of record payload");
+                result = false;
+                goto read_error;
+        }
+
+        size = size - offset + 1;
+        *body = malloc((size_t)size);
+        if (*body == NULL) {
+                telem_log(LOG_ERR, "Could not allocate memory for payload from staged file\n");
+                result = false;
+                goto read_error;
+        }
+        memset(*body, 0, (size_t)size);
+
+        //read rest of file
+        size_t newlen = fread(*body, sizeof(char), (size_t)size, fp);
+        if (newlen == 0) {
+                telem_perror("Error reading staged file");
+                result = false;
+                goto read_error;
+        } else {
+                result = true;
+        }
+
+read_error:
+        if (fp) {
+                fclose(fp);
+        }
+
+        return result;
+}

src/iorecord.h

@@ -0,0 +1,28 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#include <stdbool.h>
+
+/**
+ * Reads a telemetry record
+ *
+ * @param fullpath pointer to full path file name
+ * @param headers pointer to array of headers and values
+ * @param body record message content
+ *
+ * @return true if successful otherwise false
+ */
+bool read_record(char *fullpath, char *headers[], char **body, char **cfg);

src/journal/cli.c

@@ -0,0 +1,115 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <getopt.h>
+
+#include "common.h"
+#include "journal.h"
+
+static void print_usage(void)
+{
+        printf("Usage:\n");
+        printf("  telem_journal [-Vi] [-r <record_id>] [-e <event_id>] [-c <classification>] [-b <boot_id>]\n\n");
+        printf("Where:\n");
+        printf("  -r,  --record_id        Print record with specific record_id\n");
+        printf("  -e,  --event_id         Print records with specific event_id\n");
+        printf("  -c,  --classification   Print records with specific classification\n");
+        printf("  -b,  --boot_id          Print records with specific boot_id\n");
+        printf("  -i,  --include_record   Include record content if available.\n");
+        printf("                          Content only available when telemetry is configured\n");
+        printf("                          with \"record_retention_enabled=true\"\n");
+        printf("  -V,  --verbose          Verbose output\n");
+        printf("  -h,  --help             Display this help message\n");
+}
+
+int main(int argc, char **argv)
+{
+
+        int rc = EXIT_SUCCESS;
+        int count = 0;
+        bool verbose_output = false;
+        bool record = false;
+        char *boot_id = NULL;
+        char *record_id = NULL;
+        char *event_id = NULL;
+        char *classification = NULL;
+        struct TelemJournal *telem_journal = NULL;
+
+        /** opts */
+        int c;
+        int opt_index = 0;
+        struct option opts[] = {
+                { "record_id", 1, NULL, 'r' },
+                { "event_id", 1, NULL, 'e' },
+                { "classification", 1, NULL, 'c' },
+                { "boot_id", 1, NULL, 'b' },
+                { "verbose", 0, NULL, 'V' },
+                { "include_record", 0, NULL, 'i' },
+                { "help", 0, NULL, 'h' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "r:e:c:b:Vih", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'r':
+                                record_id = optarg;
+                                break;
+                        case 'e':
+                                event_id = optarg;
+                                break;
+                        case 'c':
+                                classification = optarg;
+                                break;
+                        case 'b':
+                                boot_id = optarg;
+                                break;
+                        case 'V':
+                                verbose_output = true;
+                                break;
+                        case 'i':
+                                record = true;
+                                break;
+                        case 'h':
+                                print_usage();
+                                exit(EXIT_SUCCESS);
+                        default:
+                                print_usage();
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        if ((telem_journal = open_journal(JOURNAL_PATH))) {
+                if (verbose_output) {
+                        fprintf(stdout, "%-30s %-27s %-32s %-32s %-36s\n", "Classification", "Time stamp",
+                                "Record ID", "Event ID", "Boot ID");
+                }
+                count = print_journal(telem_journal, classification, record_id, event_id, boot_id, record);
+                if (verbose_output) {
+                        fprintf(stdout, "Total records: %d\n", count);
+                }
+                close_journal(telem_journal);
+        } else {
+                fprintf(stderr, "Unable to open journal\n");
+                rc = EXIT_FAILURE;
+        }
+
+        return rc;
+}

src/journal/journal.c

@@ -0,0 +1,673 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#define ID_LEN 32
+#define BOOTID_LEN 37 // Includes the \n character at the end
+#define BOOTID_FILE "/proc/sys/kernel/random/boot_id"
+#define MID_BUFF 1024
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/uio.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include "log.h"
+#include "util.h"
+#include "common.h"
+#include "journal.h"
+
+/*
+ * Packs journal entry values into one line string separated
+ * by '\036' RS.
+ *
+ * @param entry A pointer to a JournalEntry struct with values
+ *              to serialize.
+ * @param buff A pointer to be initialized and set with the
+ *             serialized values.
+ *
+ * @return 0 on success, -1 on failure.
+ */
+static int serialize_journal_entry(struct JournalEntry *entry, char **buff)
+{
+        int rc = 0;
+
+        if (entry == NULL) {
+                return -1;
+        }
+
+        if (asprintf(buff, "%s\036%lld\036%s\036%s\036%s", entry->record_id, (long long int)entry->timestamp,
+                     entry->classification, entry->event_id, entry->boot_id) < 0) {
+                telem_log(LOG_ERR, "Error: Unable to serialize data in buff\n");
+                rc = -1;
+        }
+
+        return rc;
+}
+
+/**
+ *  Frees journal entry struct members and journal entry pointer.
+ *
+ *  @param the pointer to the entry structure to free.
+ */
+static void free_journal_entry(struct JournalEntry *entry)
+{
+        if (entry) {
+                free(entry->classification);
+                free(entry->record_id);
+                free(entry->event_id);
+                free(entry->boot_id);
+                free(entry);
+        }
+}
+
+/**
+ * Unpacks a record that was stored in file and initializes a
+ * JournalEntry struct with parsed values.
+ *
+ * @param line A pointer to the data to unpack.
+ * @param entry A pointer to be initialized and set with values
+ *        parsed from parameter line.
+ *
+ * @return 0 on success, -1 on failure.
+ */
+static int deserialize_journal_entry(char *line, struct JournalEntry **entry)
+{
+        int rc = -1;
+        size_t len = 0;
+        size_t llen = 0;
+        size_t offset = 0;
+        struct JournalEntry *e = NULL;
+
+        if (line == NULL) {
+                return -1;
+        }
+
+        e = malloc(sizeof(struct JournalEntry));
+        if (!e) {
+                return -1;
+        }
+
+        llen = strlen(line);
+
+        for (int i = 0; i < 5; i++) {
+                char *out = NULL;
+                if (offset > llen) {
+                        rc = -1;
+                        break;
+                }
+                len = strcspn(line + offset, "\036\n");
+                out = strndup((line + offset), len);
+                offset += len + 1;
+                if (out) {
+                        switch (i) {
+                                case 0:
+                                        e->record_id = out;
+                                        break;
+                                case 1:
+                                        e->timestamp = atoll(out);
+                                        free(out);
+                                        break;
+                                case 2:
+                                        e->classification = out;
+                                        break;
+                                case 3:
+                                        e->event_id = out;
+                                        break;
+                                case 4:
+                                        e->boot_id = out;
+                                        rc = 0;
+                                        break;
+                                default:
+                                        assert(i < 0 && i > 4);
+                                        free(out);
+                                        rc = 1;
+                        }
+                } else {
+                        rc = -1;
+                        break;
+                }
+        }
+
+        if (rc == 0) {
+                *entry = e;
+        } else {
+                free_journal_entry(e);
+        }
+
+        return rc;
+}
+
+/**
+ * Counts the number of records in journal.
+ *
+ * @param telem_journal Pointer to a telemetry journal
+ *        to get record count.
+ *
+ * @return record count
+ *
+ */
+static int get_record_count(struct TelemJournal *telem_journal)
+{
+        int count = 0;
+        size_t len;
+        ssize_t read;
+        char *line = NULL;
+
+        rewind(telem_journal->fptr);
+        // read until the end line by line
+        while ((read = getline(&line, &len, telem_journal->fptr)) != -1) {
+                count++;
+        }
+        free(line);
+
+        return count;
+}
+
+/**
+ * Serializes JournalEntry and saves it to file.
+ *
+ * @param fptr A FILE pointer.
+ * @param entry A pointer to data.
+ *
+ * @return 0 on succes, !=0 on failure
+ */
+static int save_entry(FILE *fptr, struct JournalEntry *entry)
+{
+        int rc = -1;
+        char *serialized_data = NULL;
+
+        if (fptr == NULL || entry == NULL) {
+                return rc;
+        }
+
+        if ((rc = serialize_journal_entry(entry, &serialized_data)) == 0) {
+                telem_debug("DEBUG: Saving: %s\n", serialized_data);
+                fprintf(fptr, "%s\n", serialized_data);
+                fflush(fptr);
+                free(serialized_data);
+        }
+
+        return rc;
+}
+
+/**
+ * Reads the boot unique identifier from BOOTID_FILE
+ *
+ * @param buff pointer to a BOOTID_LEN allocated
+ *
+ * @return 0 on success, -1 on failure
+ */
+static int read_boot_id(char buff[])
+{
+        int rc = -1;
+        FILE *fs = NULL;
+
+        fs = fopen(BOOTID_FILE, "r");
+        if (!fs) {
+                telem_log(LOG_ERR, "Error: Unable to open %s for reading: %d\n", BOOTID_FILE, errno);
+                return rc;
+        }
+
+        if (fgets(buff, BOOTID_LEN, fs)) {
+                rc = 0;
+        }
+        fclose(fs);
+
+        return rc;
+}
+
+/**
+ * Moves the file position indicator n lines.
+ *
+ * @param n Number of lines to skip.
+ * @param fptr A file pointer.
+ * @param found_record A pointer to a function to be called
+ *                     every time a valid record is found, if
+ *                     pointer is not set to null.
+ *
+ * @returns 0 on success, on failure -1 if n > existing
+ *            lines in file or errno if failure happens
+ *            during file operation.
+ */
+static int skip_n_lines(int n, FILE *fptr, int (*found_record)(char *))
+{
+        int rc = 0;
+        size_t len = 0;
+        char *line = NULL;
+        struct JournalEntry *entry = NULL;
+
+        if (fptr == NULL) {
+                telem_log(LOG_ERR, "fptr argument to skip_n_lines is invalid\n");
+                // File descriptor in bad state
+                return EBADFD;
+        }
+
+        if (fseek(fptr, 0, 0) != 0) {
+                return errno;
+        }
+
+        for (int i = 0; i < n; i++) {
+                if (getline(&line, &len, fptr) == -1) {
+                        rc = -1;
+                        break;
+                }
+                if (found_record != NULL) {
+                        deserialize_journal_entry(line, &entry);
+                        if (entry) {
+                                found_record(entry->record_id);
+                                free_journal_entry(entry);
+                                entry = NULL;
+                        }
+                }
+        }
+        free(line);
+
+        return rc;
+}
+
+/**
+ * Copies contents of fptr to a known location.
+ *
+ * @param fptr A file pointer from source file.
+ *
+ * @returns 0 on success, errno on failure
+ */
+static int copy_to_tmp(FILE *fptr, char *tmp_path)
+{
+        int rc = 0;
+        size_t len;
+        char *line = NULL;
+        FILE *fptr_tmp = NULL;
+
+        if (fptr == NULL) {
+                return -1;
+        }
+
+        fptr_tmp = fopen(tmp_path, "w");
+        if (!fptr_tmp) {
+                return errno;
+        }
+
+        // Copy lines to new file
+        while (getline(&line, &len, fptr) > -1) {
+                if (fprintf(fptr_tmp, "%s", line) < 0) {
+                        rc = errno;
+                        break;
+                }
+        }
+
+        if (fclose(fptr_tmp) != 0) {
+                rc = errno;
+                telem_perror("Error");
+        }
+        free(line);
+
+        return rc;
+}
+
+/* Exported function */
+TelemJournal *open_journal(const char *journal_file)
+{
+        FILE *fptr = NULL;
+        char boot_id[BOOTID_LEN] = { '\0' };
+        struct TelemJournal *telem_journal;
+
+        // Use default location if journal_file parameter is NULL
+        if (journal_file == NULL) {
+                fptr = fopen(JOURNAL_PATH, "a+");
+        } else {
+                fptr = fopen(journal_file, "a+");
+        }
+        if (fptr == NULL) {
+                telem_perror("Error while opening journal file");
+                return NULL;
+        }
+
+        if (read_boot_id(boot_id) != 0) {
+                telem_perror("Error while reading boot_id");
+                return NULL;
+        }
+
+        telem_journal = malloc(sizeof(struct TelemJournal));
+        if (!telem_journal) {
+                telem_log(LOG_CRIT, "CRIT: Unable to allocate memory\n");
+                return NULL;
+        }
+
+        telem_journal->fptr = fptr;
+        telem_journal->journal_file = \
+                (journal_file == NULL) ? strdup(JOURNAL_PATH) : strdup(journal_file);
+        /* boot_id includes \n at the end, strip RC during duplication */
+        telem_journal->boot_id = strndup(boot_id, BOOTID_LEN - 1);
+        telem_journal->record_count = get_record_count(telem_journal);
+        telem_journal->record_count_limit = RECORD_LIMIT;
+        telem_journal->latest_record_id = NULL;
+        telem_journal->prune_entry_callback = NULL;
+
+        telem_debug("Records in db: %d\n", telem_journal->record_count);
+
+        return telem_journal;
+}
+
+/* Exported function */
+void close_journal(TelemJournal *telem_journal)
+{
+        if (telem_journal) {
+                free(telem_journal->boot_id);
+                free(telem_journal->journal_file);
+                free(telem_journal->latest_record_id);
+                fclose(telem_journal->fptr);
+                free(telem_journal);
+        }
+}
+
+/**
+ * Checks if class is a prefix i.e. t/\* or t/t/\*
+ *
+ * @param class A pointer to string with classification
+ *              value to check.
+ * @return 1 if class is a prefix, 0 otherwise
+ */
+static int is_class_prefix(char *class)
+{
+        size_t class_len = strlen(class);
+        return (strcmp((char *)(class + class_len - 2), "/*") == 0) ? 1 : 0;
+}
+
+/**
+ * Print records content
+ *
+ * @param record_id Unique record identifier
+ *
+ */
+static void print_record(char *record_id)
+{
+        int rc = 0;
+        size_t read = 0;
+        char buff[MID_BUFF] = { '\0' };
+        char *filepath = NULL;
+        FILE *recordfp = NULL;
+
+        rc = asprintf(&filepath, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (rc == -1) {
+                // just bail out, there are worse problems
+                return;
+        }
+
+        recordfp = fopen(filepath, "r");
+        if (!recordfp) {
+                telem_log(LOG_INFO, "Could not open record %s: %s\n", record_id, strerror(errno));
+                return;
+        }
+
+        while ((read = fread(buff, 1, sizeof(buff), recordfp)) > 0) {
+                fwrite(buff, 1, read, stdout);
+        }
+
+        free(filepath);
+        fclose(recordfp);
+}
+
+/* Exported function */
+int print_journal(TelemJournal *telem_journal, char *classification,
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record)
+{
+        int n = 0;
+        int rc = 0;
+        int count = 0;
+        char str_time[80] = { '\0' };
+        char *line = NULL;
+        size_t len = 0;
+        FILE *journal_fileptr = NULL;
+        struct tm ts;
+        struct JournalEntry *entry = NULL;
+
+        if (telem_journal == NULL) {
+                return -1;
+        }
+
+        journal_fileptr = fopen(telem_journal->journal_file, "r");
+        if (!journal_fileptr) {
+                return -1;
+        }
+
+        // Advance file pointer to line n
+        n = telem_journal->record_count - telem_journal->record_count_limit;
+        if (n > 0) {
+                rc = skip_n_lines(n, journal_fileptr, NULL);
+                if (rc == -1) {
+                        return rc;
+                } else if (rc != 0) {
+                        telem_log(LOG_ERR, "An error occurred while advancing journal file: %s\n", strerror(rc));
+                }
+        }
+
+        while (getline(&line, &len, journal_fileptr) != -1) {
+                deserialize_journal_entry(line, &entry);
+                if (entry) {
+                        /* filter entry out if one is provided */
+                        if (record_id != NULL && strcmp(entry->record_id, record_id) != 0) {
+                                goto skip_print;
+                        }
+                        if (boot_id != NULL && strcmp(entry->boot_id, boot_id) != 0) {
+                                goto skip_print;
+                        }
+                        if (event_id != NULL && strcmp(entry->event_id, event_id) != 0) {
+                                goto skip_print;
+                        }
+                        // In the case of class checking prefixes is an option
+                        if (classification != NULL) {
+                                // Check prefixes when classification ends in /*, otherwise use strcomp
+                                if (is_class_prefix(classification)) {
+                                        if (strncmp(entry->classification, classification, strlen(classification) - 1) != 0) {
+                                                continue;
+                                        }
+                                } else if (strcmp(entry->classification, classification) != 0) {
+                                        continue;
+                                }
+                        }
+                        /* end filters section */
+                        ts = *localtime(&entry->timestamp);
+                        if (strftime(str_time, sizeof(str_time), "%a %Y-%m-%d %H:%M:%S %Z", &ts) == 0) {
+                                continue;
+                        }
+                        /* print record metadata */
+                        fprintf(stdout, "%-30s %s %s %s %s\n", entry->classification, str_time, entry->record_id, entry->event_id, entry->boot_id);
+                        /* print record content */
+                        if (include_record) {
+                                print_record(entry->record_id);
+                        }
+                        count++;
+                }
+skip_print:
+                free_journal_entry(entry);
+        }
+        free(line);
+        fclose(journal_fileptr);
+
+        return count;
+}
+
+/**
+ * Validation for event id. Thic check makes sure
+ * that an event_id is a 32 hexadecimal chars long.
+ *
+ * @param event_id A pointer to id value.
+ *
+ * @return 0 on success, 1 on failure
+ */
+static int validate_event_id(char *event_id)
+{
+        const char alphab[] = EVENT_ID_ALPHAB;
+
+        if (event_id == NULL) {
+                return 1;
+        }
+
+        if (strlen(event_id) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        if (strspn(event_id, alphab) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        return 0;
+}
+
+/* Exported function */
+int new_journal_entry(TelemJournal *telem_journal, char *classification,
+                      time_t timestamp, char *event_id)
+{
+        int rc = 1;
+        char boot_id[BOOTID_LEN] = { '\0' };
+        char *record_id = NULL;
+        struct JournalEntry *entry = NULL;
+
+        if (telem_journal == NULL) {
+                telem_log(LOG_ERR, "telem_journal was not initialized\n");
+                return rc;
+        }
+
+        if (validate_classification(classification) != 0) {
+                return rc;
+        }
+
+        if (validate_event_id(event_id) != 0) {
+                return rc;
+        }
+
+        entry = malloc(sizeof(struct JournalEntry));
+        if (!entry) {
+                telem_log(LOG_CRIT, "CRIT: unable to allocate memory\n");
+                return rc;
+        }
+        entry->classification = NULL;
+        entry->record_id = NULL;
+        entry->event_id = NULL;
+        entry->boot_id = NULL;
+
+        if (get_random_id(&record_id) != 0) {
+                telem_log(LOG_ERR, "Erorr: Unable to generate random id\n");
+                goto quit;
+        }
+
+        if (read_boot_id(boot_id) != 0) {
+                telem_log(LOG_ERR, "Error: Unable to read boot_id\n");
+                goto quit;
+        }
+
+        if ((entry->classification = strdup(classification)) == NULL) {
+                goto quit;
+        }
+        if ((entry->event_id = strdup(event_id)) == NULL) {
+                goto quit;
+        }
+
+        entry->record_id = record_id;
+        /* boot_id includes \n at the end, strip RC during duplication */
+        entry->boot_id = strndup(boot_id, BOOTID_LEN - 1);
+        entry->timestamp = timestamp;
+
+        if ((rc = save_entry(telem_journal->fptr, entry)) == 0) {
+                telem_journal->record_count = telem_journal->record_count + 1;
+                telem_debug("DEBUG: %d records in journal\n", telem_journal->record_count);
+        }
+
+        free(telem_journal->latest_record_id);
+        telem_journal->latest_record_id = strdup(entry->record_id);
+
+quit:
+        free_journal_entry(entry);
+
+        return rc;
+}
+
+/* Exported function */
+int prune_journal(struct TelemJournal *telem_journal, char *tmp_dir)
+{
+
+        int rc = 1;
+        int count = 0;
+        int deviation = DEVIATION;
+        char *tmp_file_path = NULL;
+
+        if (telem_journal == NULL) {
+                return rc;
+        }
+
+        if (telem_journal->record_count > (deviation + telem_journal->record_count_limit)) {
+                count = telem_journal->record_count - telem_journal->record_count_limit;
+
+                // jump to line# count
+                if ((rc = skip_n_lines(count, telem_journal->fptr, telem_journal->prune_entry_callback)) != 0) {
+                        telem_log(LOG_ERR, "Error skipping %d journal lines\n", count);
+                        // if rc == -1 (n > #lines in file, change to errno style)
+                        return (rc == -1) ? EADDRNOTAVAIL : rc;
+                }
+                // Use default if not tmp_dir is specified
+                if (tmp_dir == NULL) {
+                        if ((asprintf(&tmp_file_path, "%s/.journal", JOURNAL_TMPDIR) == -1)) {
+                                return ENOMEM;
+                        }
+                } else {
+                        if ((asprintf(&tmp_file_path, "%s/.journal", tmp_dir) == -1)) {
+                                return ENOMEM;
+                        }
+                }
+                // create new file with rest of file
+                if ((rc = copy_to_tmp(telem_journal->fptr, tmp_file_path)) != 0) {
+                        telem_log(LOG_ERR, "Error copying partial journal to temp journal file\n");
+                        goto quit;
+                }
+                // close file handler
+                if ((rc = fclose(telem_journal->fptr)) != 0) {
+                        telem_log(LOG_ERR, "Error closing journal file handler\n");
+                        goto quit;
+                }
+                // overwrite file
+                if ((rc = rename(tmp_file_path, telem_journal->journal_file)) != 0) {
+                        telem_log(LOG_ERR, "Error while overwriting journal file\n");
+                        goto quit;
+                }
+                // reopen file handler
+                telem_journal->fptr = fopen(telem_journal->journal_file, "a+");
+                if (!telem_journal->fptr) {
+                        telem_log(LOG_ERR, "Error re-opening journal file\n");
+                        return rc;
+                }
+                // update record count
+                telem_journal->record_count = telem_journal->record_count - count;
+                telem_debug("DEBUG: record_count: %d\n", telem_journal->record_count);
+        }
+        rc = 0;
+
+quit:
+        free(tmp_file_path);
+
+        return rc;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/journal/journal.h

@@ -0,0 +1,117 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+/* default record limit */
+#define RECORD_LIMIT 100
+#define DEVIATION 50
+
+#include <time.h>
+#include <stdio.h>
+#include <stdbool.h>
+
+/* Journal entry type */
+typedef struct JournalEntry {
+        time_t timestamp;
+        char *classification;
+        char *record_id;
+        char *event_id;
+        char *boot_id;
+} JournalEntry;
+
+/* Telemetry journal type */
+typedef struct TelemJournal {
+        FILE *fptr;
+        char *journal_file;
+        char *boot_id;
+        char *latest_record_id;
+        int record_count;
+        int record_count_limit;
+        int (*prune_entry_callback)(char *);
+} TelemJournal;
+
+/**
+ * Telemetry journal initialization.
+ *
+ * @param journal_file A pointer to a string containing
+ *        the full path to file used as journal storage.
+ *        If journal_file is set to NULL a default value
+ *        will be used.
+ *
+ * @returns a telemetry journal structure in success or
+ *          NULL in case of failure.
+ */
+TelemJournal *open_journal(const char *journal_file);
+
+/**
+ * Closes journal file and deallocates memory that was
+ * previously initialized by open_journal call.
+ *
+ * @param telem_journal A pointer to struct that has a
+ *         reference to journal file pointer and other
+ *         members.
+ */
+void close_journal(TelemJournal *telem_journal);
+
+/**
+ * Prints journal contents to stdout. Use function parameters
+ * to filter journal entries to print.
+ *
+ * @param telem_journal A pointer to struct initialized
+ *        by open_journal call.
+ * @param classification A pointer to string used as
+ *        classification filter.
+ * @param record_id A pointer to string used as record_id filter.
+ * @param event_id A pointer to string used as event_id filter.
+ * @param boot_id A pointer to string used as boor_id filter.
+ * @param include_record A flag to control record content print.
+ *
+ * @return the number of lines printed to stdout on success, -1
+ *         on failure.
+ */
+int print_journal(TelemJournal *telem_journal, char *classification,
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record);
+
+/**
+ * Creates a new entry in journal.
+ *
+ * @param telem_journal A pointer to telemetry journal.
+ * @param classification A pointer to a classification value.
+ * @param timestamp Record creation time stamp value.
+ * @param event_id A pointer to a unique id value for record
+ *        event.
+ *
+ * @return 0 on success 1 on failure.
+ */
+int new_journal_entry(TelemJournal *telem_journal, char *classification,
+                      time_t timestamp, char *event_id);
+
+/**
+ * Checks number of lines in log and prunes the oldest records
+ * if journal grows more than telem_journal->record_count_limit.
+ *
+ * @param telem_journal A pointer to telemetry journal struct
+ *        returned by open_journal call.
+ * @param tmp_dir A pointer to a string with the path to use for
+ *                temp file during pruning process.
+ *
+ * @return 0 on success, errno on failure
+ */
+int prune_journal(TelemJournal *telem_journal, char *tmp_dir);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/journal/local.mk

@@ -0,0 +1,9 @@
+bin_PROGRAMS += \
+	%D%/telem_journal
+
+%C%_telem_journal_SOURCES = %D%/cli.c \
+	%D%/journal.c \
+	src/util.c \
+	src/common.c
+
+# vim: filetype=automake tabstop=8 shiftwidth=8 noexpandtab

src/local.mk

@@ -1,42 +1,86 @@
-bin_PROGRAMS = %D%/telemd
+bin_PROGRAMS = \
+	%D%/telemprobd \
+	%D%/telempostd
 
-%C%_telemd_SOURCES = \
-	%D%/main.c \
+%C%_telemprobd_SOURCES = \
+	%D%/probe.c \
 	%D%/telemdaemon.c \
 	%D%/telemdaemon.h \
-	%D%/spool.c \
-	%D%/spool.h
+	%D%/journal/journal.c \
+	%D%/journal/journal.h
 
-%C%_telemd_LDADD = $(CURL_LIBS) \
-	%D%/libtelem-shared.la
+%C%_telemprobd_LDADD = $(CURL_LIBS) \
+	%D%/libtelem-shared.la \
+	%D%/libtelemetry.la
 
-%C%_telemd_CFLAGS = \
+%C%_telemprobd_CFLAGS = \
 	$(AM_CFLAGS)
 
-%C%_telemd_LDFLAGS = \
+%C%_telemprobd_LDFLAGS = \
 	$(AM_LDFLAGS) \
 	-pie
 
 if HAVE_SYSTEMD_DAEMON
-%C%_telemd_CFLAGS += \
-        $(SYSTEMD_DAEMON_CFLAGS)
-%C%_telemd_LDADD += \
-        $(SYSTEMD_DAEMON_LIBS)
+%C%_telemprobd_CFLAGS += \
+	$(SYSTEMD_DAEMON_CFLAGS)
+%C%_telemprobd_LDADD += \
+	$(SYSTEMD_DAEMON_LIBS)
 endif
 
 if LOG_SYSTEMD
 if HAVE_SYSTEMD_JOURNAL
-%C%_telemd_CFLAGS += \
-        $(SYSTEMD_JOURNAL_CFLAGS)
-%C%_telemd_LDADD += \
-        $(SYSTEMD_JOURNAL_LIBS)
+%C%_telemprobd_CFLAGS += \
+	$(SYSTEMD_JOURNAL_CFLAGS)
+%C%_telemprobd_LDADD += \
+	$(SYSTEMD_JOURNAL_LIBS)
+endif
+endif
+
+# Telemetry post daemon
+%C%_telempostd_SOURCES = \
+	%D%/post.c \
+	%D%/telempostdaemon.c \
+	%D%/telempostdaemon.h \
+	%D%/journal/journal.c \
+	%D%/journal/journal.h \
+	%D%/spool.h \
+	%D%/spool.c \
+	%D%/retention.h \
+	%D%/retention.c \
+	%D%/iorecord.c \
+	%D%/iorecord.h
+
+%C%_telempostd_LDADD = $(CURL_LIBS) \
+	%D%/libtelem-shared.la \
+	%D%/libtelemetry.la
+
+%C%_telempostd_CFLAGS = \
+	$(AM_CFLAGS)
+
+%C%_telempostd_LDFLAGS = \
+	$(AM_LDFLAGS) \
+	-pie
+
+if HAVE_SYSTEMD_DAEMON
+%C%_telempostd_CFLAGS += \
+	$(SYSTEMD_DAEMON_CFLAGS)
+%C%_telempostd_LDADD += \
+	$(SYSTEMD_DAEMON_LIBS)
+endif
+
+if LOG_SYSTEMD
+if HAVE_SYSTEMD_JOURNAL
+%C%_telempostd_CFLAGS += \
+	$(SYSTEMD_JOURNAL_CFLAGS)
+%C%_telempostd_LDADD += \
+	$(SYSTEMD_JOURNAL_LIBS)
 endif
 endif
 
 # set library version info
-SHAREDLIB_CURRENT=3
+SHAREDLIB_CURRENT=4
 SHAREDLIB_REVISION=0
-SHAREDLIB_AGE=0
+SHAREDLIB_AGE=1
 
 noinst_LTLIBRARIES = %D%/libtelem-shared.la
 
@@ -44,20 +88,19 @@ noinst_LTLIBRARIES = %D%/libtelem-shared.la
 	%D%/util.c \
 	%D%/util.h \
 	%D%/configuration.c \
+	%D%/nica/inifile.c \
+	%D%/nica/hashmap.c \
+	%D%/nica/b64enc.c \
 	%D%/configuration.h \
 	%D%/common.c \
 	%D%/common.h
 
 %C%_libtelem_shared_la_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 
 %C%_libtelem_shared_la_LDFLAGS = \
 	$(AM_LDFLAGS)
 
-%C%_libtelem_shared_la_LIBADD = \
-	$(GLIB_LIBS)
-
 lib_LTLIBRARIES = \
 	%D%/libtelemetry.la
 
@@ -69,12 +112,10 @@ include_HEADERS = %D%/telemetry.h
 noinst_HEADERS = %D%/log.h
 
 %C%_libtelemetry_la_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 
 %C%_libtelemetry_la_LDFLAGS = \
 	$(AM_LDFLAGS) \
-	$(GLIB_LIBS) \
 	-version-info $(SHAREDLIB_CURRENT):$(SHAREDLIB_REVISION):$(SHAREDLIB_AGE) \
 	-Wl,--version-script=$(top_srcdir)/src/telemetry.sym
 

src/log.h

@@ -20,6 +20,7 @@
 
 #include "config.h"
 
+#ifdef TM_LOG_SYSTEMD
 #ifdef HAVE_SYSTEMD_SD_JOURNAL_H
 
 /* Certain static analysis tools do not understand GCC's __INCLUDE_LEVEL__
@@ -32,6 +33,7 @@
 #endif
 
 #include <systemd/sd-journal.h>
+#endif
 #else
 #include <errno.h>
 #include <string.h>
@@ -40,8 +42,8 @@
 
 #ifdef DEBUG
 #define telem_debug(...) do { \
-                (telem_log(LOG_DEBUG, "%s():[%d]", __func__, __LINE__), \
-                 telem_log(LOG_DEBUG, __VA_ARGS__)); \
+                telem_log(LOG_DEBUG, "%s():[%d] ", __func__, __LINE__); \
+                telem_log(LOG_DEBUG, __VA_ARGS__); \
 } while (0);
 #else
 #define telem_debug(...) do {} while (0);
@@ -79,7 +81,7 @@
 #ifdef TM_LOG_STDERR
 #define telem_log(priority, ...) do { \
                 if (priority <= MAX_LOG_LEVEL) { \
-                        fprintf(stderr, __VA_ARGS__ ); \
+                        fprintf(stderr, __VA_ARGS__); \
                 } \
 } while (0);
 #endif
@@ -88,10 +90,10 @@
 #define telem_perror(msg) sd_journal_perror(msg)
 #endif
 #ifdef TM_LOG_SYSLOG
-#define telem_perror(msg) syslog(LOG_ERR, msg ": %s", strerror(errno))
+#define telem_perror(msg) syslog(LOG_ERR, msg ": %s\n", strerror(errno))
 #endif
 #ifdef TM_LOG_STDERR
-#define telem_perror(msg) fprintf(stderr, ": %s", msg)
+#define telem_perror(msg) fprintf(stderr, "ERROR: " msg ": %s\n", strerror(errno))
 #endif
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/nica/b64enc.c

@@ -0,0 +1,154 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdint.h>
+#include "b64enc.h"
+
+#define B64_LINE_LEN 76
+
+static int padding[] = {0, 2, 1, 0};
+static char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+
+/*
+ * base 64 encoding of 3 bytes to 4 printable characters
+ * (https://tools.ietf.org/html/rfc4648).
+ *
+ * i.e. (Reference: https://en.wikipedia.org/wiki/Base64)
+ *
+ *      [   M  ] [   a  ] [  n  ]
+ * bin  01001101 01100001 01101110
+ *
+ * b64  01001101 01100001 01101110  >> 18 & 0x3F -> 010011 -> ( 19 dec) T
+ *      ------
+ *
+ * b64  01001101 01100001 01101110  >> 12 & 0x3F -> 010110 -> ( 22 dec) W
+ *            -------
+ *
+ * b64  01001101 01100001 01101110  >>  6 & 0x3F -> 000101 -> (  5 dec) F
+ *                   -------
+ *
+ * b64  01001101 01100001 01101110  >>  0 & 0x3F -> 101110 -> (117 dec) u
+ *                          ------
+ *
+ */
+static void b64_3b(char bin[3], char *out) {
+
+    uint32_t x = (((uint32_t) bin[0] << 16) & 0xFF0000) |
+                 (((uint32_t) bin[1] <<  8) & 0xFF00)   |
+                 (((uint32_t) bin[2])       & 0xFF);
+
+    out[3] = table[x       & 0x3F];
+    out[2] = table[x >> 6  & 0x3F];
+    out[1] = table[x >> 12 & 0x3F];
+    out[0] = table[x >> 18 & 0x3F];
+
+}
+
+
+/*
+ * Provide padding to already initialized b64 output
+ */
+static void b64_padding(char *out, int pad_size) {
+
+    if (pad_size > 0) {
+        *out++ = '=';
+    }
+
+    if (pad_size > 1) {
+        *out++ = '=';
+    }
+
+    *out = '\0';
+}
+
+
+/*
+ * Encode file contents as b64 string
+ */
+int nc_b64enc_file(FILE *fh, char *buff, size_t buff_size) {
+
+    int ret_val = 1;
+    int out_count = 0;
+    int pad_size = 0;
+    char *buff_ptr = buff;
+    char bin[3] = {0, 0, 0};
+    size_t len = 0;
+
+    while ((len = fread(bin, 1, 3, fh)) > 0) {
+        b64_3b(bin, buff_ptr);
+        /*
+         Reset 2nd and 3rd bytes in case only one byte was read. If these
+         bytes are not clean b64 output will be corrupted with old bits.
+         */
+        bin[1] = bin[2] = 0;
+        /*
+         Advance pointer to the next free position of the b64 encoded
+         buffer. This happens to be len + 1 (as longs as len >= 1 and len <= 3)
+
+            len = 1 byte  -> b64 -> 2 bytes (or len + 1)
+            len = 2 bytes -> b64 -> 3 bytes (or len + 1)
+            len = 3 bytes -> b64 -> 4 bytes (or len + 1)
+        */
+        buff_ptr += (len + 1);
+        /*
+         Line break every 76 characters
+        */
+        out_count += (int) (len + 1);
+        if (out_count % B64_LINE_LEN == 0) {
+            *buff_ptr++ = '\n';
+        }
+        pad_size = padding[len];
+        /*
+         Check not to overflow buffer
+
+            buffer size <= b64 characters + padding length + null termination
+        */
+        if (buff_size <= (size_t) (buff_ptr - buff) + (size_t) pad_size + 1) {
+           ret_val = 0;
+           goto end_b64_enc;
+        }
+    }
+
+    b64_padding(buff_ptr, pad_size);
+
+end_b64_enc:
+
+    return ret_val;
+}
+
+
+int nc_b64enc_filename(const char *filename, char *buff, size_t buff_size) {
+
+    int ret = 0;
+    FILE *fh = NULL;
+
+    fh = fopen(filename, "rb");
+
+    if (fh == NULL) {
+        goto nc_b64_clean;
+    }
+
+    ret = nc_b64enc_file(fh, buff, buff_size);
+
+nc_b64_clean:
+
+    if (fh) {
+        fclose(fh);
+    }
+
+    return ret;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/nica/b64enc.h

@@ -0,0 +1,42 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * This is a chained base64 encode implementation.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include "macros.h"
+
+/*
+ * Encodes contents of file handler in base64
+ *
+ * @param fh File handler to encode as base64
+ * @param buff Output buffer where the base64 output should be written
+ * @param buff_size Size of the output buffer
+ *
+ * @returns 1 in success and 0 in failure
+ */
+_nica_public_ int nc_b64enc_file(FILE *fh, char *buff, size_t buff_size);
+
+
+/*
+ * Encodes contents of file (filename) in base64
+ *
+ * @param filename File to encode as base64
+ * @param buff Output buffer where the base64 output should be written
+ * @param buff_size Size of the output buffer
+ *
+ * @returns 1 in success and 0 in failure
+ */
+_nica_public_ int nc_b64enc_filename(const char *filename, char *buff, size_t buff_size);
+

src/nica/hashmap.c

@@ -0,0 +1,469 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "hashmap.h"
+
+#define INITIAL_SIZE 61
+
+/* When we're "full", 70% */
+#define FULL_FACTOR 0.7
+
+/* Multiple to increase bucket size by */
+#define INCREASE_FACTOR 4
+
+/**
+ * An bucket/chain within the hashmap
+ */
+typedef struct NcHashmapEntry {
+        void *hash;                  /**<The key for this item */
+        void *value;                 /**<Value for this item */
+        struct NcHashmapEntry *next; /**<Next item in the chain */
+        bool occ;                    /**<Whether this bucket is occupied */
+} NcHashmapEntry;
+
+/**
+ * A NcHashmap
+ */
+struct NcHashmap {
+        int size;                /**<Current size of the hashmap */
+        int next_size;           /**<Next size at which we need to resize */
+        int n_buckets;           /**<Current number of buckets */
+        NcHashmapEntry *buckets; /**<Stores our bucket chains */
+
+        nc_hash_create_func hash;     /**<Hash generation function */
+        nc_hash_compare_func compare; /**<Key comparison function */
+        nc_hash_free_func key_free;   /**<Cleanup function for keys */
+        nc_hash_free_func value_free; /**<Cleanup function for values */
+};
+
+/**
+ * Iteration object
+ */
+typedef struct _NcHashmapIter {
+        int bucket;     /**<Current bucket position */
+        NcHashmap *map; /**<Associated NcHashmap */
+        void *item;     /**<Current item in this iteration */
+} _NcHashmapIter;
+
+static void nc_hashmap_update_next_size(NcHashmap *self);
+static bool nc_hashmap_resize(NcHashmap *self);
+
+static inline bool nc_hashmap_maybe_resize(NcHashmap *self)
+{
+        if (!self) {
+                return false;
+        }
+        if (self->size >= self->next_size) {
+                return true;
+        }
+        return false;
+}
+
+static NcHashmap *nc_hashmap_new_internal(nc_hash_create_func create, nc_hash_compare_func compare,
+                                          nc_hash_free_func key_free, nc_hash_free_func value_free)
+{
+        NcHashmap *map = NULL;
+        NcHashmapEntry *buckets = NULL;
+
+        map = calloc(1, sizeof(NcHashmap));
+        if (!map) {
+                return NULL;
+        }
+
+        buckets = calloc(INITIAL_SIZE, sizeof(NcHashmapEntry));
+        if (!buckets) {
+                free(map);
+                return NULL;
+        }
+        map->buckets = buckets;
+        map->n_buckets = INITIAL_SIZE;
+        map->hash = create ? create : nc_simple_hash;
+        map->compare = compare ? compare : nc_simple_compare;
+        map->key_free = key_free;
+        map->value_free = value_free;
+        map->size = 0;
+
+        nc_hashmap_update_next_size(map);
+
+        return map;
+}
+
+NcHashmap *nc_hashmap_new(nc_hash_create_func create, nc_hash_compare_func compare)
+{
+        return nc_hashmap_new_internal(create, compare, NULL, NULL);
+}
+
+NcHashmap *nc_hashmap_new_full(nc_hash_create_func create, nc_hash_compare_func compare,
+                               nc_hash_free_func key_free, nc_hash_free_func value_free)
+{
+        return nc_hashmap_new_internal(create, compare, key_free, value_free);
+}
+
+static inline unsigned nc_hashmap_get_hash(NcHashmap *self, const void *key)
+{
+        unsigned hash = self->hash(key);
+        return hash;
+}
+
+static int nc_hashmap_insert_bucket(NcHashmap *self, NcHashmapEntry *buckets, int n_buckets,
+                                    unsigned hash, const void *key, void *value)
+{
+        NcHashmapEntry *row = &(buckets[hash % (unsigned)n_buckets]);
+        NcHashmapEntry *head = NULL;
+        NcHashmapEntry *parent = head = row;
+        NcHashmapEntry *tomb = NULL;
+        int ret = 1;
+
+        while (row) {
+                if (!row->occ) {
+                        tomb = row;
+                }
+                parent = row;
+                if (row->occ && self->compare(row->hash, key)) {
+                        if (self->value_free) {
+                                self->value_free(row->value);
+                        }
+                        if (self->key_free) {
+                                self->key_free(row->hash);
+                        }
+                        row->hash = (void *)key;
+                        row->value = value;
+                        return 0;
+                }
+                row = row->next;
+        }
+
+        if (tomb) {
+                row = tomb;
+        }
+
+        if (!row) {
+                row = calloc(1, sizeof(NcHashmapEntry));
+                if (!row) {
+                        return -1;
+                }
+        }
+
+        row->hash = (void *)key;
+        row->value = value;
+        row->occ = true;
+        if (parent != row && parent) {
+                parent->next = row;
+        }
+
+        return ret;
+}
+
+bool nc_hashmap_put(NcHashmap *self, const void *key, void *value)
+{
+        if (!self) {
+                return false;
+        }
+        int inc;
+
+        if (nc_hashmap_maybe_resize(self)) {
+                if (!nc_hashmap_resize(self)) {
+                        return false;
+                }
+        }
+        unsigned hash = nc_hashmap_get_hash(self, key);
+        inc = nc_hashmap_insert_bucket(self, self->buckets, self->n_buckets, hash, key, value);
+        if (inc >= 0) {
+                self->size += inc;
+                return true;
+        } else {
+                return false;
+        }
+}
+
+static NcHashmapEntry *nc_hashmap_get_entry(NcHashmap *self, const void *key)
+{
+        if (!self) {
+                return NULL;
+        }
+
+        unsigned hash = nc_hashmap_get_hash(self, key);
+        NcHashmapEntry *row = &(self->buckets[hash % (unsigned)self->n_buckets]);
+
+        while (row) {
+                if (self->compare(row->hash, key)) {
+                        return row;
+                }
+                row = row->next;
+        }
+        return NULL;
+}
+
+void *nc_hashmap_get(NcHashmap *self, const void *key)
+{
+        if (!self) {
+                return NULL;
+        }
+
+        NcHashmapEntry *row = nc_hashmap_get_entry(self, key);
+        if (row) {
+                return row->value;
+        }
+        return NULL;
+}
+
+static bool nc_hashmap_remove_internal(NcHashmap *self, const void *key, bool remove)
+{
+        if (!self) {
+                return false;
+        }
+        NcHashmapEntry *row = nc_hashmap_get_entry(self, key);
+
+        if (!row) {
+                return false;
+        }
+
+        if (remove) {
+                if (self->key_free) {
+                        self->key_free(row->hash);
+                }
+                if (self->value_free) {
+                        self->value_free(row->value);
+                }
+        }
+        self->size -= 1;
+        row->hash = NULL;
+        row->value = NULL;
+        row->occ = false;
+
+        return true;
+}
+
+bool nc_hashmap_steal(NcHashmap *self, const void *key)
+{
+        return nc_hashmap_remove_internal(self, key, false);
+}
+
+bool nc_hashmap_remove(NcHashmap *self, const void *key)
+{
+        return nc_hashmap_remove_internal(self, key, true);
+}
+
+bool nc_hashmap_contains(NcHashmap *self, const void *key)
+{
+        return (nc_hashmap_get(self, key)) != NULL;
+}
+
+static inline void nc_hashmap_free_bucket(NcHashmap *self, NcHashmapEntry *bucket, bool nuke)
+{
+        if (!self) {
+                return;
+        }
+        NcHashmapEntry *tmp = bucket;
+        NcHashmapEntry *bk = bucket;
+        NcHashmapEntry *root = bucket;
+
+        while (tmp) {
+                bk = NULL;
+                if (tmp->next) {
+                        bk = tmp->next;
+                }
+
+                if (nuke && tmp->occ) {
+                        if (self->key_free) {
+                                self->key_free(tmp->hash);
+                        }
+                        if (self->value_free) {
+                                self->value_free(tmp->value);
+                        }
+                }
+                if (tmp != root) {
+                        free(tmp);
+                }
+                tmp = bk;
+        }
+}
+
+void nc_hashmap_free(NcHashmap *self)
+{
+        if (!self) {
+                return;
+        }
+        for (int i = 0; i < self->n_buckets; i++) {
+                NcHashmapEntry *row = &(self->buckets[i]);
+                nc_hashmap_free_bucket(self, row, true);
+        }
+        if (self->buckets) {
+                free(self->buckets);
+        }
+
+        free(self);
+}
+
+static void nc_hashmap_update_next_size(NcHashmap *self)
+{
+        if (!self) {
+                return;
+        }
+        self->next_size = (int)(self->n_buckets * FULL_FACTOR);
+}
+
+int nc_hashmap_size(NcHashmap *self)
+{
+        if (!self) {
+                return -1;
+        }
+        return self->size;
+}
+
+static bool nc_hashmap_resize(NcHashmap *self)
+{
+        if (!self || !self->buckets) {
+                return false;
+        }
+
+        NcHashmapEntry *old_buckets = self->buckets;
+        NcHashmapEntry *new_buckets = NULL;
+        NcHashmapEntry *entry = NULL;
+        int incr;
+
+        int old_size, new_size;
+        int items = 0;
+
+        new_size = old_size = self->n_buckets;
+        new_size *= INCREASE_FACTOR;
+
+        new_buckets = calloc((size_t)new_size, sizeof(NcHashmapEntry));
+        if (!new_buckets) {
+                return false;
+        }
+
+        for (int i = 0; i < old_size; i++) {
+                entry = &(old_buckets[i]);
+                while (entry) {
+                        if (entry->occ) {
+                                unsigned hash = nc_hashmap_get_hash(self, entry->hash);
+                                if ((incr = nc_hashmap_insert_bucket(self,
+                                                                     new_buckets,
+                                                                     new_size,
+                                                                     hash,
+                                                                     entry->hash,
+                                                                     entry->value)) > 0) {
+                                        items += incr;
+                                } else {
+                                        /* Likely a memory issue */
+                                        goto failure;
+                                }
+                        }
+                        entry = entry->next;
+                }
+        }
+        /* Successfully resized - do this separately because we need to
+         * guarantee old data is preserved */
+        for (int i = 0; i < old_size; i++) {
+                nc_hashmap_free_bucket(self, &(old_buckets[i]), false);
+        }
+
+        free(old_buckets);
+        self->n_buckets = new_size;
+        self->size = items;
+        self->buckets = new_buckets;
+
+        nc_hashmap_update_next_size(self);
+        return true;
+
+failure:
+        for (int i = 0; i < new_size; i++) {
+                nc_hashmap_free_bucket(self, &(new_buckets[i]), true);
+        }
+        free(new_buckets);
+        return false;
+}
+
+void nc_hashmap_iter_init(NcHashmap *map, NcHashmapIter *citer)
+{
+        _NcHashmapIter *iter = NULL;
+        if (!map || !citer) {
+                return;
+        }
+        iter = (_NcHashmapIter *)citer;
+        _NcHashmapIter it = {
+                .bucket = -1, .map = map, .item = NULL,
+        };
+        *iter = it;
+}
+
+bool nc_hashmap_iter_next(NcHashmapIter *citer, void **key, void **value)
+{
+        _NcHashmapIter *iter = NULL;
+        NcHashmapEntry *item = NULL;
+        NcHashmap *map = NULL;
+        int n_buckets = 0;
+
+        if (!citer) {
+                return false;
+        }
+
+        iter = (_NcHashmapIter *)citer;
+        map = iter->map;
+        if (!map) {
+                return false;
+        }
+        n_buckets = map->n_buckets;
+        item = iter->item;
+
+        for (;;) {
+                if (iter->bucket >= n_buckets) {
+                        if (item && !item->next) {
+                                return false;
+                        }
+                }
+                if (!item) {
+                        iter->bucket++;
+                        if (iter->bucket > n_buckets - 1) {
+                                return false;
+                        }
+                        item = &(map->buckets[iter->bucket]);
+                }
+                if (item) {
+                        if (item->occ) {
+                                goto success;
+                        }
+                        item = item->next;
+                }
+        }
+        return false;
+
+success:
+        iter->item = item->next;
+        if (key) {
+                *key = item->hash;
+        }
+        if (value) {
+                *value = item->value;
+        }
+
+        return true;
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/hashmap.h

@@ -0,0 +1,241 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ *
+ * This is a chained Hashmap implementation. It focuses on being clean
+ * and efficient, and is comparable (at -O2) to open addressing hashmaps
+ * up until around 105,000 elements, where open addressing will begin
+ * to come out in front. At 1 million elements, open addressing is a clear
+ * winner, however currently we only need a maximum of 70k elements in
+ * our implementation.
+ *
+ * Given the memory required to even begin to deal with 1 million elements,
+ * it becomes very questionable whether a hashmap should have even been
+ * used in the first place (mmap'd db?)
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "macros.h"
+#include "util.h"
+
+/* Convert between uint and void* */
+#define NC_HASH_KEY(x) ((void *)((uintptr_t)(x)))
+#define NC_HASH_VALUE(x) NC_HASH_KEY(x)
+#define NC_UNHASH_KEY(x) ((unsigned int)((uintptr_t)(x)))
+#define NC_UNHASH_VALUE(x) NC_UNHASH_KEY(x)
+
+typedef struct NcHashmap NcHashmap;
+
+/**
+ * Iteration object
+ */
+typedef struct NcHashmapIter {
+        int n0;
+        void *n1;
+        void *n2;
+} NcHashmapIter;
+
+/**
+ * Hash comparison function definition
+ *
+ * @param l First value to compare
+ * @param r Second value to compare
+ *
+ * @return true if l and r both match, otherwise false
+ */
+typedef bool (*nc_hash_compare_func)(const void *l, const void *r);
+
+/**
+ * Hash creation function definition
+ *
+ * @param key Key to generate a hash for
+ *
+ * @return an unsigned integer hash result
+ */
+typedef unsigned (*nc_hash_create_func)(const void *key);
+
+/**
+ * Callback definition to free keys and values
+ *
+ * @param p Single-depth pointer to either a key or value that should be freed
+ */
+typedef void (*nc_hash_free_func)(void *p);
+
+/**
+ * Default hash/comparison functions
+ *
+ * @note These are only used for comparison of *keys*, not values. Unless
+ * explicitly using string keys, you should most likely stick with the default
+ * nc_simple_hash and nc_simple_compare functions
+ */
+
+/* Default string hash */
+static inline unsigned nc_string_hash(const void *key)
+{
+        unsigned hash = 5381;
+        const signed char *c;
+
+        /* DJB's hash function */
+        for (c = key; *c != '\0'; c++) {
+                hash = (hash << 5) + hash + (unsigned)*c;
+        }
+        return hash;
+}
+
+/**
+ * Trivial pointer->uint hash
+ */
+static inline unsigned nc_simple_hash(const void *source)
+{
+        return NC_UNHASH_KEY(source);
+}
+
+/**
+ * Comparison of string keys
+ */
+static inline bool nc_string_compare(const void *l, const void *r)
+{
+        if (!l || !r) {
+                return false;
+        }
+        return (strcmp(l, r) == 0);
+}
+
+/**
+ * Trivial pointer comparison
+ */
+static inline bool nc_simple_compare(const void *l, const void *r)
+{
+        return (l == r);
+}
+
+/**
+ * Create a new NcHashmap
+ *
+ * @param hash Hash creation function
+ * @param compare Key comparison function
+ *
+ * @return A newly allocated NcHashmap
+ */
+_nica_public_ NcHashmap *nc_hashmap_new(nc_hash_create_func hash, nc_hash_compare_func compare);
+
+/**
+ * Create a new NcHashmap with cleanup functions
+ *
+ * @param hash Hash creation function
+ * @param compare Key comparison function
+ * @param key_free Function to free keys when removed/destroyed
+ * @param value_free Function to free values when removed/destroyed
+ *
+ * @return A newly allocated NcHashmap
+ */
+_nica_public_ NcHashmap *nc_hashmap_new_full(nc_hash_create_func hash, nc_hash_compare_func compare,
+                                             nc_hash_free_func key_free,
+                                             nc_hash_free_func value_free);
+
+/**
+ * Store a key/value pair in the hashmap
+ *
+ * @note This will displace duplicate keys, and may free both the key
+ * and value if key_free and value_free are non null
+ *
+ * @param key Key to store in the hashmap
+ * @param value Value to be associated with the key
+ *
+ * @return true if the operation succeeded.
+ */
+_nica_public_ bool nc_hashmap_put(NcHashmap *map, const void *key, void *value);
+
+/**
+ * Get the value associated with the unique key
+ *
+ * @param key Unique key to obtain a value for
+ * @return The associated value if it exists, otherwise NULL
+ */
+_nica_public_ void *nc_hashmap_get(NcHashmap *map, const void *key);
+
+/**
+ * Determine if the key has an associated value in the NcHashmap
+ *
+ * @param key Unique key to check value for
+ * @return True if the key exists in the hashmap
+ */
+_nica_public_ bool nc_hashmap_contains(NcHashmap *self, const void *key);
+
+/**
+ * Free the given NcHashmap, and all keys/values if appropriate
+ */
+_nica_public_ void nc_hashmap_free(NcHashmap *map);
+
+/**
+ * Remove the value and key identified by key.
+ *
+ * @note If key_free or value_free are non-NULL, they will be invoked
+ * for both the key and value being removed
+ *
+ * @param key The unique key to remove
+ * @return true if the key/value pair were removed
+ */
+_nica_public_ bool nc_hashmap_remove(NcHashmap *map, const void *key);
+
+/**
+ * Remove the value and key identified by key without freeing them
+ *
+ * @return true if the key/value pair were stolen
+ */
+_nica_public_ bool nc_hashmap_steal(NcHashmap *map, const void *key);
+
+/**
+ * Return the current size of the hashmap
+ *
+ * @return size of the current hashmap (element count)
+ */
+_nica_public_ int nc_hashmap_size(NcHashmap *map);
+
+/**
+ * Initialise a NcHashmapIter for iteration purposes
+ *
+ * @note The iter *must* be re-inited to re-use, or if it becomes
+ * exhausted from a previous iteration run
+ *
+ * @param iter pointer to a NcHashmapIter
+ */
+_nica_public_ void nc_hashmap_iter_init(NcHashmap *map, NcHashmapIter *iter);
+
+/**
+ * Iterate every key/value pair in the hashmap
+ *
+ * @param iter A correctly initialised NcHashmapIter
+ * @param key Pointers to store the key in, may be NULL to skip
+ * @param value Pointer to store the value in, may be NULL to skip
+ *
+ * @return true if it's possible to iterate
+ */
+_nica_public_ bool nc_hashmap_iter_next(NcHashmapIter *iter, void **key, void **value);
+
+DEF_AUTOFREE(NcHashmap, nc_hashmap_free)
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/inifile.c

@@ -0,0 +1,386 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "hashmap.h"
+#include "inifile.h"
+
+/**
+ * Mapping of @NcIniError to static strings
+ */
+static const char *_errors[] =
+    {[NC_INI_ERROR_FILE] = "",
+     [NC_INI_ERROR_EMPTY_KEY] = "Encountered empty key",
+     [NC_INI_ERROR_NOT_CLOSED] = "Expected closing \']\' for section",
+     [NC_INI_ERROR_NO_SECTION] = "Encountered key=value mapping without a valid section",
+     [NC_INI_ERROR_INVALID_LINE] = "Expected key=value notation",
+     [NC_INI_ERROR_INTERNAL] = "A fatal internal error was encountered" };
+
+const char *nc_ini_error(enum NcIniError error)
+{
+        int j = abs(error);
+        if (j < NC_INI_ERROR_FILE || j >= NC_INI_ERROR_MAX) {
+                return "[Unknown Error]";
+        }
+        return _errors[j];
+}
+
+/**
+ * Strip leading whitespace from string (left)
+ */
+static char *lstrip(char *str, ssize_t len, ssize_t *out_len)
+{
+        ssize_t skip_len = 0;
+
+        for (ssize_t i = 0; i < len; i++) {
+                if (str[i] == ' ' || str[i] == '\t') {
+                        ++skip_len;
+                        continue;
+                }
+                break;
+        }
+        *out_len = len - skip_len;
+        if (skip_len > 0) {
+                char *c = strdup(str + skip_len);
+                free(str);
+                return c;
+        }
+        return str + skip_len;
+}
+
+/**
+ * Strip trailing whitespace from string (right)
+ */
+static char *rstrip(char *str, ssize_t len, ssize_t *out_len)
+{
+        ssize_t skip_len = 0;
+
+        for (ssize_t i = len; i > 0; i--) {
+                if (str[i] == ' ' || str[i] == '\t') {
+                        ++skip_len;
+                        continue;
+                } else if (str[i] == '\0') {
+                        continue;
+                }
+                break;
+        }
+        *out_len = len - skip_len;
+        if (len == skip_len) {
+                return str;
+        }
+        str[(len - skip_len)] = '\0';
+        return str;
+}
+
+/**
+ * Chew both ends of a null terminated string of whitespace
+ * If the left side is whitespace padded, this will always result
+ * in a new allocation due to fast-forwarding the pointer. Keep
+ * this in mind.
+ */
+static char *string_chew_terminated(char *inp)
+{
+        int skip_offset = 0;
+        int len = 0;
+        int end_len = 0;
+        bool count = true;
+
+        if (!inp) {
+                return NULL;
+        }
+
+        for (char *c = inp; *c; c++) {
+                if (count && (*c == ' ' || *c == '\t')) {
+                        ++skip_offset;
+                        continue;
+                } else {
+                        count = false;
+                }
+                ++len;
+        }
+        for (int i = len; i > skip_offset; i--) {
+                if (inp[i] == ' ' || inp[i] == '\t') {
+                        ++end_len;
+                        continue;
+                } else if (inp[i] == '\0') {
+                        continue;
+                }
+                break;
+        }
+
+        if (end_len > 0) {
+                inp[(len - end_len)] = '\0';
+        }
+        if (skip_offset > 0) {
+                char *c = strdup(inp + skip_offset);
+                free(inp);
+                return c;
+        }
+        return inp;
+}
+
+/**
+ * Munch both ends of the string
+ */
+static char *string_strip(char *str, ssize_t len, ssize_t *out_len)
+{
+        ssize_t mylen = len;
+
+        char *c = lstrip(str, len, &mylen);
+        c = rstrip(c, mylen, &mylen);
+        if (out_len) {
+                *out_len = mylen;
+        }
+        return c;
+}
+
+/* eat \s */
+static char *string_unescape(char *str)
+{
+	char *c, *c1, *c2, *c3;;
+
+	if (!str)
+		return NULL;
+
+	c = calloc(strlen(str) + 1, 1);
+	c1 = c;
+
+	c2 = str;
+	while (c2 && *c2 != 0) {
+		c3 = c2 + 1;
+		if (*c2 == '\\') {
+			if (*c3 == 's') {
+				*c = ' ';
+				c++;
+				c2 += 2;
+			} else {
+				c2++;
+			}
+		} else {
+			*c = *c2;
+			c++;
+			c2++;
+		}
+	}
+
+	free(str);	
+        return c1;
+}
+
+NcHashmap *nc_ini_file_parse(const char *path)
+{
+        NcHashmap *ret = NULL;
+        int error_line = 0;
+        int r = 0;
+
+        r = nc_ini_file_parse_full(path, &ret, &error_line);
+        if (r != 0) {
+                if (abs(r) == NC_INI_ERROR_FILE) {
+                        fprintf(stderr, "[inifile] %s: %s\n", strerror(errno), path);
+                } else {
+                        fprintf(stderr,
+                                "[inifile] %s [L%d]: %s\n",
+                                nc_ini_error(r),
+                                error_line,
+                                path);
+                }
+                return NULL;
+        }
+        return ret;
+}
+
+int nc_ini_file_parse_full(const char *path, NcHashmap **out_map, int *error_line_number)
+{
+        autofree(FILE) *file = NULL;
+        char *buf = NULL;
+        ssize_t r = 0;
+        size_t sn = 0;
+        int line_count = 1;
+        char *current_section = NULL;
+        NcHashmap *root_map = NULL;
+        NcHashmap *section_map = NULL;
+        bool failed = false;
+        int err_ret = 0;
+
+        file = fopen(path, "r");
+        if (!file) {
+                return -(NC_INI_ERROR_FILE);
+        }
+
+        if (!out_map) {
+                fprintf(stderr, "nc_ini_file_parse_full(): NcHashmap pointer invalid\n");
+                return -1;
+        }
+
+        root_map = nc_hashmap_new_full(nc_string_hash,
+                                       nc_string_compare,
+                                       free,
+                                       (nc_hash_free_func)nc_hashmap_free);
+
+        while ((r = getline(&buf, &sn, file)) != -1) {
+                char *ch = NULL;
+                ssize_t str_len = r;
+                char *key = NULL;
+                char *value = NULL;
+
+                /* Fix newline */
+                if (buf[r - 1] == '\n') {
+                        buf[r - 1] = '\0';
+                        --r;
+                }
+                str_len = r;
+
+                buf = string_strip(buf, r, &str_len);
+                /* Empty lines are fine */
+                if (streq(buf, "")) {
+                        goto next;
+                }
+
+                if (buf[0] == '[') {
+                        /* Validate section start */
+                        if (buf[str_len - 1] != ']') {
+                                /* Throw error */
+                                err_ret = NC_INI_ERROR_NOT_CLOSED;
+                                goto fail;
+                        }
+                        /* Grab the section name, and "close" last section */
+                        buf[str_len - 1] = '\0';
+                        if (current_section) {
+                                free(current_section);
+                        }
+                        current_section = strdup(buf + 1);
+                        section_map = nc_hashmap_get(root_map, current_section);
+                        if (!section_map) {
+                                /* Create a new section dynamically */
+                                section_map = nc_hashmap_new_full(nc_string_hash,
+                                                                  nc_string_compare,
+                                                                  free,
+                                                                  free);
+                                nc_hashmap_put(root_map, strdup(current_section), section_map);
+                        }
+                        goto next;
+                } else if (buf[0] == '#' || buf[0] == ';') {
+                        /* Skip comment */
+                        goto next;
+                }
+
+                /* Look for key = value */
+                ch = strchr(buf, '=');
+                if (!ch) {
+                        /* Throw error */
+                        err_ret = NC_INI_ERROR_INVALID_LINE;
+                        goto fail;
+                }
+
+                /* Can't have sectionless k->v */
+                if (!current_section) {
+                        err_ret = NC_INI_ERROR_NO_SECTION;
+                        goto fail;
+                }
+
+                long int offset = ch - buf;
+
+                /* Grab the key->value from this assignment line */
+                value = strdup((buf + offset) + 1);
+                buf[offset] = '\0';
+                key = strdup(buf);
+                key = string_chew_terminated(key);
+                value = string_chew_terminated(value);
+		value = string_unescape(value);
+
+                if (streq(key, "")) {
+                        err_ret = NC_INI_ERROR_EMPTY_KEY;
+                        free(key);
+                        key = NULL;
+                        free(value);
+                        value = NULL;
+                        goto fail;
+                }
+
+                /* Ensure a section mapping exists */
+                section_map = nc_hashmap_get(root_map, current_section);
+                if (!section_map) {
+                        err_ret = NC_INI_ERROR_INTERNAL;
+                        fprintf(stderr,
+                                "[inifile] Fatal! No section map for named "
+                                "section: %s\n",
+                                current_section);
+                        goto fail;
+                }
+
+                /* Insert these guys into the map */
+                if (!nc_hashmap_put(section_map, key, value)) {
+                        err_ret = NC_INI_ERROR_INTERNAL;
+                        fprintf(stderr, "[inifile] Fatal! Out of memory\n");
+                        goto fail;
+                }
+        /* Progression + cleanup */
+        next:
+                if (buf) {
+                        free(buf);
+                        buf = NULL;
+                }
+                ++line_count;
+                sn = 0;
+                continue;
+        /* Parsing error, bail */
+        fail:
+                failed = true;
+                if (error_line_number) {
+                        *error_line_number = line_count;
+                }
+                if (key) {
+                        free(key);
+                }
+                if (value) {
+                        free(value);
+                }
+                break;
+        }
+
+        if (buf) {
+                free(buf);
+                buf = NULL;
+        }
+
+        if (current_section) {
+                free(current_section);
+                current_section = NULL;
+        }
+
+        if (failed) {
+                if (root_map) {
+                        nc_hashmap_free(root_map);
+                }
+                return -(err_ret);
+        }
+        *out_map = root_map;
+        return 0;
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/inifile.h

@@ -0,0 +1,114 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#include "hashmap.h"
+#include "macros.h"
+#include "util.h"
+
+/**
+ * The Nica INI Parser focuses on simplicity and ease of use, therefore it
+ * currently only supports a read-only parse operation.
+ *
+ * The parse functions are used to return a _root_ @NcHashmap which contains
+ * name to section mappings. Each section is in turn an @NcHashmap with string
+ * to string mappings.
+ *
+ * Consider the following example:
+ *
+ *      [Person]
+ *      alive = true
+ *      ; This person is alive
+ *
+ * The returned @NcHashmap will contain the key "Person", which corresponds to
+ * another @NcHashmap. This @NcHashmap will contain the key "alive", which has
+ * the value "true". Therefore we can access this variable using the @NcHashmap
+ * API:
+ *
+ *      char *alive = nc_hashmap_get(nc_hashmap_get(ini, "Person"), "alive");
+ *
+ * Conversely, we can use the API to determine whether a section or key is set:
+ *
+ *      if (nc_hashmap_contains(ini, "Person"))
+ *
+ * As @NcHashmap returns NULL in nc_hashmap_get, you can happily pass NULL
+ * keys, allowing quick key fetches without checking if the section exists.
+ *
+ * In our INI file, lines beginning with ";" or "#" are considered as comments
+ * and are not processed. A line following the "key = value" notation is an
+ * _assignment_. "[Section]" is considered to be a section in the INI file, and
+ * all INI files must have at least one section.
+ *
+ * We do allow duplication section definitions, by following a merge policy. If
+ * a key is redefined then the original key->value mapping is freed and the new
+ * key->value mapping is inserted.
+ *
+ * The parser will return a correctly configured @NcHashmap, the only cleanup
+ * required is to call nc_hashmap_free on it. Alternatively, use the autofree
+ * helper, for a RAII approach:
+ *
+ *      autofree(NcHashmap) *map = nc_ini_file_parse("myfile.ini");
+ */
+
+/**
+ * Errors that are reported when parsing an ini file
+ */
+enum NcIniError {
+        NC_INI_ERROR_MIN = 0,
+        NC_INI_ERROR_FILE,         /**< File based error, check strerror(errno) */
+        NC_INI_ERROR_EMPTY_KEY,    /**< Empty key in an assignment line */
+        NC_INI_ERROR_NOT_CLOSED,   /**< Encountered section start that wasn't closed with a ']' */
+        NC_INI_ERROR_NO_SECTION,   /**< Key assignment with no defined sections */
+        NC_INI_ERROR_INVALID_LINE, /**< Encountered an invalid line (syntax) */
+        NC_INI_ERROR_INTERNAL,     /**< Fatal internal error encountered. Caller should exit */
+        NC_INI_ERROR_MAX
+};
+/**
+
+ * Convenience wrapper for nc_ini_file_parse_full, reports errors to stderrr
+ * and returns an NcHashmap if the parsing succeeded
+ */
+_nica_public_ NcHashmap *nc_ini_file_parse(const char *path);
+
+/**
+ * Parse an INI file into a hash of hashes.
+ *
+ * @param path Path on the filesystem to parse, must be in INI syntax.
+ * @param out_map Pointer to store the resulting root NcHashmap in
+ * @param error_line_number If not null, then the erronous line number is
+ * stored.
+ *
+ * @return 0 if the call was successful, or a negative integer. See nc_ini_error
+ */
+_nica_public_ int nc_ini_file_parse_full(const char *path, NcHashmap **out_map,
+                                         int *error_line_number);
+
+/**
+ * Return a string representation for a given @NcIniError
+ *
+ * @param error Valid NcIniError code
+ * @return a static string owned by the implementation
+ */
+_nica_public_ const char *nc_ini_error(enum NcIniError error);
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/local.mk

@@ -0,0 +1,6 @@
+noinst_HEADERS += \
+	%D%/hashmap.h \
+	%D%/inifile.h \
+	%D%/macros.h \
+	%D%/nc-string.h \
+	%D%/util.h

src/nica/macros.h

@@ -0,0 +1,29 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#define _nica_public_ __attribute__((visibility("default")))
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/nc-string.c

@@ -0,0 +1,144 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "nc-string.h"
+
+nc_string *nc_string_dup(const char *str)
+{
+        if (!str) {
+                return NULL;
+        }
+        struct nc_string_t *st = calloc(1, sizeof(struct nc_string_t));
+        if (!st) {
+                return NULL;
+        }
+        st->len = asprintf(&st->str, "%s", str);
+        if (st->len < 0 || !st->str) {
+                free(st);
+                return NULL;
+        }
+        return st;
+}
+
+nc_string *nc_string_dup_printf(const char *ptn, ...)
+{
+        if (!ptn) {
+                return NULL;
+        }
+
+        struct nc_string_t *st = calloc(1, sizeof(struct nc_string_t));
+        if (!st) {
+                return NULL;
+        }
+
+        va_list va;
+        va_start(va, ptn);
+
+        st->len = vasprintf(&st->str, ptn, va);
+        if (st->len < 0 || !st->str) {
+                free(st);
+                st = NULL;
+                goto end;
+        }
+end:
+        va_end(va);
+
+        return st;
+}
+
+nc_string *nc_string_append_printf(nc_string *st, const char *ptn, ...)
+{
+	char *newstr;
+	char *newstr2;
+	int ret;
+
+        if (!ptn) {
+                return NULL;
+        }
+
+        va_list va;
+        va_start(va, ptn);
+
+        ret = vasprintf(&newstr, ptn, va);
+	if (ret <= 0)
+		newstr = strdup("");
+
+	st->len = asprintf(&newstr2, "%s%s", st->str, newstr);
+	free(st->str);
+	st->str = newstr2;
+	free(newstr);
+        va_end(va);
+
+        return st;
+}
+
+
+bool nc_string_cat(nc_string *s, const char *append)
+{
+        char *p = NULL;
+        int len = 0;
+
+        if (!s || !append) {
+                return false;
+        }
+        if (!s->str) {
+                return false;
+        }
+        len = asprintf(&p, "%s%s", s->str, append);
+        if (!p || len < s->len) {
+                return false;
+        }
+        free(s->str);
+        s->str = p;
+        s->len = len;
+        return true;
+}
+
+bool nc_string_prepend(nc_string *s, const char *prepend)
+{
+        char *p = NULL;
+        int len = 0;
+
+        if (!s || !prepend) {
+                return false;
+        }
+        if (!s->str) {
+                return false;
+        }
+        len = asprintf(&p, "%s%s", prepend, s->str);
+        if (!p || len < s->len) {
+                return false;
+        }
+        free(s->str);
+        s->str = p;
+        s->len = len;
+        return true;
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/nc-string.h

@@ -0,0 +1,220 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "macros.h"
+
+/**
+ * Safely represent and store a buffer as a string
+ */
+typedef struct nc_string_t {
+        char *str; /**<Buffer holding a NUL-terminated string */
+        int len;   /**<Current length of the string */
+} nc_string;
+
+/**
+ * Construct a new string using printf style syntax
+ *
+ * @param format Printf-style format string
+ * @param ... Variable arguments
+ *
+ * @return a new nc_string if allocation succeeded, otherwise NULL
+ */
+_nica_public_ nc_string *nc_string_dup_printf(const char *format, ...)
+    __attribute__((format(printf, 1, 2)));
+
+
+_nica_public_ nc_string *nc_string_append_printf(nc_string *st, const char *ptn, ...)
+	__attribute__((format(printf, 2, 3)));
+
+/**
+ * Duplicate a string into a new NUL-terminated nc_string
+ *
+ * @param str Pointer to a string
+ * @return a new nc_string if allocation succeeded, otherwise NULL
+ */
+_nica_public_ nc_string *nc_string_dup(const char *str);
+
+/**
+ * Free an existing string and its contents
+ *
+ * @param str Pointer to an existing string
+ */
+static inline void nc_string_free(nc_string *str)
+{
+        if (!str) {
+                return;
+        }
+        if (str->str) {
+                free(str->str);
+        }
+        free(str);
+}
+
+/**
+ * Append the contents of 'append' into the given nc_string
+ *
+ * @param str Pointer to a valid nc_string
+ * @param append Text to append into the nc_string
+ *
+ * @return a boolean value indicating success
+ */
+_nica_public_ bool nc_string_cat(nc_string *str, const char *append);
+
+_nica_public_ bool nc_string_prepend(nc_string *s, const char *prepend);
+
+
+/**
+ * Determine if string A is equal to string B
+ *
+ * @note This function will not check beyond the length of string A
+ *
+ * @param a string to check
+ * @param b string to check against
+ * @return a boolean value, true if the strings match, otherwise false
+ */
+static inline bool nc_string_equal(nc_string *a, nc_string *b)
+{
+        if (!a || !b) {
+                return false;
+        }
+        if (!a->str || !b->str) {
+                return false;
+        }
+        return strncmp(a->str, b->str, (size_t)a->len) == 0;
+}
+
+/**
+ * Determine if string A is equal to string B
+ *
+ * @note This function will not check beyond the length of string A
+ *
+ * @param a string to check
+ * @param b const char* string to check against
+ * @return a boolean value, true if the strings match, otherwise false
+ */
+static inline bool nc_string_const_equal(nc_string *a, const char *b)
+{
+        if (!a || !b) {
+                return false;
+        }
+        if (!a->str) {
+                return false;
+        }
+        return strncmp(a->str, b, (size_t)a->len) == 0;
+}
+
+/**
+ * Convenience wrapper to obtain string length
+ *
+ * @note This will assert the string is not NULL, ensuring a termination.
+ *
+ * @return length of the given string
+ */
+static inline int ncstrlen(nc_string *str)
+{
+        assert(str != NULL);
+        return str->len;
+}
+
+/**
+ * Determine if a string has the given suffix
+ *
+ * @param str Valid nc_string
+ * @param suffix A suffix to check
+ * @param len Length of the suffix
+ *
+ * @return True if @str has the given suffix
+ */
+static inline bool nc_string_has_suffix_const(nc_string *str, char *suffix, ssize_t len)
+{
+        if (!str || !suffix) {
+                return false;
+        }
+        if (len > str->len || len <= 0) {
+                return false;
+        }
+        return (strncmp(str->str + (str->len - (len)), suffix, (size_t)len) == 0);
+}
+
+/**
+ * Determine if a string has the given prefix
+ *
+ * @param str Valid nc_string
+ * @param prefix A prefix to check
+ * @param len Length of the prefix
+ *
+ * @return True if @str has the given prefix
+ */
+static inline bool nc_string_has_prefix_const(nc_string *str, char *prefix, ssize_t len)
+{
+        if (!str || !prefix) {
+                return false;
+        }
+        if (len > str->len || len <= 0) {
+                return false;
+        }
+        return (strncmp(str->str, prefix, (size_t)len) == 0);
+}
+
+/**
+ * Perform inplace modification of a strip to return
+ * a stripped pointer, that is, without start and end whitespace.
+ *
+ * This does not modify the length of the string.
+ */
+static inline char *nc_string_strip(nc_string *str)
+{
+        int i;
+
+        if (!str || !str->str) {
+                return NULL;
+        }
+        for (i = 0; i < str->len; i++) {
+                if (str->str[i] != ' ') {
+                        break;
+                }
+        }
+        for (int j = str->len - 1; j > i; j--) {
+                if (str->str[j] != ' ') {
+                        str->str[j + 1] = '\0';
+                        break;
+                }
+        }
+
+        return str->str + i;
+}
+/**
+ * To be used only with compile time constants.
+ */
+#define nc_string_has_suffix(a, suff) nc_string_has_suffix_const(a, suff, (sizeof(suff) - 1))
+#define nc_string_has_prefix(a, pref) nc_string_has_prefix_const(a, pref, (sizeof(pref) - 1))
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/nica/util.h

@@ -0,0 +1,69 @@
+/*
+ * This file is part of libnica.
+ *
+ * Copyright © 2016-2017 Intel Corporation
+ *
+ * libnica is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1
+ * of the License, or (at your option) any later version.
+ */
+
+#pragma once
+
+#define _GNU_SOURCE
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <nica/macros.h>
+
+#define DEF_AUTOFREE(N, C)                                                                         \
+        static inline void _autofree_func_##N(void *p)                                             \
+        {                                                                                          \
+                if (p && *(N **)p) {                                                               \
+                        C(*(N **)p);                                                               \
+                        (*(void **)p) = NULL;                                                      \
+                }                                                                                  \
+        }
+
+#define autofree(N) __attribute__((cleanup(_autofree_func_##N))) N
+
+/**
+ * Dump any leaked file descriptors
+ */
+_nica_public_ void nc_dump_file_descriptor_leaks(void);
+
+_nica_public_ void *greedy_realloc(void **p, size_t *allocated, size_t need);
+
+/**
+ * Test string equality of two null terminated strings
+ */
+__attribute__((always_inline)) static inline bool streq(const char *a, const char *b)
+{
+        if (a == b) {
+                return true;
+        }
+        if (!a || !b) {
+                return false;
+        }
+        return strcmp(a, b) == 0;
+}
+
+DEF_AUTOFREE(char, free)
+DEF_AUTOFREE(FILE, fclose)
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 8
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=8 tabstop=8 expandtab:
+ * :indentSize=8:tabSize=8:noTabs=true:
+ */

src/post.c

@@ -0,0 +1,97 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <unistd.h>
+#include <stdbool.h>
+#include <sys/stat.h>
+
+#include "telemetry.h"
+#include "log.h"
+#include "spool.h"
+#include "configuration.h"
+#include "telempostdaemon.h"
+
+/*
+ *  Using a function pointer for unit testing to isolate the call to actual post function.
+ *  The call to post the record to the server is stubbed out in the unit tests
+ *  using pointer to a fake function.
+ */
+
+bool (*post_record_ptr)(char *[], char *, char *) = post_record_http;
+
+void print_usage(char *prog)
+{
+        printf("%s: Usage\n", prog);
+        printf("  -f,  --config_file    Configuration file. This overides the other parameters\n");
+        printf("  -h,  --help           Display this help message\n");
+        printf("  -V,  --version        Print the program version\n");
+}
+
+int main(int argc, char **argv)
+{
+
+        int c;
+        int opt_index = 0;
+        TelemPostDaemon daemon;
+
+        struct option opts[] = {
+                { "config_file", 1, NULL, 'f' },
+                { "help", 0, NULL, 'h' },
+                { "version", 0, NULL, 'V' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "f:hV", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case 'h':
+                                print_usage(argv[0]);
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                print_usage(argv[0]);
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        initialize_post_daemon(&daemon);
+
+        daemon.current_spool_size = get_spool_dir_size();
+
+        /* When path activated this will process
+         * the activating message or previously
+         * spooled data */
+        staging_records_loop(&daemon);
+
+        /* This function is blocking, it will
+        * block until a signal is received */
+        run_daemon(&daemon);
+
+        close_daemon(&daemon);
+
+        return 0;
+}

src/probe.c

@@ -31,6 +31,7 @@
 #include <signal.h>
 #include <malloc.h>
 
+#include "telemetry.h"
 #include "config.h"
 #include "common.h"
 #ifdef  HAVE_SYSTEMD_SD_DAEMON_H
@@ -49,14 +50,6 @@
 #include "log.h"
 #include "telemdaemon.h"
 #include "configuration.h"
-#include "spool.h"
-
-/*
- *  Using a function pointer for unit testing to isolate the call to actual post function.
- *  The call to post the record to the server is stubbed out in the unit tests
- *  using pointer to a fake function.
- */
-bool (*post_record_ptr)(char *[], char *, bool) = post_record_http;
 
 void print_usage(char *prog)
 {
@@ -76,7 +69,6 @@ int main(int argc, char **argv)
         client *cl = NULL;
         client *current_client = NULL;
         int c;
-        char *config_file = NULL;
         int opt_index = 0;
         sigset_t mask;
         //bool interrupted = false;
@@ -94,18 +86,12 @@ int main(int argc, char **argv)
 
         while ((c = getopt_long(argc, argv, "f:hV", opts, &opt_index)) != -1) {
                 switch (c) {
-                        case 'f':
-                                config_file = optarg;
-                                struct stat buf;
-                                ret = stat(config_file, &buf);
-
-                                /* check if the file exists and is a regular file */
-                                if (ret == -1 || !S_ISREG(buf.st_mode)) {
-                                        telem_log(LOG_ERR, "Configuration file"
+                         case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
                                                   " path not valid\n");
-                                        exit(EXIT_FAILURE);
+                                    exit(EXIT_FAILURE);
                                 }
-                                set_config_file(config_file);
                                 break;
                         case 'h':
                                 print_usage(argv[0]);
@@ -119,7 +105,7 @@ int main(int argc, char **argv)
                                 exit(EXIT_FAILURE);
                 }
         }
-        initialize_daemon(&daemon);
+        initialize_probe_daemon(&daemon);
 
         sigemptyset(&mask);
 
@@ -220,17 +206,9 @@ int main(int argc, char **argv)
 
         telem_log(LOG_INFO, "Listening on socket...\n");
 
-        /* Calculate the spool directory size */
-        if (is_spool_valid()) {
-                daemon.is_spool_valid = true;
-                daemon.current_spool_size = get_spool_dir_size();
-        }
-
+        bool daemon_recycling_enabled = daemon_recycling_enabled_config();
         int spool_process_time = spool_process_time_config();
-
-        spool_records_loop(&(daemon.current_spool_size));
-        time_t last_spool_run_time = time(NULL);
-        time_t last_daemon_start_time = time(NULL);
+        time_t last_record_received = time(NULL);
 
         ret = update_machine_id();
         if (ret == -1) {
@@ -265,8 +243,8 @@ int main(int argc, char **argv)
                                         }
 
                                         if (fdsi.ssi_signo == SIGTERM || fdsi.ssi_signo == SIGINT) {
-                                                telem_log(LOG_INFO, "Received either a \
-                                                                     SIGINT/SIGTERM signal\n");
+                                                telem_log(LOG_INFO, "Received either a "
+                                                                     "SIGINT/SIGTERM signal\n");
                                                 goto clean_exit;
                                         }
 
@@ -274,12 +252,6 @@ int main(int argc, char **argv)
                                                 telem_log(LOG_INFO, "Received a SIGHUP signal\n");
                                                 /* reload configuration file */
                                                 reload_config();
-
-                                                if (is_spool_valid()) {
-                                                        daemon.is_spool_valid = true;
-                                                        //calc_spool_dir_size();
-                                                        daemon.current_spool_size = get_spool_dir_size();
-                                                }
                                         }
                                 }
 
@@ -331,25 +303,20 @@ int main(int argc, char **argv)
                                         }
                                         assert(current_client);
                                         handle_client(&daemon, i, current_client);
+                                        last_record_received = time(NULL);
                                 }
                         }
                 } else {
                         time_t now = time(NULL);
-                        /* If spool is empty and time to recycle the daemon has elapsed*/
-                        if ((daemon.current_spool_size == 0) &&
-                            difftime(now, last_daemon_start_time) >= TM_DAEMON_EXIT_TIME) {
+                        /* time to recycle the daemon has elapsed*/
+                        if (daemon_recycling_enabled &&
+                            difftime(now, last_record_received) >= TM_DAEMON_EXIT_TIME) {
                                 /* Exit */
                                 telem_log(LOG_INFO, "Daemon exiting for recycling\n");
                                 goto clean_exit;
                         }
                 }
-
                 time_t now = time(NULL);
-                if (difftime(now, last_spool_run_time) >= spool_process_time) {
-                        spool_records_loop(&(daemon.current_spool_size));
-                        last_spool_run_time = time(NULL);
-                }
-
                 if (difftime(now, last_refresh_time) >= TM_REFRESH_RATE) {
                         int ret = update_machine_id();
                         if (ret == -1) {
@@ -360,12 +327,13 @@ int main(int argc, char **argv)
         }
 
 clean_exit:
+
         /* Free memory before exiting */
         while ((cl = LIST_FIRST(&(daemon.client_head))) != NULL) {
                 remove_client(&(daemon.client_head), cl);
         }
         free(daemon.pollfds);
-
+        free(daemon.machine_id_override);
         if (LIST_EMPTY(&(daemon.client_head))) {
                 telem_log(LOG_INFO, "Client list cleared\n");
         }

src/probes/README.md

@@ -0,0 +1,55 @@
+# Probes
+
+The default probes provided along the telemetry client code are:
+
+* bertprobe: this probe reports on the Boot Error Region Table if such
+  entry is found in ```/sys/firmware/acpi/tables/BERT```.
+
+* crashprobe: This probe processes core dump files. It can be registered as
+  the kernel core file handler in /proc/sys/kernel/core_pattern.
+
+* hprobe: a simple probe that sends a keep alive message. This probe can also
+  be useful for testing
+
+* journalprobe: a probe that monitors the systemd journal for log messages
+  from failed services.
+
+* klogscanner: a probe to collect 'oops messages' when the kernel detects a
+  problem.
+
+* pstoreprobe: probe to collect messages left on pstore filesystem.
+
+* pythonprobe: a probe that monitors Python generated traceback files.
+
+* telem-record-gen: this is a "general-purpose probe" for sending custom
+  records on-the-fly. This tool can be used stand alone or as a part of a
+  script to implement a probe.
+
+For example to generate an hprobe like message we can execute the following
+command:
+
+```
+$ telem-record-gen --class org.example/hello/world --payload hello
+```
+
+for more options use the ```--help``` switch i.e.:
+
+```
+$ telem-record-gen --help
+Usage:
+  telem-record-gen [OPTIONS] - create and send a custom telemetry record
+
+Help Options:
+  -h, --help            Show help options
+
+Application Options:
+  -f, --config-file     Path to configuration file (not implemented yet)
+  -V, --version         Print the program version
+  -s, --severity        Severity level (1-4) - (default 1)
+  -c, --class           Classification level_1/level_2/level_3
+  -p, --payload         Record body (max size = 8k)
+  -P, --payload-file    File to read payload from
+  -R, --record-version  Version number for format of payload (default 1)
+  -e, --event-id        Event id to use in the record
+
+```

src/probes/bert_probe.c

@@ -0,0 +1,117 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <getopt.h>
+#include "common.h"
+#include "telemetry.h"
+#include "nica/b64enc.h"
+
+#include "config.h"
+#include "log.h"
+
+static const char bert_record_file[] = "/sys/firmware/acpi/tables/data/BERT";
+static const char telem_record_class[] = "org.clearlinux/bert/debug";
+static uint32_t severity = 2;
+static uint32_t payload_version = 1;
+
+void print_usage(char *prog)
+{
+        printf("%s: Usage\n", prog);
+        printf("  -f,  --config_file    Specify a configuration file other than default\n");
+        printf("  -h,  --help           Display this help message\n");
+        printf("  -V,  --version        Print the program version\n");
+}
+
+int main(int argc, char **argv)
+{
+        struct telem_ref *tm_handle = NULL;
+        char *classification = (char *)telem_record_class;
+        char *payload;
+        int ret;
+
+        // Following vars are for arg parsing.
+        int c;
+        int opt_index = 0;
+        struct option opts[] = {
+                { "config_file", 1, NULL, 'f' },
+                { "help", 0, NULL, 'h' },
+                { "version", 0, NULL, 'V' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        while ((c = getopt_long(argc, argv, "f:hV", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 'h':
+                                print_usage(argv[0]);
+                                exit(EXIT_SUCCESS);
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        payload = calloc(sizeof(char), MAX_PAYLOAD_LENGTH);
+        if (!payload) {
+                printf("Unable to allocate more memory.\n");
+                return -ENOMEM;
+        }
+
+        if (!(ret = nc_b64enc_filename(bert_record_file, payload, MAX_PAYLOAD_LENGTH))) {
+                printf("Failed to read payload from: %s\n", bert_record_file);
+                ret = EXIT_FAILURE;
+                goto done;
+        }
+
+        if ((ret = tm_create_record(&tm_handle, severity, classification,
+                                    payload_version)) < 0) {
+                printf("Failed to create record: %s\n", strerror(-ret));
+                goto done;
+        }
+
+        if ((ret = tm_set_payload(tm_handle, payload)) < 0) {
+                printf("Failed to set record payload: %s\n", strerror(-ret));
+                goto done;
+        }
+
+        if ((ret = tm_send_record(tm_handle)) < 0) {
+                printf("Failed to send record to daemon: %s\n", strerror(-ret));
+                goto done;
+        }
+
+        ret = EXIT_SUCCESS;
+done:
+        free(payload);
+        tm_free_record(tm_handle);
+        tm_handle = NULL;
+
+        return ret;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/crash_probe.c

@@ -17,6 +17,7 @@
 #define _GNU_SOURCE
 #include <assert.h>
 #include <fcntl.h>
+#include <getopt.h>
 #include <grp.h>
 #include <inttypes.h>
 #include <limits.h>
@@ -31,13 +32,13 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
-#include <glib.h>
-
 #include <libelf.h>
 #include <elfutils/libdwfl.h>
 
+#include "nica/nc-string.h"
 #include "config.h"
 #include "log.h"
+#include "probe.h"
 #include "telemetry.h"
 
 static Dwfl *d_core = NULL;
@@ -53,10 +54,10 @@ static Dwfl *d_core = NULL;
 static char *debuginfo_path = "-/usr/lib/debug";
 
 static unsigned int frame_counter = 0;
-static gchar *proc_name = NULL;
+static char *proc_name = NULL;
 static pid_t core_for_pid = 0;
-static GString *header = NULL;
-static gchar *errorstr = NULL;
+static nc_string *header = NULL;
+static char *errorstr = NULL;
 
 static uint32_t unknown_severity = 2;
 static uint32_t default_severity = 3;
@@ -67,6 +68,8 @@ static char clr_build_class[40] = "org.clearlinux/crash/clr-build";
 static char error_class[30] = "org.clearlinux/crash/error";
 static char unknown_class[30] = "org.clearlinux/crash/unknown";
 
+static char temp_core[] = "/tmp/corefile-XXXXXX";
+
 static const Dwfl_Callbacks cb =
 {
         .find_elf = dwfl_build_id_find_elf,
@@ -84,9 +87,17 @@ static inline void tm_dwfl_err(const char *msg)
         telem_log(LOG_ERR, "%s: %s\n", msg, dwfl_errmsg(-1));
 }
 
-static gchar *replace_exclamations(gchar *str)
+static char *replace_exclamations(char *str)
 {
-        return g_strdelimit(str, "!", '/');
+        char *c;
+        while (1) {
+                c = strchr(str, '!');
+                if (!c) {
+                        break;
+                }
+                *c = '/';
+        }
+        return str;
 }
 
 static void drop_privs(void)
@@ -138,23 +149,14 @@ static int temp_core_file(void)
         int tmp;
         ssize_t ret;
 
-        char core[PATH_MAX] = "/tmp/corefile-XXXXXX";
-
         /* mkstemp() opens the file with O_EXCL and 0600 permissions, so no need
          * to change umask or manipulate the fd to meet those requirements.
          */
-        if ((tmp = mkstemp(core)) < 0) {
+        if ((tmp = mkstemp(temp_core)) < 0) {
                 telem_perror("Failed to create temp core file");
                 return -1;
         }
 
-#ifndef DEBUG
-        if (unlink(core) < 0) {
-                telem_perror("Failed to unlink temp core file");
-                return -1;
-        }
-#endif
-
         while (true) {
                 // Use Linux-specific splice(2) here;
                 // simplifies copying data from pipe->file
@@ -176,6 +178,55 @@ static int temp_core_file(void)
         return tmp;
 }
 
+/* There are a number of initialization routines required to initialize the Elf
+ * and Dwfl objects used by libdwfl to later process a core file. The argument
+ * e_core is the address of an Elf pointer declared by the caller, and core_fd
+ * is the open file descriptor for the core file.
+ */
+static int prepare_corefile(Elf **e_core, int core_fd)
+{
+        // Cleanup previous corefile processing if needed
+        if (d_core) {
+                dwfl_end(d_core);
+        }
+        if (*e_core) {
+                elf_end(*e_core);
+        }
+
+        // Boilerplate initialization for elfutils (libdwfl)
+        if (!(*e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
+                tm_elf_err("Failed to get file descriptor for ELF core file");
+                goto fail;
+        }
+
+        if (!(d_core = dwfl_begin(&cb))) {
+                tm_dwfl_err("Failed to start new libdwfl session");
+                goto fail;
+        }
+
+        int core_modules = 0;
+        core_modules = dwfl_core_file_report(d_core, *e_core, NULL);
+        if (core_modules == -1) {
+                tm_dwfl_err("Failed to report modules for ELF core file");
+                goto fail;
+        }
+
+        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
+                tm_dwfl_err("Failed to finish reporting modules");
+                goto fail;
+        }
+
+        if ((core_for_pid = dwfl_core_file_attach(d_core, *e_core)) < 0) {
+                tm_dwfl_err("Failed to prepare libdwfl session for thread"
+                            " iteration");
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
 /* This callback is invoked for every frame in a thread. From a Dwfl_Frame, we
  * are able to extract the program counter (PC), and from that, the procedure
  * name via a Dwfl_Module.
@@ -185,15 +236,20 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
         Dwarf_Addr pc;
         Dwarf_Addr pc_adjusted;
         Dwfl_Module *module;
+        Dwfl_Line *line;
         const char *procname;
         const char *modname;
         bool activation;
-        GString **bt = (GString **)userdata;
+        nc_string **bt = (nc_string **)userdata;
 
         if (!dwfl_frame_pc(frame, &pc, &activation)) {
-                errorstr = g_strdup_printf("Failed to find program counter for"
-                                           " current frame: %s\n",
-                                           dwfl_errmsg(-1));
+                int ret;
+                ret = asprintf(&errorstr, "Failed to find program counter for"
+                               " current frame: %s\n",
+                               dwfl_errmsg(-1));
+                if (ret < 0) {
+                        return DWARF_CB_ABORT;
+                }
                 return DWARF_CB_ABORT;
         }
 
@@ -210,8 +266,8 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
         module = dwfl_addrmodule(d_core, pc_adjusted);
 
         if (!module) {
-                errorstr = g_strdup("Failed to find module for current"
-                                    " frame\n");
+                errorstr = strdup("Failed to find module for current"
+                                  " frame\n");
                 return DWARF_CB_ABORT;
         }
 
@@ -219,41 +275,54 @@ static int frame_cb(Dwfl_Frame *frame, void *userdata)
                                    NULL);
         procname = dwfl_module_addrname(module, pc_adjusted);
 
+        line = dwfl_module_getsrc(module, pc_adjusted);
+
         if (procname && modname) {
-                g_string_append_printf(*bt, "#%u %s() - [%s]\n",
-                                       frame_counter++, procname, modname);
+                nc_string_append_printf(*bt, "#%u %s() - [%s]",
+                                        frame_counter++, procname, modname);
         } else if (modname) {
-                g_string_append_printf(*bt, "#%u ??? - [%s]\n",
-                                       frame_counter++, modname);
+                nc_string_append_printf(*bt, "#%u ??? - [%s]",
+                                        frame_counter++, modname);
         } else {
                 // TODO: decide on "no symbol" representation
-                g_string_append_printf(*bt, "#%u (no symbols)\n",
-                                       frame_counter++);
+                nc_string_append_printf(*bt, "#%u (no symbols)",
+                                        frame_counter++);
         }
 
+        if (line) {
+                const char *src;
+                int lineno, linecol;
+                src =  dwfl_lineinfo(line, &pc_adjusted, &lineno, &linecol, NULL, NULL);
+                if (src) {
+                        nc_string_append_printf(*bt, " - %s:%i", src, lineno);
+                }
+        }
+        nc_string_append_printf(*bt, "\n");
         return DWARF_CB_OK;
 }
 
 static int thread_cb(Dwfl_Thread *thread, void *userdata)
 {
         int ret;
-        GString **bt = (GString **)userdata;
+        nc_string **bt = (nc_string **)userdata;
         pid_t tid;
 
         tid = dwfl_thread_tid(thread);
 
-        g_string_append_printf(*bt, "\nBacktrace (TID %u):\n",
-                               (unsigned int)tid);
+        nc_string_append_printf(*bt, "\nBacktrace (TID %u):\n",
+                                (unsigned int)tid);
 
         ret = dwfl_thread_getframes(thread, frame_cb, userdata);
 
         switch (ret) {
                 case -1:
-                        errorstr = g_strdup_printf("Error while iterating"
-                                                   " through frames for thread"
-                                                   " %u: %s\n",
-                                                   (unsigned int)tid,
-                                                   dwfl_errmsg(-1));
+                        if (asprintf(&errorstr, "Error while iterating"
+                                     " through frames for thread"
+                                     " %u: %s\n",
+                                     (unsigned int)tid,
+                                     dwfl_errmsg(-1)) < 0) {
+                                errorstr = NULL;
+                        }
                         return DWARF_CB_ABORT;
                 case DWARF_CB_ABORT:
                         /* already set the error string in frame_cb */
@@ -269,14 +338,14 @@ static int thread_cb(Dwfl_Thread *thread, void *userdata)
         frame_counter = 0;
 
 #if 0
-        g_string_append_printf(*bt, "\nRegisters (TID %u):\nTODO\n", current,
-                               (unsigned int)tid);
+        nc_string_append_printf(*bt, "\nRegisters (TID %u):\nTODO\n", current,
+                                (unsigned int)tid);
 #endif
 
         return DWARF_CB_OK;
 }
 
-static bool send_data(GString **backtrace, uint32_t severity, char *class)
+static bool send_data(nc_string **backtrace, uint32_t severity, char *class)
 {
         struct telem_ref *handle = NULL;
         int ret;
@@ -305,80 +374,144 @@ fail:
         return false;
 }
 
-static bool in_clr_build(gchar *fullpath)
+/* This is the entry point for libdwfl to unwind the backtrace from the core
+ * file. All data necessary for processing is referenced by d_core (a *Dwfl),
+ * which must be initialized prior to calling this function.  The callback
+ * function for processing each core thread is passed as the second argument to
+ * dwfl_getthreads(). The backtrace argument is the address of a pointer to a
+ * nc_string object declared by the caller which stores the backtrace data as a
+ * string.
+ */
+static int process_corefile(nc_string **backtrace)
 {
+        if (*backtrace) {
+                nc_string_free(*backtrace);
+        }
+        *backtrace = nc_string_dup("");
+
+        if (dwfl_getthreads(d_core, thread_cb, backtrace) != DWARF_CB_OK) {
+                /* When errors occur during the unwinding, we reach this point.
+                 * If an error string is set for the particular error, send an
+                 * "error" record to capture at least a partial backtrace if
+                 * some frames were processed.
+                 */
+                if (errorstr != NULL) {
+                        telem_log(LOG_ERR, "%s", errorstr);
+                        nc_string_append_printf(header, "Error: %s", errorstr);
+                        nc_string_prepend(*backtrace, header->str);
+                        send_data(backtrace, error_severity, error_class);
+                }
+                goto fail;
+        }
+
+        return 0;
+fail:
+        return -1;
+}
+
+static bool startswith(const char *full, const char *prefix)
+{
+        while (*prefix) {
+                if (*prefix != *full) {
+                        return false;
+                }
+                full++;
+                prefix++;
+        }
+        return true;
+}
+
+static bool in_clr_build(char *fullpath)
+{
+        // Global override for privacy filters
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                return false;
+        }
+
         /*
          * The build environment for Clear Linux packages is set up by 'mock',
          * and the chroot in which rpmbuild builds the packages has this prefix.
          */
-        if (g_regex_match_simple("^[!/]builddir[!/]build[!/]BUILD[!/]",
-                                 fullpath, 0, 0) == TRUE) {
+        if (startswith(fullpath, "/builddir/build/BUILD/") ||
+            startswith(fullpath, "!builddir!build!BUILD!")) {
                 return true;
         }
 
         return false;
 }
 
-static bool filter_binaries(gchar *fullpath)
+static bool is_banned_path(char *fullpath)
 {
-        // Anything outside of /usr/, or in /usr/local/, we consider third-party
-        if (g_regex_match_simple("^[!/]usr[!/]", fullpath, 0, 0) == FALSE) {
-                return false;
-        } else if (g_regex_match_simple("^[!/]usr[!/]local[!/]", fullpath,
-                                        0, 0) == TRUE) {
+        // Global override for privacy filters
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
                 return false;
         }
 
-        return true;
+        // Anything outside of /usr/, or in /usr/local/, we consider third-party
+
+        if (!startswith(fullpath, "/usr/") &&
+            !startswith(fullpath, "!usr!")) {
+                return true;
+        }
+
+        if (startswith(fullpath, "/usr/local/") ||
+            startswith(fullpath, "!usr!local!")) {
+                return true;
+        }
+
+        return false;
 }
 
-static gchar *config_file = NULL;
-static gchar *core_file = NULL;
-static gchar *proc_path = NULL;
-static gint signal_num = -1;
-static gboolean version_p = FALSE;
-static gboolean verbose = FALSE;
+static char *core_file = NULL;
+static char *proc_path = NULL;
+static long int signal_num = -1;
+static bool verbose = false;
 
-static GOptionEntry options[] = {
-        { "config-file", 'f', 0, G_OPTION_ARG_FILENAME, &config_file,
-          "Path to configuration file (not implemented yet)", NULL },
-        { "core-file", 'c', 0, G_OPTION_ARG_FILENAME, &core_file,
-          "Path to core file to process", NULL },
-        { "process-name", 'p', 0, G_OPTION_ARG_STRING, &proc_name,
-          "Name of process for crash report (required)", NULL },
-        { "process-path", 'E', 0, G_OPTION_ARG_STRING, &proc_path,
-          "Absolute path of crashed process, with ! or / delimiters", NULL },
-        { "signal", 's', 0, G_OPTION_ARG_INT, &signal_num,
-          "Signal number that crashed the process", NULL },
-        { "version", 'V', 0, G_OPTION_ARG_NONE, &version_p,
-          "Print the program version", NULL },
-        { "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose,
-          "Print the crash probe payload to stdout", NULL },
-        { NULL }
+static const struct option prog_opts[] = {
+        { "help", no_argument, 0, 'h' },
+        { "config-file", required_argument, 0, 'f' },
+        { "core-file", required_argument, 0, 'c' },
+        { "process-name", required_argument, 0, 'p' },
+        { "process-path", required_argument, 0, 'E' },
+        { "signal", required_argument, 0, 's' },
+        { "version", no_argument, 0, 'V' },
+        { "verbose", no_argument, 0, 'v' },
+        { 0, 0, 0, 0 }
 };
 
-static void free_glib_strings(void)
+static void print_help(void)
 {
-        if (core_file) {
-                g_free(core_file);
-        }
-        if (proc_name) {
-                g_free(proc_name);
-        }
-        if (proc_path) {
-                g_free(proc_path);
-        }
+        printf("Usage:\n");
+        printf("  crashprobe [OPTIONS] - collect data from core files\n");
+        printf("\n");
+        printf("Help Options:\n");
+        printf("  -h, --help            Show help options\n");
+        printf("\n");
+        printf("Application Options:\n");
+        printf("  -f, --config_file     Specify a configuration file other than default\n");
+        printf("  -c, --core-file       Path to core file to process\n");
+        printf("  -p, --process-name    Name of process for crash report (required)\n");
+        printf("  -E, --process-path    Absolute path of crashed process, with ! or / delimiters\n");
+        printf("  -s, --signal          Signal number that crashed the process\n");
+        printf("  -V, --version         Print the program version\n");
+        printf("  -v, --verbose         Print the crash payload to stdout\n");
+        printf("\n");
 }
 
 int main(int argc, char **argv)
 {
         Elf *e_core = NULL;
-        int core_modules = 0;
         int ret = EXIT_FAILURE;
         int core_fd = STDIN_FILENO;
-        GString *backtrace = NULL;
-        GError *error = NULL;
-        GOptionContext *context;
+        nc_string *backtrace = NULL;
+
+        if (fcntl(STDERR_FILENO, F_GETFL) < 0) {
+                // redirect stderr to avoid bad things to happen with
+                // fd 2 when launched from kernel core pattern
+                if (freopen("/dev/null", "w", stderr) == NULL) {
+                        exit(EXIT_FAILURE);
+                }
+        }
 
         // Since this program handles core files, make sure it does not produce
         // core files itself, or else it will be invoked endlessly...
@@ -386,46 +519,58 @@ int main(int argc, char **argv)
 
         drop_privs();
 
-        context = g_option_context_new("- collect data from core files");
-        g_option_context_add_main_entries(context, options, NULL);
-        g_option_context_set_translate_func(context, NULL, NULL, NULL);
-        if (!g_option_context_parse(context, &argc, &argv, &error)) {
-                g_print("Failed to parse options: %s\n", error->message);
-                exit(EXIT_FAILURE);
-        }
+        int opt;
 
-        if (version_p) {
-                g_print(PACKAGE_VERSION "\n");
-                exit(EXIT_SUCCESS);
+        while ((opt = getopt_long(argc, argv, "hf:c:p:E:s:Vv", prog_opts, NULL)) != -1) {
+                switch (opt) {
+                        case 'h':
+                                print_help();
+                                goto success;
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                goto success;
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 'c':
+                                core_file = strdup(optarg);
+                                break;
+                        case 'p':
+                                proc_name = strdup(optarg);
+                                break;
+                        case 'E':
+                                proc_path = strdup(optarg);
+                                break;
+                        case 's':
+                                errno = 0;
+                                char *endptr = NULL;
+
+                                signal_num = strtol(optarg, &endptr, 10);
+
+                                if (errno != 0) {
+                                        telem_perror("Failed to convert signal number");
+                                        goto fail;
+                                }
+                                if (endptr && *endptr != '\0') {
+                                        telem_log(LOG_ERR, "Invalid signal number. Must be an integer\n");
+                                        goto fail;
+                                }
+                                break;
+                        case 'v':
+                                verbose = true;
+                                break;
+                }
         }
 
         if (!proc_name) {
-                g_print("Missing required -p option. See --help output\n");
+                printf("Missing required -p option. See --help output\n");
                 exit(EXIT_FAILURE);
         }
 
-        if (proc_path && in_clr_build(proc_path)) {
-                telem_log(LOG_NOTICE, "Ignoring core (from mock build)\n");
-
-                backtrace = g_string_new("Crash from Clear package build\n");
-
-                if (!send_data(&backtrace, unknown_severity, clr_build_class)) {
-                        goto fail;
-                }
-                goto success;
-        }
-
-        if (proc_path && !filter_binaries(proc_path)) {
-                telem_log(LOG_NOTICE, "Ignoring core (third-party binary)\n");
-
-                backtrace = g_string_new("Crash from third party\n");
-
-                if (!send_data(&backtrace, unknown_severity, unknown_class)) {
-                        goto fail;
-                }
-                goto success;
-        }
-
         if (core_file) {
                 core_fd = open(core_file, O_RDONLY);
                 if (core_fd == -1) {
@@ -452,62 +597,66 @@ int main(int argc, char **argv)
                                 goto fail;
                         }
                 } else {
-                        g_print("Cannot process core file. Use the -c option,"
-                                " or pass the core file on stdin.\n");
+                        printf("Cannot process core file. Use the -c option,"
+                               " or pass the core file on stdin.\n");
                         goto fail;
                 }
         }
 
+        if (proc_path && in_clr_build(proc_path)) {
+                telem_log(LOG_NOTICE, "Ignoring core (from mock build)\n");
+
+                backtrace = nc_string_dup("Crash from Clear package build\n");
+
+                if (!send_data(&backtrace, unknown_severity, clr_build_class)) {
+                        goto fail;
+                }
+                goto success;
+        }
+
+        if (proc_path && is_banned_path(proc_path)) {
+                telem_log(LOG_NOTICE, "Ignoring core (third-party binary)\n");
+
+                backtrace = nc_string_dup("Crash from third party\n");
+
+                if (!send_data(&backtrace, unknown_severity, unknown_class)) {
+                        goto fail;
+                }
+                goto success;
+        }
+
         elf_version(EV_CURRENT);
 
-        if (!(e_core = elf_begin(core_fd, ELF_C_READ, NULL))) {
-                tm_elf_err("Failed to get file descriptor for ELF core file");
+        if (prepare_corefile(&e_core, core_fd) < 0) {
                 goto fail;
         }
 
-        if (!(d_core = dwfl_begin(&cb))) {
-                tm_dwfl_err("Failed to start new libdwfl session");
-                goto fail;
-        }
-
-        core_modules = dwfl_core_file_report(d_core, e_core, NULL);
-        if (core_modules == -1) {
-                tm_dwfl_err("Failed to report modules for ELF core file");
-                goto fail;
-        }
-
-        if (dwfl_report_end(d_core, NULL, NULL) != 0) {
-                tm_dwfl_err("Failed to finish reporting modules");
-                goto fail;
-        }
-
-        if ((core_for_pid = dwfl_core_file_attach(d_core, e_core)) < 0) {
-                tm_dwfl_err("Failed to prepare libdwfl session for thread"
-                            " iteration");
-                goto fail;
-        }
-
-        header = g_string_new(NULL);
-        g_string_printf(header, "Process: %s\nPID: %u\n",
-                        proc_path ? replace_exclamations(proc_path) : proc_name,
-                        (unsigned int)core_for_pid);
+        header = nc_string_dup_printf("Process: %s\nPID: %u\n",
+                                      proc_path ? replace_exclamations(proc_path) : proc_name,
+                                      (unsigned int)core_for_pid);
 
         if (signal_num >= 0) {
-                g_string_append_printf(header, "Signal: %d\n", signal_num);
+                nc_string_append_printf(header, "Signal: %ld\n", signal_num);
         }
 
-        backtrace = g_string_new(NULL);
-        if (dwfl_getthreads(d_core, thread_cb, &backtrace) != DWARF_CB_OK) {
-                if (errorstr != NULL) {
-                        telem_log(LOG_ERR, "%s", errorstr);
-                        g_string_append_printf(header, "Error: %s", errorstr);
-                        g_string_prepend(backtrace, header->str);
-                        send_data(&backtrace, error_severity, error_class);
+        /* On Clear Linux OS, missing symbols may appear if automatic debuginfo
+         * downloads are still in flight. So if any missing symbols appear on
+         * the first run (indicated by the presence of "??? - ["), wait 10
+         * seconds and try again. Also retry if errors occur in the first run.
+         */
+        if ((process_corefile(&backtrace) < 0) || (strstr(backtrace->str, "??? - ["))) {
+                sleep(10);
+
+                if (prepare_corefile(&e_core, core_fd) < 0) {
+                        goto fail;
+                }
+
+                if (process_corefile(&backtrace) < 0) {
+                        goto fail;
                 }
-                goto fail;
         }
 
-        g_string_prepend(backtrace, header->str);
+        nc_string_prepend(backtrace, header->str);
 
         if (!send_data(&backtrace, default_severity, clr_class)) {
                 goto fail;
@@ -522,22 +671,20 @@ success:
 
         ret = EXIT_SUCCESS;
 fail:
-        free_glib_strings();
-
-        if (context) {
-                g_option_context_free(context);
-        }
+        free(core_file);
+        free(proc_name);
+        free(proc_path);
 
         if (header) {
-                g_string_free(header, TRUE);
+                nc_string_free(header);
         }
 
         if (backtrace) {
-                g_string_free(backtrace, TRUE);
+                nc_string_free(backtrace);
         }
 
         if (errorstr) {
-                g_free(errorstr);
+                free(errorstr);
         }
 
         if (d_core) {
@@ -552,6 +699,12 @@ fail:
                 close(core_fd);
         }
 
+        // Remove the core file by default, except when the --core-file option
+        // is specified.
+        if (!core_file) {
+                unlink(temp_core);
+        }
+
         return ret;
 }
 

src/probes/hello.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -20,63 +20,156 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <getopt.h>
+#include <locale.h>
+#include <sys/sysinfo.h>
+#include <dirent.h>
+#include "common.h"
 #include "telemetry.h"
-
 #include "config.h"
+#include "log.h"
 
-void print_usage(char *prog)
+#define PAYLOAD_LOCALE  0x01
+#define PAYLOAD_UPTIME  0x02
+#define PAYLOAD_BUNDLES 0x04
+
+static void print_usage(char *prog)
 {
         printf("%s: Usage\n", prog);
         printf("  -f,  --config_file    Configuration file. This overides the other parameters\n");
         printf("  -H,  --heartbeat      Create a \"heartbeat\" record instead\n");
         printf("  -h,  --help           Display this help message\n");
+        printf("  -l,  --locale         Include locale in the record\n");
+        printf("  -u,  --uptime         Include uptime in the record\n");
+        printf("  -b,  --bundles        Include installed bundles in the record\n");
         printf("  -V,  --version        Print the program version\n");
 }
 
+static long get_uptime(void)
+{
+        struct sysinfo s_info;
+        if (sysinfo(&s_info) == 0) {
+                return s_info.uptime;
+        }
+        return 0;
+}
+
+static int nodots(const struct dirent *dp)
+{
+        return (dp->d_name[0] != '.');
+}
+
+static char *create_payload(unsigned options)
+{
+        int n, buflen;
+        char *ppos, *payload;
+
+        payload = calloc(sizeof(char), MAX_PAYLOAD_LENGTH);
+        if (payload == NULL) {
+                return NULL;
+        }
+
+        ppos = payload;
+        buflen = MAX_PAYLOAD_LENGTH;
+
+        n = snprintf(ppos, (size_t)buflen, "hello\n");
+        buflen -= n, ppos += n;
+
+        if (options & PAYLOAD_LOCALE) {
+                n = snprintf(ppos, (size_t)buflen, "LC_ALL: %s\n", setlocale(LC_ALL, NULL));
+                buflen -= n, ppos += n;
+        }
+
+        if (options & PAYLOAD_UPTIME) {
+                n = snprintf(ppos, (size_t)buflen, "uptime: %ld\n", get_uptime());
+                buflen -= n, ppos += n;
+        }
+
+        if (options & PAYLOAD_BUNDLES) {
+                int num_bundles, i;
+                struct dirent **entries;
+
+                num_bundles = scandir("/usr/share/clear/bundles", &entries, nodots, alphasort);
+                if (num_bundles < 0) {
+                        telem_perror("scandir failed");
+                        free(payload);
+                        return NULL;
+                }
+                n = snprintf(ppos, (size_t)buflen, "\nBundles (%d):\n", num_bundles);
+                buflen -= n, ppos += n;
+
+                for (i = 0; i < num_bundles; i++) {
+                        n = snprintf(ppos, (size_t)buflen, "%s\n", entries[i]->d_name);
+                        /* Test if truncated output */
+                        if (n >= buflen) {
+                                int lastix = MAX_PAYLOAD_LENGTH-1;
+                                payload[lastix] = '\0';
+                                payload[lastix-1] = '.';
+                                payload[lastix-2] = '.';
+                                payload[lastix-3] = '.';
+                                payload[lastix-4] = '\n';
+                                break;
+                        }
+                        buflen -= n, ppos += n;
+                }
+
+                for (i= 0; i < num_bundles; i++) {
+                        free(entries[i]);
+                }
+
+                free(entries);
+        }
+
+        return payload;
+}
+
 int main(int argc, char **argv)
 {
         uint32_t severity = 1;
         uint32_t payload_version = 1;
         char classification[30] = "org.clearlinux/hello/world";
         struct telem_ref *tm_handle = NULL;
-        char *payload2;
+        char *payload;
         char *str;
+        int ret;
+        unsigned payload_options = 0;
 
         // Following vars are for arg parsing.
         int c;
-        char *cfile = NULL;
         int opt_index = 0;
-        int ret = 0;
 
         struct option opts[] = {
-                { "cfile", 1, NULL, 'f' },
+                { "config_file", 1, NULL, 'f' },
                 { "heartbeat", 0, NULL, 'H' },
                 { "help", 0, NULL, 'h' },
                 { "version", 0, NULL, 'V' },
+                { "locale", 0, NULL, 'l' },
+                { "uptime", 0, NULL, 'u' },
+                { "bundles", 0, NULL, 'b' },
                 { NULL, 0, NULL, 0 }
         };
 
-        while ((c = getopt_long(argc, argv, "f:hHV", opts, &opt_index)) != -1) {
+        while ((c = getopt_long(argc, argv, "f:hHVlub", opts, &opt_index)) != -1) {
                 switch (c) {
                         case 'f':
-                                if (asprintf(&cfile, "%s", (char *)optarg) < 0) {
-                                        exit(EXIT_FAILURE);
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
                                 }
-
-                                struct stat buf;
-                                ret = stat(cfile, &buf);
-
-                                /* check if the file exists and is a regular file */
-                                if (ret == -1 || !S_ISREG(buf.st_mode)) {
-                                        exit(EXIT_FAILURE);
-                                }
-
-                                tm_set_config_file(cfile);
                                 break;
                         case 'H':
                                 str = "org.clearlinux/heartbeat/ping";
                                 memcpy(classification, str, strlen(str));
                                 break;
+                        case 'l':
+                                payload_options |= PAYLOAD_LOCALE;
+                                break;
+                        case 'u':
+                                payload_options |= PAYLOAD_UPTIME;
+                                break;
+                        case 'b':
+                                payload_options |= PAYLOAD_BUNDLES;
+                                break;
                         case 'h':
                                 print_usage(argv[0]);
                                 exit(EXIT_SUCCESS);
@@ -88,34 +181,39 @@ int main(int argc, char **argv)
                 }
         }
 
-        if (asprintf(&payload2, "hello\n") < 0) {
+        setlocale(LC_ALL, "");
+        payload = create_payload(payload_options);
+        if (payload == NULL) {
                 exit(EXIT_FAILURE);
         }
 
         if ((ret = tm_create_record(&tm_handle, severity, classification,
                                     payload_version)) < 0) {
-                printf("Failed to create record: %s\n", strerror(-ret));
-                ret = 1;
-                goto fail;
+                telem_log(LOG_ERR, "Failed to create record: %s\n",
+                       strerror(-ret));
+                ret = EXIT_FAILURE;
+                goto done;
         }
 
-        if ((ret = tm_set_payload(tm_handle, payload2)) < 0) {
-                printf("Failed to set record payload: %s\n", strerror(-ret));
-                ret = 1;
-                goto fail;
+        if ((ret = tm_set_payload(tm_handle, payload)) < 0) {
+                telem_log(LOG_ERR, "Failed to set record payload: %s\n",
+                        strerror(-ret));
+                ret = EXIT_FAILURE;
+                goto done;
         }
 
-        free(payload2);
+        free(payload);
 
         if ((ret = tm_send_record(tm_handle)) < 0) {
-                printf("Failed to send record to daemon: %s\n", strerror(-ret));
-                ret = 1;
-                goto fail;
+                telem_log(LOG_ERR, "Failed to send record to daemon: %s\n",
+                        strerror(-ret));
+                ret = EXIT_FAILURE;
+                goto done;
         } else {
                 printf("Successfully sent record to daemon.\n");
-                ret = 0;
+                ret = EXIT_SUCCESS;
         }
-fail:
+done:
         tm_free_record(tm_handle);
         tm_handle = NULL;
 

src/probes/journal.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2015-2017 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -15,6 +15,7 @@
  */
 
 #define _GNU_SOURCE
+#include <getopt.h>
 #include <poll.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -22,8 +23,6 @@
 #include <string.h>
 #include <unistd.h>
 
-#include <glib.h>
-
 /* Certain static analysis tools do not understand GCC's __INCLUDE_LEVEL__
  * macro; the conditional definition below is used to fix the build with
  * systemd's _sd-common.h, included by the public systemd headers, which rely on
@@ -39,10 +38,11 @@
 #include "config.h"
 #include "log.h"
 #include "telemetry.h"
-
+#include "probe.h"
+#include "nica/nc-string.h"
 #define BOOT_ID_LEN 33
 
-static GString *payload = NULL;
+static nc_string *payload = NULL;
 static uint32_t severity = 2;
 static uint32_t payload_version = 1;
 static char error_class[30] = "org.clearlinux/journal/error";
@@ -61,12 +61,11 @@ static inline void tm_journal_match_err(int ret)
 static void add_to_payload(const void *data, size_t length)
 {
         if (payload != NULL) {
-                g_string_append_printf(payload, "%.*s", (int)length,
-                                       (char *)data);
+                nc_string_append_printf(payload, "%.*s\n", (int)length,
+                                        (char *)data);
         } else {
-                payload = g_string_new(NULL);
-                g_string_printf(payload, "%.*s", (int)length,
-                                (char *)data);
+                payload = nc_string_dup_printf("%.*s\n", (int)length,
+                                               (char *)data);
         }
 }
 
@@ -77,16 +76,17 @@ static bool send_data(char *class)
 
         if ((ret = tm_create_record(&handle, severity, class,
                                     payload_version)) < 0) {
-                telem_log(LOG_ERR, "Failed to create record: %s",
+                telem_log(LOG_ERR, "Failed to create record: %s\n",
                           strerror(-ret));
                 goto fail;
         }
 
-        gchar *payload_str = g_string_free(payload, FALSE);
+        char *payload_str = strdup(payload->str);
+        nc_string_free(payload);
         payload = NULL;
 
         if ((ret = tm_set_payload(handle, (char *)payload_str)) < 0) {
-                telem_log(LOG_ERR, "Failed to set payload: %s", strerror(-ret));
+                telem_log(LOG_ERR, "Failed to set payload: %s\n", strerror(-ret));
                 free(payload_str);
                 tm_free_record(handle);
                 goto fail;
@@ -95,7 +95,7 @@ static bool send_data(char *class)
         free(payload_str);
 
         if ((ret = tm_send_record(handle)) < 0) {
-                telem_log(LOG_ERR, "Failed to send record: %s", strerror(-ret));
+                telem_log(LOG_ERR, "Failed to send record: %s\n", strerror(-ret));
                 tm_free_record(handle);
                 goto fail;
         }
@@ -122,6 +122,16 @@ static int read_new_entries(sd_journal *journal)
 
                 add_to_payload(data, length);
 
+                // For now, we send one record per log message, in case the
+                // there is a large backlog of messages and we exceed the
+                // payload size limit (8KB). And ignore errors, hoping that it's
+                // a transient problem.
+
+                if (!send_data(error_class)) {
+                        telem_log(LOG_ERR, "Failed to send data. Ignoring.\n");
+                        return num_entries;
+                }
+
                 num_entries++;
         }
 
@@ -148,19 +158,13 @@ static bool process_existing_entries(sd_journal *journal)
                 return false;
         }
 
-        if (!read_new_entries(journal)) {
+        ret = read_new_entries(journal);
+        if (ret < 0) {
                 return false;
-        }
-
-        if (!payload) {
-                telem_log(LOG_DEBUG, "No existing entries found\n");
+        } else if (ret == 0) {
                 return true;
         }
 
-        if (!send_data(error_class)) {
-                return false;
-        }
-
         return true;
 }
 
@@ -185,6 +189,33 @@ static bool get_boot_id(char **data)
         return true;
 }
 
+#define JOURNAL_MATCH(data) \
+        do { \
+                r = sd_journal_add_match(journal, data, 0); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
+#define JOURNAL_AND \
+        do { \
+                r = sd_journal_add_conjunction(journal); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
+#define JOURNAL_OR \
+        do { \
+                r = sd_journal_add_disjunction(journal); \
+                if (r < 0) { \
+                        tm_journal_match_err(r); \
+                        return false; \
+                } \
+        } while (0);
+
 static bool add_filters(sd_journal *journal)
 {
         char *data = NULL;
@@ -194,29 +225,47 @@ static bool add_filters(sd_journal *journal)
                 return false;
         }
 
-        r = sd_journal_add_match(journal, data, 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
-        }
+        /* The semantics of how journal entry matching works is described in
+         * detail in sd_journal_add_match(3).
+         *
+         * When the privacy filter override is enabled, the matches declared
+         * here correspond to this logical expression:
+         *
+         *   BOOTID && ((P0 || P1 || P2 || P3) || EXITED)
+         *
+         * Otherwise, the expression is:
+         *
+         *   BOOTID && EXITED
+         *
+         * BOOTID is short for _BOOT_ID=VAL, where VAL is the boot ID for the
+         * current boot. P0, P1, etc stand for PRIORITY=0, etc. And EXITED is
+         * short for EXIT_CODE=exited.
+         */
+
+        JOURNAL_MATCH(data);
         free(data);
+        JOURNAL_AND;
 
-        r = sd_journal_add_match(journal, "SYSLOG_IDENTIFIER=crashprobe", 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
+        // Filter messages with the four highest log levels, all indicating
+        // errors, but only when the privacy filters override is in effect;
+        // because the log messages contain arbitrary strings, and this probe
+        // does not yet keep a whitelist of allowed patterns or a blacklist of
+        // banned patterns.
+        if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                JOURNAL_MATCH("PRIORITY=0");
+                JOURNAL_MATCH("PRIORITY=1");
+                JOURNAL_MATCH("PRIORITY=2");
+                JOURNAL_MATCH("PRIORITY=3");
+                JOURNAL_OR;
         }
 
-        r = sd_journal_add_match(journal, "PRIORITY=3", 0);
-        if (r < 0) {
-                tm_journal_match_err(r);
-                return false;
-        }
+        // Only set for service-level error conditions
+        JOURNAL_MATCH("EXIT_CODE=exited");
 
         return true;
 }
 
-static bool process_journal(void)
+static bool process_journal(bool process_existing)
 {
         int r;
 
@@ -230,8 +279,14 @@ static bool process_journal(void)
                 return false;
         }
 
-        if (!process_existing_entries(journal)) {
-                return false;
+        if (process_existing) {
+                if (!process_existing_entries(journal)) {
+                        return false;
+                }
+        } else {
+                if ((sd_journal_seek_tail(journal) < 0) || (sd_journal_previous(journal) < 0)) {
+                        return false;
+                }
         }
 
         struct pollfd pfd;
@@ -258,71 +313,74 @@ static bool process_journal(void)
                                 } else if (r < 0) {
                                         return false;
                                 }
-
-                                if (!send_data(error_class)) {
-                                        return false;
-                                }
                         }
                 }
         }
 }
 
-static gchar *config_file = NULL;
-static gboolean version_p = FALSE;
-
-static GOptionEntry options[] = {
-        { "config-file", 'f', 0, G_OPTION_ARG_FILENAME, &config_file,
-          "Path to configuration file (not implemented yet)", NULL },
-        { "version", 'V', 0, G_OPTION_ARG_NONE, &version_p,
-          "Print the program version", NULL },
-        { NULL }
+static const struct option prog_opts[] = {
+        { "help", no_argument, 0, 'h' },
+        { "config-file", required_argument, 0, 'f' },
+        { "version", no_argument, 0, 'V' },
+        { "tail", no_argument, 0, 't' },
+        { 0, 0, 0, 0 }
 };
 
-static void free_glib_strings(void)
+static void print_help(void)
 {
-        if (config_file) {
-                g_free(config_file);
-        }
+        printf("Usage:\n");
+        printf("  journalprobe [OPTIONS] - collect data from systemd journal\n");
+        printf("\n");
+        printf("Help Options:\n");
+        printf("  -h, --help            Show help options\n");
+        printf("\n");
+        printf("Application Options:\n");
+        printf("  -f, --config_file     Specify a configuration file other than default\n");
+        printf("  -t, --tail            Don't process entries already in the journal\n");
+        printf("  -V, --version         Print the program version\n");
+        printf("\n");
 }
 
 int main(int argc, char **argv)
 {
         int ret = EXIT_FAILURE;
-        GError *error = NULL;
-        GOptionContext *context;
+        int opt;
+        bool process_existing = true;
 
-        context = g_option_context_new("- collect data from systemd journal");
-        g_option_context_add_main_entries(context, options, NULL);
-        g_option_context_set_translate_func(context, NULL, NULL, NULL);
-        if (!g_option_context_parse(context, &argc, &argv, &error)) {
-                g_print("Failed to parse options: %s\n", error->message);
-                goto fail;
+        while ((opt = getopt_long(argc, argv, "hf:Vt", prog_opts, NULL)) != -1) {
+                switch (opt) {
+                        case 'h':
+                                print_help();
+                                goto success;
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                goto success;
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 't':
+                                process_existing = false;
+                                break;
+                }
         }
 
-        if (version_p) {
-                g_print(PACKAGE_VERSION "\n");
-                goto success;
-        }
-
-        if (!process_journal()) {
+        if (!process_journal(process_existing)) {
                 goto fail;
         }
 
 success:
         ret = EXIT_SUCCESS;
 fail:
-        free_glib_strings();
-
         if (journal) {
                 sd_journal_close(journal);
         }
 
-        if (context) {
-                g_option_context_free(context);
-        }
-
         if (payload) {
-                g_string_free(payload, TRUE);
+                nc_string_free(payload);
         }
 
         return ret;

src/probes/klog_main.c

@@ -1,12 +1,28 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright © 2015-2019 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
 #define _GNU_SOURCE
 
 #include <stdlib.h>
 #include <stdio.h>
 #include <sys/klog.h>
-#include <fcntl.h>
 #include <errno.h>
 #include <unistd.h>
 #include <malloc.h>
+#include <signal.h>
 
 #include "log.h"
 #include "oops_parser.h"
@@ -21,54 +37,48 @@ int main(void)
         int log_size = 0;
         char *bufp = NULL;
         size_t buflen = 0;
-        int bytes = 0;
-        int loopcount = 0;
 
-        oops_parser_init(write_oops_to_file);
+        oops_parser_init(klog_process_oops_msgs);
 
-        // Allocate space to write the oops file to
-        if (allocate_filespace() < 0) {
-                telem_log(LOG_ERR, "%s\n", strerror(errno));
-                return 1;
-        }
-
-        // Signal Handling to close and unlink empty files
-        if (signal (SIGINT, signal_handler) == SIG_ERR) {
+        // Signal Handling to terminate the probe.
+        if (signal (SIGINT, signal_handler_fail) == SIG_ERR) {
                 telem_log(LOG_ERR, "Error handling interrupt signal\n");
         }
-        if (signal (SIGTERM, signal_handler) == SIG_ERR) {
+        if (signal (SIGTERM, signal_handler_success) == SIG_ERR) {
                 telem_log(LOG_ERR, "Error handling terminating signal\n");
         }
 
-        while (1) {
-                // Gets the size of the kernel ring buffer
-                log_size = klogctl(SYSLOG_ACTION_SIZE_BUFFER, NULL, 0);
-                if (log_size < 0) {
-                        telem_log(LOG_ERR, "Cannot read size of kernel ring buffer\n");
-                        return 1;
-                }
-
-                buflen = (size_t)log_size;
-
-                if (loopcount++ > 0 && buflen > MAX_BUF)
-                        buflen = MAX_BUF;
-
-                // Gets the contents of the kernel ring buffer
-                bufp = (char *)calloc(buflen, sizeof(char));
-
-                malloc_trim(0);
-                bytes = klogctl(SYSLOG_ACTION_READ, bufp, log_size);
-                if (bytes < 0) {
-                        telem_log(LOG_ERR, "Cannot read contents of kernel ring buffer: %s\n", strerror(errno));
-                        return 1;
-                }
-
-                // Splits the buffer into separate lines with /0 terminating
-                split_buf_by_line(bufp, (size_t)bytes);
-
-                free(bufp);
+        // Gets the size of the kernel ring buffer
+        log_size = klogctl(SYSLOG_ACTION_SIZE_BUFFER, NULL, 0);
+        if (log_size < 0) {
+                telem_perror("Cannot read size of kernel ring buffer");
+                return 1;
         }
 
+        buflen = (size_t)log_size;
+        if (buflen > MAX_BUF) {
+                buflen = MAX_BUF;
+        }
+
+        // Gets the contents of the kernel ring buffer
+        bufp = (char *)malloc(buflen);
+
+        while (1) {
+                int bytes_read;
+ 
+                malloc_trim(0);
+                memset(bufp, 0, buflen);
+                bytes_read = klogctl(SYSLOG_ACTION_READ, bufp, (int)buflen);
+                if (bytes_read < 0) {
+                        telem_perror("Cannot read contents of kernel ring buffer");
+                        return 1;
+                }
+
+                klog_process_buffer(bufp, (size_t)bytes_read);
+        }
+
+        // Not reached
+        free(bufp);
         return 0;
 }
 

src/probes/klog_scanner.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright © 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -15,33 +15,65 @@
  */
 
 #define _GNU_SOURCE
-
 #include <stdlib.h>
 #include <stdio.h>
+#include <sys/types.h>
 #include <sys/klog.h>
-#include <fcntl.h>
 #include <errno.h>
 #include <unistd.h>
-#include <glib.h>
 #include <sys/stat.h>
 
 #include "log.h"
 #include "oops_parser.h"
 #include "klog_scanner.h"
+#include "telemetry.h"
+#include "nica/nc-string.h"
 
-#define MAX_OOPS_MSG_LENGTH 1024
-#define FILE_PATH KERNELOOPSDIR "/kerneloopsXXXXXX"
+static bool oops_processed = false;
 
-static int fd;
-char filename[] = FILE_PATH;
-int last_line_length;
+static uint32_t version = 1;
+
+static bool send_data(char *backtrace, char *class, uint32_t severity)
+{
+        struct telem_ref *handle = NULL;
+        int ret;
+
+        if ((ret = tm_create_record(&handle, severity, class, version)) < 0) {
+                telem_log(LOG_ERR, "Failed to create record: %s",
+                          strerror(-ret));
+                return false;
+        }
+
+        if ((ret = tm_set_payload(handle, backtrace)) < 0) {
+                telem_log(LOG_ERR, "Failed to set payload: %s", strerror(-ret));
+                tm_free_record(handle);
+                return false;
+        }
+
+        if ((ret = tm_send_record(handle)) < 0) {
+                telem_log(LOG_ERR, "Failed to send record: %s", strerror(-ret));
+                tm_free_record(handle);
+                return false;
+        }
+
+        tm_free_record(handle);
+
+        return true;
+}
+
+int klog_process_buffer(char *bufp, int bytes)
+{
+        // Splits the buffer into separate lines with /0 terminating
+        split_buf_by_line(bufp, (size_t)bytes);
+        return 0;
+}
 
-bool oops_written = false;
 void split_buf_by_line(char *bufp, int bytes)
 {
         char *start;
         char *eol;
         size_t linelength = 0;
+        oops_processed = false;
 
         while (bytes > 0) {
                 eol = memchr(bufp, '\n', (size_t)bytes);
@@ -58,87 +90,77 @@ void split_buf_by_line(char *bufp, int bytes)
                 if (bytes >= 0) {
                         start[linelength] = '\0';
                         parse_single_line(start, linelength);
+                        if (oops_processed == true) {
+                                telem_debug("oops_processed detected !\n");
+                                break;
+                        }
                         linelength = 0;
                 }
         }
 }
 
-int allocate_filespace()
+void klog_process_oops_msgs(struct oops_log_msg *msg)
 {
-        int allocate_disk = 0;
-        strcpy(filename, FILE_PATH);
-
-        fd = mkstemp(filename);
-        if (fchmod(fd, 0644) == -1) {
-                telem_log(LOG_ERR, "Error while changing permissions for file %s: %s\n",
-                          filename, strerror(errno));
-        }
-
-        if (fd == -1) {
-                telem_log(LOG_ERR, "fd error: %s\n", strerror(errno));
-                return -1;
-        }
-
-        // Allocates disk space to write kernel oops to
-        allocate_disk = fallocate(fd, 0, 0, MAX_OOPS_MSG_LENGTH);
-        if (allocate_disk != 0) {
-                telem_log(LOG_ERR, "Error allocating disk space: %s\n", strerror(errno));
-                return -1;
-        }
-        return 0;
-}
-
-void write_oops_to_file(struct oops_log_msg *msg)
-{
-        ssize_t writer;
-        size_t linelength;
+        size_t linelength, size;
         char *line;
+        char *contents = NULL;
+        nc_string *payload;
 
+        // pass 1: calculate buffer size
+        size = 0;
+        for (int i = 0; i < msg->length; i++) {
+                size += strlen(msg->lines[i]) + 1;
+        }
+
+        contents = malloc(size);
+        if (!contents) {
+                telem_log(LOG_ERR, "Call to malloc failed");
+                oops_processed = true;
+                return;
+        }
+
+        // pass 2: copy strings into the buffer
+        size_t done = 0;
+        char *bp = contents;
         for (int i = 0; i < msg->length; i++) {
                 //Add the newline character to the end of each line
                 line = msg->lines[i];
                 linelength = strlen(line);
-                line[linelength] = '\n';
-                linelength++;
+                // Copy the line (without the terminating NULL)
+                memcpy(bp, line, linelength);
+                bp[linelength] = '\n';
+                done += linelength + 1;
+                bp = contents + done;
+        }
 
-                // Write each line of oops to file
-                writer = write(fd, line, linelength);
-                if (writer == -1) {
-                        telem_log(LOG_ERR, "Error writing buffer to file: %s\n", strerror(errno));
-                        exit(EXIT_FAILURE);
+        struct oops_log_msg oops_msg;
+        if (handle_entire_oops(contents, (long)size, &oops_msg)) {
+#ifdef DEBUG
+                telem_debug("DEBUG: Raw oops message:\n");
+                for (int i = 0; i < oops_msg.length; i++) {
+                        telem_log(LOG_DEBUG, "%s\n", oops_msg.lines[i]);
                 }
+#endif
+                payload = parse_payload(&oops_msg);
+                telem_debug("DEBUG: Payload Parsed :%s\n", payload->str);
+                oops_msg_cleanup(&oops_msg);
+                send_data(payload->str, (char *)oops_msg.pattern->classification, (uint32_t)oops_msg.pattern->severity);
+                nc_string_free(payload);
+        } else {
+                /* File does no contain an oops! */
+                telem_log(LOG_ERR, "Did not find an oops in the oops directory\n");
         }
 
-        telem_log(LOG_INFO, "Oops written to file\n");
-        oops_written = true;
-        close(fd);
-
-        // Allocate more space to prepare for another oops
-        if (allocate_filespace() < 0) {
-                telem_log(LOG_ERR, "%s\n", strerror(errno));
-                exit(EXIT_FAILURE);
-        }
+        free(contents);
+        oops_processed = true;
 }
 
-bool get_oops_written(void)
+void signal_handler_success(int signum)
 {
-        return oops_written;
+        exit(EXIT_SUCCESS);
 }
 
-char *get_file_name(void)
-{
-        return filename;
-}
-
-__attribute__((destructor))
-void clean_up(void)
-{
-        // Unlink empty file and close any open file
-        unlink(filename);
-        close(fd);
-}
-
-void signal_handler(int signum)
+void signal_handler_fail(int signum)
 {
         exit(EXIT_FAILURE);
 }

src/probes/klog_scanner.h

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright © 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -17,45 +17,41 @@
 #pragma once
 #include "oops_parser.h"
 
+/**
+ * Process the buffer and send it to the backend
+ *
+ * @param *bufp A pointer to a buffer containing the bytes read
+ *		from the kernel ring buffer
+ * @param bytes Number of bytes read from ring buffer into bufp
+ * 
+ * @return 0 on succes, -1 on failure
+ *
+ */
+int klog_process_buffer(char *bufp, int bytes);
 /**
  * Splits the buffer into individual lines
  *
  * @param *bufp A pointer to a buffer containing the bytes read
  *		from the kernel ring buffer
- * @param bytes Bytes read from ring buffer into bufp
+ * @param bytes Number of bytes read from ring buffer into bufp
  *
  */
 void split_buf_by_line(char *bufp, int bytes);
 
 /**
- * Allocates memory for oops message to write to file
- *
- * @return 0 on succes, -1 on failure
- */
-int allocate_filespace(void);
-
-/**
- * Writes the oops message to tmp file
+ * Process the oops message and send it to the backend
  *
  * @param msg Struct that contains oops message lines and linecount
  *
  */
-void write_oops_to_file(struct oops_log_msg *msg);
-
+void klog_process_oops_msgs(struct oops_log_msg *msg);
 /**
  * Signal handler for cleanup purposes
  *
  * @param signum An integer for the signal type
  *
  */
-void signal_handler(int signum);
-
-/**
- * For Unit Tests Purposes
- * Determines whether an oops has been written to a file
- *
- * @return true if oops is written, false if not
- */
-bool get_oops_written(void);
+void signal_handler_fail(int signum);
+void signal_handler_success(int signum);
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/local.mk

@@ -4,8 +4,15 @@ bin_PROGRAMS += \
 	%D%/telem-record-gen \
 	%D%/klogscanner \
 	%D%/pstoreprobe \
-	%D%/oopsprobe \
-	%D%/pstoreclean
+	%D%/pythonprobe \
+	%D%/pstoreclean \
+	%D%/bertprobe
+
+%C%_pythonprobe_SOURCES = %D%/python-probe.c
+%C%_pythonprobe_LDADD = $(top_builddir)/src/libtelemetry.la
+%C%_pythonprobe_LDFLAGS = \
+	$(AM_LDFLAGS) \
+	-pie
 
 %C%_hprobe_SOURCES = %D%/hello.c
 %C%_hprobe_LDADD = $(top_builddir)/src/libtelemetry.la
@@ -13,12 +20,17 @@ bin_PROGRAMS += \
 	$(AM_LDFLAGS) \
 	-pie
 
+%C%_bertprobe_SOURCES = %D%/bert_probe.c \
+	src/nica/b64enc.c
+%C%_bertprobe_LDADD = $(top_builddir)/src/libtelemetry.la
+%C%_bertprobe_LDFLAGS = \
+        $(AM_LDFLAGS) \
+        -pie
+
 %C%_telem_record_gen_SOURCES = %D%/telem_record_gen.c
 %C%_telem_record_gen_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
-%C%_telem_record_gen_LDADD = $(top_builddir)/src/libtelemetry.la \
-	$(GLIB_LIBS)
+	$(AM_CFLAGS)
+%C%_telem_record_gen_LDADD = $(top_builddir)/src/libtelemetry.la
 %C%_telem_record_gen_LDFLAGS = \
 	$(AM_LDFLAGS) \
 	-pie
@@ -44,15 +56,15 @@ endif
 
 
 %C%_crashprobe_SOURCES = \
-	%D%/crash_probe.c
+	%D%/crash_probe.c \
+	src/nica/nc-string.c \
+	%D%/probe.h
 %C%_crashprobe_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 %C%_crashprobe_LDADD = \
 	$(top_builddir)/src/libtelemetry.la \
 	$(top_builddir)/src/libtelem-shared.la \
-	@ELFUTILS_LIBS@ \
-	$(GLIB_LIBS)
+	@ELFUTILS_LIBS@
 %C%_crashprobe_LDFLAGS = \
        $(AM_LDFLAGS) \
        -pie
@@ -68,14 +80,13 @@ endif
 
 %C%_pstoreprobe_SOURCES = \
 	%D%/pstore_probe.c \
+	src/nica/nc-string.c \
 	%D%/oops_parser.c
 %C%_pstoreprobe_CFLAGS = \
-	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+	$(AM_CFLAGS)
 %C%_pstoreprobe_LDADD = \
 	$(top_builddir)/src/libtelemetry.la \
-	$(top_builddir)/src/libtelem-shared.la \
-	$(GLIB_LIBS)
+	$(top_builddir)/src/libtelem-shared.la
 
 if LOG_SYSTEMD
 if HAVE_SYSTEMD_JOURNAL
@@ -91,13 +102,12 @@ endif
         %D%/klog_scanner.c \
 	%D%/klog_scanner.h \
 	%D%/oops_parser.h \
+	src/nica/nc-string.c \
 	%D%/oops_parser.c
 %C%_klogscanner_CFLAGS = \
-        $(AM_CFLAGS) \
-	$(GLIB_CFLAGS)
+        $(AM_CFLAGS)
 %C%_klogscanner_LDADD = \
-        $(top_builddir)/src/libtelemetry.la \
-	$(GLIB_LIBS)
+        $(top_builddir)/src/libtelemetry.la
 %C%_klogscanner_LDFLAGS = \
         $(AM_LDFLAGS) \
         -pie
@@ -118,25 +128,6 @@ if HAVE_SYSTEMD_JOURNAL
 endif
 endif
 
-%C%_oopsprobe_SOURCES = \
-        %D%/oops_probe.c \
-	%D%/oops_parser.c
-%C%_oopsprobe_CFLAGS = \
-        $(AM_CFLAGS) \
-        $(GLIB_CFLAGS)
-%C%_oopsprobe_LDADD = \
-        $(top_builddir)/src/libtelemetry.la \
-        $(top_builddir)/src/libtelem-shared.la \
-        $(GLIB_LIBS)
-
-if LOG_SYSTEMD
-if HAVE_SYSTEMD_JOURNAL
-%C%_oopsprobe_CFLAGS += \
-        $(SYSTEMD_JOURNAL_CFLAGS)
-%C%_oopsprobe_LDADD += \
-        $(SYSTEMD_JOURNAL_LIBS)
-endif
-endif
 
 if HAVE_SYSTEMD_JOURNAL
 if HAVE_SYSTEMD_ID128
@@ -144,16 +135,15 @@ bin_PROGRAMS += \
 	%D%/journalprobe
 
 %C%_journalprobe_SOURCES = \
+	src/nica/nc-string.c \
 	%D%/journal.c
 %C%_journalprobe_CFLAGS = \
 	$(AM_CFLAGS) \
-	$(GLIB_CFLAGS) \
 	$(SYSTEMD_ID128_CFLAGS) \
 	$(SYSTEMD_JOURNAL_CFLAGS)
 %C%_journalprobe_LDADD = \
 	$(top_builddir)/src/libtelemetry.la \
 	$(top_builddir)/src/libtelem-shared.la \
-	$(GLIB_LIBS) \
 	$(SYSTEMD_ID128_LIBS) \
 	$(SYSTEMD_JOURNAL_LIBS)
 %C%_journalprobe_LDFLAGS = \

src/probes/oops_parser.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2015-2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -21,9 +21,11 @@
 #include <string.h>
 #include <ctype.h>
 #include <inttypes.h>
+#include <unistd.h>
 
 #include "oops_parser.h"
 #include "log.h"
+#include "probe.h"
 
 #define NUM_TAINTED_FLAGS 16
 
@@ -150,9 +152,9 @@ struct oops_pattern oops_patterns_arr[] = {
         },
 };
 
-int oops_patterns_cnt = sizeof(oops_patterns_arr) / sizeof(struct oops_pattern);
+static int oops_patterns_cnt = sizeof(oops_patterns_arr) / sizeof(struct oops_pattern);
 
-char *skip_log_level(char *line)
+static char *skip_log_level(char *line)
 {
         char *start = line;
 
@@ -167,7 +169,7 @@ char *skip_log_level(char *line)
         return start;
 }
 
-char *skip_timestamp(char *line)
+static char *skip_timestamp(char *line)
 {
         char *start = line;
 
@@ -182,7 +184,7 @@ char *skip_timestamp(char *line)
         return start;
 }
 
-char *skip_spaces(char *line)
+static char *skip_space(char *line)
 {
         char *start = line;
 
@@ -192,7 +194,17 @@ char *skip_spaces(char *line)
         return start;
 }
 
-bool starts_with(const char *line, const char *line_end, const char *substr)
+static char *skip_spaces(char *line)
+{
+        char *start = line;
+
+        while (*start && isspace(*start)) {
+                start++;
+        }
+        return start;
+}
+
+static bool starts_with(const char *line, const char *line_end, const char *substr)
 {
         size_t substrlen = strlen(substr);
 
@@ -202,7 +214,7 @@ bool starts_with(const char *line, const char *line_end, const char *substr)
         return false;
 }
 
-bool str_starts_with_casei(const char *line, const char *substr)
+static bool str_starts_with_casei(const char *line, const char *substr)
 {
         size_t len = strlen(substr);
 
@@ -212,7 +224,7 @@ bool str_starts_with_casei(const char *line, const char *substr)
         return false;
 }
 
-bool pattern_matches(char *line, char *line_end, struct oops_pattern *pattern)
+static bool pattern_matches(char *line, char *line_end, struct oops_pattern *pattern)
 {
         if (pattern->is_regex) {
                 return (regexec(&(pattern->regex), line, 0, NULL, 0) == 0);
@@ -221,7 +233,7 @@ bool pattern_matches(char *line, char *line_end, struct oops_pattern *pattern)
         }
 }
 
-void init_pattern_regex(void)
+static void init_pattern_regex(void)
 {
         for (int i = 0; i < oops_patterns_cnt; i++) {
                 struct oops_pattern *pattern = &oops_patterns_arr[i];
@@ -232,7 +244,7 @@ void init_pattern_regex(void)
         }
 }
 
-void free_pattern_regex(void)
+static void free_pattern_regex(void)
 {
         for (int i = 0; i < oops_patterns_cnt; i++) {
                 struct oops_pattern *pattern = &oops_patterns_arr[i];
@@ -249,7 +261,6 @@ bool handle_entire_oops(char *buf, long size, struct oops_log_msg *msg)
         int i;
 
         msg->length = 0;
-        init_pattern_regex();
 
         line_end = memchr(buf, '\n', (size_t)size);
         if (line_end == NULL) {
@@ -306,7 +317,6 @@ void oops_msg_cleanup(struct oops_log_msg *msg)
                 free(msg->lines[i]);
         }
         msg->length = 0;
-        free_pattern_regex();
 }
 
 static struct oops_log_msg oops_msg;
@@ -321,7 +331,7 @@ void oops_parser_init(oops_handler_t handler)
         init_pattern_regex();
 }
 
-void handle_msg_end(void)
+static void handle_msg_end(void)
 {
         for (int i = 0; i < oops_msg.length; i++) {
                 free(oops_msg.lines[i]);
@@ -344,7 +354,7 @@ void parse_single_line(char *line, size_t size)
         line_end = line + size;
         start = skip_log_level(line);
         start = skip_timestamp(start);
-        start = skip_spaces(start);
+        start = skip_space(start);
 
         struct oops_pattern *pattern;
         if (oops_msg.length == 0) {
@@ -376,13 +386,16 @@ void parse_single_line(char *line, size_t size)
                 } else if (strstr(start, "[ end trace")) {
                         end_found = true;
                 } else if (!in_stack_dump) {
-                        if (starts_with(start, line_end, " [<") ||
-                            starts_with(start, line_end, " <") ||      //case : <EOI> [<fff73..]
-                            starts_with(start, line_end, "Call Trace:")) {
+                        // This line indicates the beginning of a stack trace;
+                        // the next line is most likely the topmost frame.
+                        if (starts_with(start, line_end, "Call Trace:")) {
                                 in_stack_dump = true;
                         }
                 } else if (in_stack_dump) {
-                        if (!(starts_with(start, line_end, " [") || starts_with(start, line_end, " <")) ||
+                        // In Linux 4.10+, the oops format changed, so we look
+                        // for a single leading space character to indicate a
+                        // stack frame line.
+                        if (!(starts_with(start, line_end, " ")) ||
                             (strlen(start) < 8) ||
                             (strstr(start, "Code:") != NULL) ||
                             (strstr(start, "Instruction Dump::") != NULL)) {
@@ -411,7 +424,6 @@ void parse_single_line(char *line, size_t size)
                         }
                 }
         }
-
 }
 
 struct stack_frame {
@@ -422,12 +434,23 @@ struct stack_frame {
         struct stack_frame *next;
 };
 
-void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, char *start)
+static void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, char *start)
 {
         /*
-         * Format is as :
+         * Format is:
+         *
          *  [<ffffffffa1002128>] do_one_initcall+0xb8/0x1e0
          *  [<ffffffffa118784a>] ? __vunmap+0x9a/0x100
+         *
+         * OR
+         *
+         *  <IRQ>  [<ffffffff816606a8>] dump_stack+0x19/0x1b
+         *
+         * OR (in 4.10+)
+         *
+         *  do_one_initcall+0xb8/0x1e0
+         *  ? __vunmap+0x9a/0x100
+         *
          */
 
         struct stack_frame *frame = NULL;
@@ -436,18 +459,22 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
         if (!start) {
                 return;
         }
-        start = start + 2;
+
+        // Skip leading spaces
+        start = skip_spaces(start);
 
         //case: <IRQ>  [<ffffffff816606a8>] dump_stack+0x19/0x1b
-        if (!strncmp(start, "IRQ>", 4) || !strncmp(start, "NMI>", 4) ||
-            !strncmp(start, "EOI>", 4) || !strncmp(start, "<EOE>>", 6)) {
-                while (*start && *start != '[') {
+        if (!strncmp(start, "<IRQ>", 5) || !strncmp(start, "<NMI>", 5) ||
+            !strncmp(start, "<EOI>", 5) || !strncmp(start, "<<EOE>>", 7)) {
+                while (*start && !isspace(*start)) {
                         start++;
                 }
-                if (*start == '\0') {
+                if (start && *start == '\0') {
                         return;
                 }
-                start++;
+
+                // Skip intervening space
+                start = skip_spaces(start);
         }
 
         frame = malloc(sizeof(struct stack_frame));
@@ -462,26 +489,29 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
                 *tail = frame;
         }
 
-        if (*start == '<') {
+        // Skip memory address parsing if the address is absent (as in Linux 4.10+).
+        if (!strncmp(start, "[", 1)) {
                 start++;
-                frame->addr = (uint64_t)strtoll(start, (char **)&start, 16);
-        } else {
-                frame->addr = 0;
-        }
 
-        /* Skip up to the space after the closing bracket */
-        while (*start && !isspace(*start)) {
-                start++;
-        }
-        if (*start) {
-                start++;
-        }
+                if (*start == '<') {
+                        start++;
+                        frame->addr = (uint64_t)strtoll(start, (char **)&start, 16);
+                } else {
+                        frame->addr = 0;
+                }
 
-        if (*start && *start == '?') {
-                start++;
-                if (*start && isspace(*start)) {
+                // Skip up to the space after the closing bracket
+                while (*start && !isspace(*start)) {
                         start++;
                 }
+                if (start && *start == '\0') {
+                        return;
+                }
+
+                // Skip intervening space
+                start = skip_spaces(start);
+        } else {
+                frame->addr = 0;
         }
 
         offset_ptr = strchr(start, '+');
@@ -503,7 +533,7 @@ void stack_frame_append(struct stack_frame **head, struct stack_frame **tail, ch
         frame->module = "kernel";
 }
 
-void stack_frame_free(struct stack_frame **head)
+static void stack_frame_free(struct stack_frame **head)
 {
         struct stack_frame *frame;
 
@@ -520,7 +550,7 @@ void stack_frame_free(struct stack_frame **head)
  * CPU: 2 PID: 0 Comm: swapper/2 Not tainted  3.10.4-100.fc18.x86_64 #1
  * CPU: 3 PID: 0 Comm: swapper/3 Not tainted 4.0.5-300.fc22.x86_64 #1
  */
-void parse_kernel_cpu_line(char *line, char **kernel_version, char **tainted)
+static void parse_kernel_cpu_line(char *line, char **kernel_version, char **tainted)
 {
         char *end = NULL;
 
@@ -593,7 +623,7 @@ static struct reg_s *reg_head = NULL;
  *
  */
 
-void parse_registers(char *line)
+static void parse_registers(char *line)
 {
         uint64_t value;
         struct reg_s *reg_entry = NULL;
@@ -706,18 +736,19 @@ void parse_registers(char *line)
 
 }
 
-void append_registers_to_bt(GString **backtrace)
+static void append_registers_to_bt(nc_string **backtrace)
 {
         struct reg_s *reg_entry = reg_head;
 
         while (reg_entry != NULL) {
-                #ifdef DEBUG
-                g_string_append_printf(*backtrace, "Register %s: %" PRIx64 "\n", reg_entry->reg_name,
-                                       reg_entry->reg_value);
-                #else
-                g_string_append_printf(*backtrace, "Register %s: %s\n", reg_entry->reg_name,
-                                       reg_entry->reg_value ? "Non-zero" : "Zero");
-                #endif
+                // Global override for privacy filters
+                if (access(TM_PRIVACY_FILTERS_OVERRIDE, F_OK) == 0) {
+                        nc_string_append_printf(*backtrace, "Register %s: %" PRIx64 "\n", reg_entry->reg_name,
+                                                reg_entry->reg_value);
+                } else {
+                        nc_string_append_printf(*backtrace, "Register %s: %s\n", reg_entry->reg_name,
+                                                reg_entry->reg_value ? "Non-zero" : "Zero");
+                }
                 reg_entry = reg_entry->next;
         }
 
@@ -728,25 +759,36 @@ void append_registers_to_bt(GString **backtrace)
         }
 }
 
-GString *parse_backtrace(struct oops_log_msg *msg)
+static nc_string *parse_backtrace(struct oops_log_msg *msg)
 {
         struct stack_frame *head = NULL, *tail = NULL, *elem = NULL;
         //int in_stack_dump = 0;
         char *line = NULL;
-        GString *backtrace = NULL;
+        nc_string *backtrace = NULL;
         int frame_counter = 1;
         char *modules = NULL, *kernel_version = NULL, *tainted = NULL;
+        // Since lines are processed from last to first, the stack trace lines
+        // will appear first.
+        bool in_trace = true;
 
         for (int i = msg->length - 1; i > 0; i--) {
                 /* Check if this line is part of a stack trace */
                 line = msg->lines[i];
 
-                if (starts_with(line, line + strlen(line), " [") ||
-                    starts_with(line, line + strlen(line), " <")) {
+                if (in_trace && starts_with(line, line + strlen(line), " ")) {
                         stack_frame_append(&head, &tail, line);
                         continue;
                 }
 
+                // We've reached the end of the stack trace
+                if (starts_with(line, line + strlen(line), "Call Trace:")) {
+                        in_trace = false;
+                }
+
+                // Stack trace lines must be consecutive, so if this point is
+                // reached, we're outside the trace boundary.
+                in_trace = false;
+
                 if (str_starts_with_casei(line, "Modules linked in: ")) {
                         modules = line + strlen("Modules linked in: ");
                         continue;
@@ -760,23 +802,23 @@ GString *parse_backtrace(struct oops_log_msg *msg)
                 parse_registers(line);
         }
 
-        backtrace = g_string_new(NULL);
+        backtrace = nc_string_dup("");
         if (kernel_version) {
-                g_string_append_printf(backtrace, "Kernel Version : %s\n", kernel_version);
+                nc_string_append_printf(backtrace, "Kernel Version : %s\n", kernel_version);
                 free(kernel_version);
         }
 
         if (tainted) {
-                g_string_append_printf(backtrace, "Tainted : %s\n", tainted);
+                nc_string_append_printf(backtrace, "Tainted : %s\n", tainted);
                 free(tainted);
         }
 
         if (modules) {
-                g_string_append_printf(backtrace, "Modules : %s\n", modules);
+                nc_string_append_printf(backtrace, "Modules : %s\n", modules);
         }
 
         if (head) {
-                g_string_append_printf(backtrace, "Backtrace :\n");
+                nc_string_append_printf(backtrace, "Backtrace :\n");
         }
 
         if (reg_head) {
@@ -784,24 +826,24 @@ GString *parse_backtrace(struct oops_log_msg *msg)
         }
 
         for (elem = head; elem != NULL; elem = elem->next, frame_counter++) {
-                g_string_append_printf(backtrace, "#%d %s - [%s]\n", frame_counter,
-                                       elem->function ? elem->function : "???",
-                                       elem->module);
+                nc_string_append_printf(backtrace, "#%d %s - [%s]\n", frame_counter,
+                                        elem->function ? elem->function : "???",
+                                        elem->module);
         }
 
         stack_frame_free(&head);
         return backtrace;
 }
 
-GString *parse_payload(struct oops_log_msg *msg)
+nc_string *parse_payload(struct oops_log_msg *msg)
 {
-        GString *payload, *backtrace;
+        nc_string *payload, *backtrace;
 
-        payload = g_string_new("Crash Report:\n");
-        g_string_append_printf(payload, "Reason: %s\n", msg->lines[0]);
+        payload = nc_string_dup("Crash Report:\n");
+        nc_string_append_printf(payload, "Reason: %s\n", msg->lines[0]);
         backtrace = parse_backtrace(msg);
-        g_string_append (payload, backtrace->str);
-        g_string_free(backtrace, true);
+        nc_string_cat(payload, backtrace->str);
+        nc_string_free(backtrace);
         return payload;
 
 }
@@ -814,4 +856,3 @@ GString *parse_payload(struct oops_log_msg *msg)
  */
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */
-

src/probes/oops_parser.h

@@ -17,7 +17,7 @@
 
 #define MAX_LINES 100
 
-#include <glib.h>
+#include "nica/nc-string.h"
 #include <stdbool.h>
 #include <regex.h>
 
@@ -63,7 +63,7 @@ void oops_parser_init(oops_handler_t handler);
 void parse_single_line(char *line, size_t size);
 
 /* Parses a payload from an oops msg*/
-GString *parse_payload(struct oops_log_msg *msg);
+nc_string *parse_payload(struct oops_log_msg *msg);
 
 /* Given an entire oops log, verifies that the log is an oops log and
  * extracts the lines into an oops struct.
@@ -74,4 +74,3 @@ bool handle_entire_oops(char *buf, long size, struct oops_log_msg *msg);
 void oops_msg_cleanup(struct oops_log_msg *msg);
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */
-

src/probes/oops_probe.c

@@ -1,229 +0,0 @@
-/*
- * This program is part of the Clear Linux Project
- *
- * Copyright 2015 Intel Corporation
- *
- * This program is free software; you can redistribute it and/or modify it under
- * the terms and conditions of the GNU Lesser General Public License, as
- * published by the Free Software Foundation; either version 2.1 of the License,
- * or (at your option) any later version.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
- * details.
- */
-
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <sys/types.h>
-#include <glib.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <glib.h>
-#include <sys/inotify.h>
-#include <poll.h>
-
-#include "oops_parser.h"
-#include "log.h"
-#include "telemetry.h"
-
-char *oops_dir_path = KERNELOOPSDIR;
-
-static uint32_t version = 1;
-
-static bool send_data(char *backtrace, char *class, uint32_t severity)
-{
-        struct telem_ref *handle = NULL;
-        int ret;
-
-        if ((ret = tm_create_record(&handle, severity, class, version)) < 0) {
-                telem_log(LOG_ERR, "Failed to create record: %s",
-                          strerror(-ret));
-                return false;
-        }
-
-        if ((ret = tm_set_payload(handle, backtrace)) < 0) {
-                telem_log(LOG_ERR, "Failed to set payload: %s", strerror(-ret));
-                tm_free_record(handle);
-                return false;
-        }
-
-        if ((ret = tm_send_record(handle)) < 0) {
-                telem_log(LOG_ERR, "Failed to send record: %s", strerror(-ret));
-                tm_free_record(handle);
-                return false;
-        }
-
-        tm_free_record(handle);
-
-        return true;
-}
-
-void handle_oops_file(const char *fname)
-{
-        int ret;
-        FILE *fp = NULL;
-        char *filename = NULL, *contents = NULL;
-        long sz;
-        size_t size, bytes_read;
-        GString *payload;
-
-        ret = asprintf(&filename, "%s/%s", oops_dir_path, fname);
-        if (ret == -1) {
-                exit(EXIT_FAILURE);
-        }
-
-        #ifdef DEBUG
-        printf("File to be handled: %s\n", filename);
-        #endif
-
-        fp = fopen(filename, "r");
-        if (fp == NULL) {
-                telem_log(LOG_ERR, "Failed to open oops log file %s: %s", fname, strerror(errno));
-                goto file_end;
-        }
-
-        if (fseek(fp, 0, SEEK_END) == -1) {
-                telem_log(LOG_ERR, "Failed to seek file %s: %s\n", fname, strerror(errno));
-                goto file_end;
-        }
-
-        sz = ftell(fp);
-        if (sz == -1L) {
-                telem_perror("ftell failed");
-                goto file_end;
-        }
-
-        /* If file is empty */
-        if (sz == 0L) {
-                telem_perror("File is empty\n");
-                goto file_end;
-        }
-
-        if (fseek(fp, 0, SEEK_SET) < 0) {
-                telem_log(LOG_ERR, "Fseek failed for file %s:%s\n", fname, strerror(errno));
-                goto  file_end;
-        }
-
-        size = (size_t)sz;
-        contents = malloc(size);
-        if (!contents) {
-                telem_log(LOG_ERR, "Call to malloc failed");
-                goto file_end;
-        }
-        memset(contents, 0, size);
-
-        bytes_read = fread(contents, sizeof(char), size, fp);
-
-        if (bytes_read < size) {
-                telem_log(LOG_ERR, "Error while reading file %s:%s\n", fname, strerror(errno));
-                goto file_end;
-        }
-
-        struct oops_log_msg oops_msg;
-        if (handle_entire_oops(contents, (long)size, &oops_msg)) {
-            #ifdef DEBUG
-                printf("Raw message:\n");
-                for (int i = 0; i < oops_msg.length; i++) {
-                        printf("%s\n", oops_msg.lines[i]);
-                }
-            #endif
-
-                payload = parse_payload(&oops_msg);
-            #ifdef DEBUG
-                printf("Payload Parsed :%s\n", payload->str);
-            #endif
-
-                oops_msg_cleanup(&oops_msg);
-                send_data(payload->str, (char *)oops_msg.pattern->classification, (uint32_t)oops_msg.pattern->severity);
-                g_string_free(payload, true);
-        } else {
-                /* File does no contain an oops! */
-                telem_log(LOG_ERR, "Did not find an oops in the oops directort\n");
-        }
-file_end:
-        if (contents) {
-                free(contents);
-        }
-        if (fp) {
-                fclose(fp);
-        }
-        if (unlink(filename) == -1) {
-                telem_log(LOG_ERR, "Failed to unlink %s: %s\n", filename, strerror(errno));
-        }
-        free(filename);
-}
-
-void handle_inotify_event(const struct inotify_event *event)
-{
-
-        if (!event) {
-                telem_perror("Null event received");
-                return;
-        }
-
-        if (event->len == 0) {
-                telem_perror("inotify event received with no file.");
-                return;
-        }
-
-        if (event->mask & IN_CLOSE_WRITE) {
-                if (event->mask & IN_ISDIR) {
-                        return;
-                }
-
-                #ifdef DEBUG
-                telem_log(LOG_DEBUG, "New file written to the oops location: %s\n", event->name);
-                #endif
-                handle_oops_file(event->name);
-        }
-}
-
-int main(int argc, char **argv)
-{
-        int inotify_fd;
-        char buf[4096];
-        char *ptr = NULL;
-        ssize_t len;
-        const struct inotify_event *event;
-
-        inotify_fd = inotify_init();
-        if (inotify_fd == -1) {
-                telem_log(LOG_ERR, "Failed to create inotify instance: %s\n", strerror(errno));
-                exit(EXIT_FAILURE);
-        }
-
-        if (inotify_add_watch(inotify_fd, oops_dir_path, IN_CLOSE_WRITE) == -1) {
-                telem_perror("Error adding watch to the inotify instance");
-        }
-
-        while (1) {
-                len = read(inotify_fd, buf, sizeof buf);
-                if (len <= 0) {
-                        telem_perror("error reading event");
-                        continue;
-                }
-
-                if (len < sizeof(struct inotify_event)) {
-                        telem_perror("Incomplete event received");
-                        continue;
-                }
-
-                telem_log(LOG_INFO, "Got inotify event/s\n");
-
-                /* Loop over all events in the buffer */
-                for (ptr = buf; ptr < buf + len;
-                     ptr += sizeof(struct inotify_event) + event->len) {
-
-                        event = (const struct inotify_event *)ptr;
-                        /* handle the event */
-                        handle_inotify_event(event);
-                }
-        }
-
-        return 0;
-}
-
-/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/probe.h

@@ -0,0 +1,25 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2017 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#pragma once
+
+/**
+ * Existence of the opt-in-no-privacy-filters file disables certain privacy
+ * filters enforced in telemetry probes.
+ */
+#define TM_PRIVACY_FILTERS_OVERRIDE "/etc/telemetrics/opt-in-no-privacy-filters"
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/probes/pstore_clean.c

@@ -132,7 +132,7 @@ int main(int argc, char **argv)
         if (mkdir(MOUNT, 0755) < 0) {
                 //If there is an error other than an existing directory
                 if (errno != EEXIST) {
-                        telem_log(LOG_ERR, "Error creating directory /dev/pstore : %s\n", strerror(errno));
+                        telem_perror("Error creating directory /dev/pstore");
                         //bail out?
                         exit(EXIT_FAILURE);
                 }

src/probes/pstore_probe.c

@@ -12,11 +12,10 @@
  * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
  * details.
  */
-
+#define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
 #include <dirent.h>
-#include <glib.h>
 #include <string.h>
 #include <errno.h>
 #include <assert.h>
@@ -26,6 +25,7 @@
 #include "log.h"
 #include "telemetry.h"
 #include "oops_parser.h"
+#include "nica/hashmap.h"
 
 char *pstore_dump_path = PSTOREDIR;
 
@@ -61,20 +61,18 @@ static bool send_data(char *backtrace, char *class, uint32_t severity)
 
 void handle_complete_oops_message(struct oops_log_msg *msg)
 {
-        GString *payload;
+        nc_string *payload;
 
 #ifdef DEBUG
         for (int i = 0; i < msg->length; i++) {
-                printf("%s\n", msg->lines[i]);
+                telem_debug("DEBUG: %s\n", msg->lines[i]);
         }
 #endif
         payload = parse_payload(msg);
 
-#ifdef DEBUG
-        printf("Payload Parsed :%s\n", payload->str);
-#endif
+        telem_debug("DEBUG: Payload Parsed :%s\n", payload->str);
         send_data(payload->str, (char *)msg->pattern->classification, (uint32_t)msg->pattern->severity);
-        g_string_free(payload, true);
+        nc_string_free(payload);
 }
 
 void handle_crash_dump(char *dump, size_t size)
@@ -100,7 +98,7 @@ void handle_crash_dump(char *dump, size_t size)
 
 char *read_contents(char *filename, size_t *bytes)
 {
-        gchar *dump_file;
+        char *dump_file = NULL;
         FILE *fp = NULL;
         char *contents = NULL;
         long sz = 0;
@@ -108,7 +106,10 @@ char *read_contents(char *filename, size_t *bytes)
         char *hdr_end;
         size_t hdr_len;
 
-        dump_file = g_strconcat(pstore_dump_path, "/", filename, NULL);
+        if (asprintf(&dump_file, "%s/%s", pstore_dump_path, filename) < 0) {
+                goto end;
+        }
+
         fp = fopen(dump_file, "r");
         if (fp == NULL) {
                 telem_log(LOG_ERR, "Failed to open pstore dump file %s:%s\n", dump_file, strerror(errno));
@@ -170,7 +171,7 @@ end:
                 fclose(fp);
         }
 
-        g_free(dump_file);
+        free(dump_file);
         return contents;
 }
 
@@ -199,8 +200,8 @@ int main(int argc, char **argv)
         DIR *pstore_dir;
         struct dirent *entry;
         struct chunk_list  *elem, *head;
-        GHashTableIter iter;
-        gpointer key, value;
+        NcHashmapIter iter;
+        void *key, *value;
         int max_part;
         size_t totalsize = 0;
         char *crash_dump = NULL;
@@ -209,7 +210,7 @@ int main(int argc, char **argv)
          * Hash table used to store chunks belonging to a oops counter
          *  key -> count, value->head of linked list of chunks
          */
-        GHashTable *hash = g_hash_table_new(g_int_hash, g_int_equal);
+        NcHashmap *hash = nc_hashmap_new(nc_simple_hash, nc_simple_compare);
 
         pstore_dir = opendir(pstore_dump_path);
         if (pstore_dir == NULL) {
@@ -229,9 +230,7 @@ int main(int argc, char **argv)
                 // Look for only dmesg logs. Ignore other types of pstore dumps for now.
                 if (sscanf(entry->d_name, "dmesg-efi-%" PRIu64, &id) == 1) {
                         parse_id(id, &count, &part);
-                        #ifdef DEBUG
-                        printf("Extracted count :%d, part : %d\n", count, part);
-                        #endif
+                        telem_debug("DEBUG: Extracted count :%d, part : %d\n", count, part);
                 } else {
                         continue;
                 }
@@ -247,24 +246,19 @@ int main(int argc, char **argv)
                 elem->contents = read_contents(entry->d_name, &(elem->size));
                 elem->next = NULL;
 
-                gpointer head = g_hash_table_lookup(hash, (void *)(&elem->count));
+                void *head = nc_hashmap_get(hash, NC_HASH_KEY(elem->count));
                 if (head != NULL) {
                         elem->next = (struct chunk_list *)head;
                 }
 
-                gint *key = g_new(gint, 1);
-                *key = elem->count;
-                g_hash_table_insert(hash, key, elem);
-
+                nc_hashmap_put(hash, NC_HASH_KEY(elem->count), elem);
         }
 
         closedir(pstore_dir);
 
-        g_hash_table_iter_init (&iter, hash);
-        while (g_hash_table_iter_next (&iter, &key, &value)) {
-                #ifdef DEBUG
-                printf("Count in the hash: %d\n", *((int *)key));
-                #endif
+        nc_hashmap_iter_init(hash, &iter);
+        while (nc_hashmap_iter_next(&iter, (void **)&key, (void **)&value)) {
+                telem_debug("DEBUG: Count in the hash: %d\n", NC_UNHASH_KEY(key));
 
                 head = (struct chunk_list *)value;
                 elem = head;

src/probes/python-probe.c

@@ -0,0 +1,293 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2019 Intel Corporation
+ *
+ *   This program is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#define _GNU_SOURCE
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/inotify.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <pwd.h>
+#include <grp.h>
+#include <getopt.h>
+
+#include "log.h"
+#include "telemetry.h"
+
+#define PYTHON_TELEMETRY_DIR "/var/lib/telemetry/python"
+
+/* We expect something like: 1.python-exception.3.N2Qy24 */
+static bool validate_file_name(const char *filename, uint32_t *severity,
+                               char** class, uint32_t *version)
+{
+        unsigned int ver, sev;
+        char cls[PATH_MAX] = { 0 };
+        char rnd[PATH_MAX] = { 0 };
+        char *filename_base = basename(filename);
+        char *str_class = NULL;
+
+        if (!filename_base) {
+                telem_perror("basename failed");
+                return false;
+        }
+
+        if (sscanf(filename_base,"%u.%[^]0-9.].%u.%s", &ver, cls, &sev, rnd) == 4) {
+                if (asprintf(&str_class, "org.clearlinux/python/%s", cls) < 29) {
+                        telem_log(LOG_ERR, "%s: invalid telemetry classification\n",
+                                  str_class);
+                        return false;
+                }
+                *class = str_class;
+                *version = ver;
+                *severity = sev;
+                return true;
+        }
+
+        return false;
+}
+
+static char *read_file_into_buffer(const char *filename)
+{
+        FILE *fp = NULL;
+        char *contents = NULL;
+        long sz;
+        size_t size, bytes_read;
+
+        fp = fopen(filename, "r");
+        if (fp == NULL) {
+                telem_log(LOG_ERR, "Failed to open Python telemetry file %s: %s\n",
+                          filename, strerror(errno));
+                return NULL;
+        }
+
+        if (fseek(fp, 0, SEEK_END) == -1) {
+                telem_log(LOG_ERR, "Failed to seek file %s: %s\n", filename,
+                          strerror(errno));
+                goto error_end;
+        }
+
+        sz = ftell(fp);
+        if (sz == -1L) {
+                telem_perror("ftell failed");
+                goto error_end;
+        }
+
+        if (sz == 0L) {
+                telem_perror("File is empty\n");
+                goto error_end;
+        }
+
+        if (fseek(fp, 0, SEEK_SET) < 0) {
+                telem_log(LOG_ERR, "fseek failed for file %s:%s\n", filename,
+                          strerror(errno));
+                goto error_end;
+        }
+
+        size = (size_t)sz;
+        contents = calloc(sizeof(char), size);
+        if (!contents) {
+                telem_log(LOG_ERR, "Call to calloc failed\n");
+                goto error_end;
+        }
+
+        bytes_read = fread(contents, sizeof(char), size, fp);
+
+        if (bytes_read < size) {
+                telem_log(LOG_ERR, "Error while reading file %s:%s\n", filename,
+                          strerror(errno));
+                goto error_end;
+        }
+
+        fclose(fp);
+        return contents;
+
+error_end:
+        fclose(fp);
+        if (contents) {
+                free(contents);
+        }
+        return NULL;
+}
+
+static void send_data(char *contents, uint32_t severity, char *class,
+                      uint32_t version)
+{
+        struct telem_ref *handle = NULL;
+        int ret;
+
+        if ((ret = tm_create_record(&handle, severity, class, version)) < 0) {
+                telem_log(LOG_ERR, "Failed to create record: %s\n",
+                          strerror(-ret));
+                return;
+        }
+
+        if ((ret = tm_set_payload(handle, contents)) < 0) {
+                telem_log(LOG_ERR, "Failed to set payload: %s\n",
+                          strerror(-ret));
+                tm_free_record(handle);
+                return;
+        }
+
+        if ((ret = tm_send_record(handle)) < 0) {
+                telem_log(LOG_ERR, "Failed to send record: %s\n",
+                          strerror(-ret));
+        }
+
+        tm_free_record(handle);
+}
+
+/* Send a valid file to the backend. */
+static void deliver_payload(const char *filename)
+{
+        uint32_t severity;
+        uint32_t version;
+        char *class = NULL;
+        char *contents = NULL;
+
+        if (validate_file_name(filename, &severity, &class, &version)) {
+                if ((contents = read_file_into_buffer(filename)) != NULL) {
+                        send_data(contents, severity, class, version);
+                        free(contents);
+                }
+
+                free(class);
+        }
+
+        /* Keep PYTHON_TELEMETRY_DIR empty. Delete the file. */
+        if (unlink(filename) != 0) {
+                telem_log(LOG_ERR, "Failed to unlink %s: %s\n", filename,
+                          strerror(errno));
+        }
+}
+
+static void drop_privs(void)
+{
+        uid_t euid;
+
+        euid = geteuid();
+        if (euid != 0) {
+                telem_log(LOG_DEBUG, "Not root; skipping privilege drop\n");
+                return;
+        }
+
+        struct passwd *pw;
+
+        pw = getpwnam("telemetry");
+        if (!pw) {
+                telem_log(LOG_ERR, "telemetry user not found\n");
+                exit(EXIT_FAILURE);
+        }
+
+        // The order is important here:
+        // change supplemental groups, our gid, and then our uid
+        if (initgroups(pw->pw_name, pw->pw_gid) != 0) {
+                telem_perror("Failed to set supplemental group list");
+                exit(EXIT_FAILURE);
+        }
+        if (setgid(pw->pw_gid) != 0) {
+                telem_perror("Failed to set GID");
+                exit(EXIT_FAILURE);
+        }
+        if (setuid(pw->pw_uid) != 0) {
+                telem_perror("Failed to set UID");
+                exit(EXIT_FAILURE);
+        }
+
+        assert(getuid() == pw->pw_uid);
+        assert(geteuid() == pw->pw_uid);
+        assert(getgid() == pw->pw_gid);
+        assert(getegid() == pw->pw_gid);
+}
+
+static void print_usage(char *prog)
+{
+        printf("%s: Usage\n", prog);
+        printf("  -f,  --config_file    Specify a configuration file other than default\n");
+        printf("  -h,  --help           Display this help message\n");
+        printf("  -V,  --version        Print the program version\n");
+}
+
+int main(int argc, char **argv)
+{
+        DIR *dir;
+        struct dirent *entry;
+        int c;
+        int opt_index = 0;
+
+        struct option opts[] = {
+                { "config_file", 1, NULL, 'f' },
+                { "help", 0, NULL, 'h' },
+                { "version", 0, NULL, 'V' },
+                { NULL, 0, NULL, 0 }
+        };
+
+        drop_privs();
+
+        while ((c = getopt_long(argc, argv, "f:hV", opts, &opt_index)) != -1) {
+                switch (c) {
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file"
+                                                  " path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                        case 'h':
+                                print_usage(argv[0]);
+                                exit(EXIT_SUCCESS);
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case '?':
+                                exit(EXIT_FAILURE);
+                }
+        }
+
+        if (chdir(PYTHON_TELEMETRY_DIR)) {
+                perror(PYTHON_TELEMETRY_DIR);
+                exit(EXIT_FAILURE);
+        }
+
+        /* Process all existing files */
+        dir = opendir(PYTHON_TELEMETRY_DIR);
+        if (!dir) {
+                perror(PYTHON_TELEMETRY_DIR);
+                exit(EXIT_FAILURE);
+        }
+
+        while (true) {
+                entry = readdir(dir);
+                if (!entry) {
+                        break;
+                }
+                if (entry->d_name[0] == '.') {
+                        continue;
+                }
+                deliver_payload(entry->d_name);
+        }
+
+        closedir(dir);
+        exit(EXIT_SUCCESS);
+}

src/probes/telem_record_gen.c

@@ -1,7 +1,7 @@
 /*
  * This program is part of the Clear Linux Project
  *
- * Copyright 2015 Intel Corporation
+ * Copyright 2019 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms and conditions of the GNU Lesser General Public License, as
@@ -15,6 +15,8 @@
  */
 
 #define _GNU_SOURCE
+#include <ctype.h>
+#include <getopt.h>
 #include <poll.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -22,135 +24,218 @@
 #include <string.h>
 #include <unistd.h>
 
-#include <glib.h>
 #include <string.h>
+#include <stdint.h>
 
 #include "config.h"
 #include "log.h"
 #include "common.h"
 #include "telemetry.h"
 
-static gchar *config_file = NULL;
-static gboolean version_p = FALSE;
-static guint severity = 1;
-static gchar *opt_class = NULL;
-static gchar *opt_payload = NULL;
-static guint payload_version = 1;
-static gchar *opt_payload_file = NULL;
+static uint32_t severity = 1;
+static char *opt_class = NULL;
+static char *opt_payload = NULL;
+static uint32_t payload_version = 1;
+static char *opt_payload_file = NULL;
+static char *opt_event_id = NULL;
+static bool opt_echo = false;
+static bool opt_nopost = false;
 
-static GOptionEntry options[] = {
-        { "config-file", 'f', 0, G_OPTION_ARG_FILENAME, &config_file,
-          "Path to configuration file (not implemented yet)", NULL },
-        { "version", 'V', 0, G_OPTION_ARG_NONE, &version_p,
-          "Print the program version", NULL },
-        { "severity", 's', 0, G_OPTION_ARG_INT, &severity,
-          "Severity level (1-4) - (default 1)", NULL },
-        { "class", 'c', 0, G_OPTION_ARG_STRING, &opt_class,
-          "Classification level_1/level_2/level_3", NULL },
-        { "payload", 'p', 0, G_OPTION_ARG_STRING, &opt_payload,
-          "Record body (max size = 8k)", NULL },
-        { "payload-file", 'P', 0, G_OPTION_ARG_STRING, &opt_payload_file,
-          "File to read payload from", NULL },
-        { "record-version", 'R', 0, G_OPTION_ARG_INT, &payload_version,
-          "Version number for format of payload (default 1)", NULL },
-        { NULL }
+static const struct option prog_opts[] = {
+        { "help", no_argument, 0, 'h' },
+        { "echo", no_argument, 0, 'o' },
+        { "no-post", no_argument, 0, 'n' },
+        { "version", no_argument, 0, 'V' },
+        { "severity", required_argument, 0, 's' },
+        { "class", required_argument, 0, 'c' },
+        { "payload", required_argument, 0, 'p' },
+        { "payload-file", required_argument, 0, 'P' },
+        { "record-version", required_argument, 0, 'R' },
+        { "event-id", required_argument, 0, 'e' },
+        { "config-file", required_argument, 0, 'f' },
+        { 0, 0, 0, 0 }
 };
 
-static void free_glib_strings(void)
+static void print_help(void)
 {
-        if (config_file) {
-                g_free(config_file);
-        }
-
-        if (opt_class) {
-                g_free(opt_class);
-        }
-
-        if (opt_payload) {
-                g_free(opt_payload);
-        }
+        printf("Usage:\n");
+        printf("  telem-record-gen [OPTIONS] - create and send a custom telemetry record\n");
+        printf("\n");
+        printf("Help Options:\n");
+        printf("  -h, --help            Show help options\n");
+        printf("\n");
+        printf("Application Options:\n");
+        printf("  -V, --version         Print the program version\n");
+        printf("  -s, --severity        Severity level (1-4) - (default 1)\n");
+        printf("  -c, --class           Classification level_1/level_2/level_3 (required)\n");
+        printf("  -p, --payload         Record body (max size = 8k) (required)\n");
+        printf("  -P, --payload-file    File to read payload from\n");
+        printf("  -R, --record-version  Version number for format of payload (default 1)\n");
+        printf("  -e, --event-id        Event id to use in the record\n");
+        printf("  -o, --echo            Echo record to stdout\n");
+        printf("  -n, --no-post         Do not post record just print\n");
+        printf("  -f, --config_file     Specify a configuration file other than default\n");
+        printf("\n");
 }
 
-int parse_options(int argc, char **argv)
+static bool parse_options(int argc, char **argv)
 {
-        int ret = 0;
-        GError *error = NULL;
-        GOptionContext *context;
+        bool ret = false;
+        char *endptr = NULL;
+        long unsigned int tmp = 0;
 
-        context = g_option_context_new(
-                "- create and send a custom telemetry record\n");
-        g_option_context_add_main_entries(context, options, NULL);
-        g_option_context_set_translate_func(context, NULL, NULL, NULL);
+        int opt;
+        while ((opt = getopt_long(argc, argv, "hc:Vs:c:p:P:R:e:onf:", prog_opts, NULL)) != -1) {
+                switch (opt) {
+                        case 'h':
+                                print_help();
+                                exit(EXIT_SUCCESS);
+                        case 'V':
+                                printf(PACKAGE_VERSION "\n");
+                                exit(EXIT_SUCCESS);
+                        case 's':
+                                errno = 0;
 
-        if (!g_option_context_parse(context, &argc, &argv, &error)) {
-                g_print("Failed to parse options: %s\n", error->message);
-                goto fail;
+                                tmp = strtoul(optarg, &endptr, 10);
+
+                                if (errno != 0) {
+                                        telem_perror("Failed to convert severity number");
+                                        goto fail;
+                                }
+                                if (endptr && *endptr != '\0') {
+                                        telem_log(LOG_ERR, "Invalid severity number. Must be an integer\n");
+                                        goto fail;
+                                }
+
+                                severity = (uint32_t)tmp;
+                                break;
+                        case 'c':
+                                opt_class = strdup(optarg);
+                                break;
+                        case 'p':
+                                opt_payload = strdup(optarg);
+                                break;
+                        case 'P':
+                                opt_payload_file = strdup(optarg);
+                                break;
+                        case 'R':
+                                errno = 0;
+
+                                tmp = strtoul(optarg, &endptr, 10);
+
+                                if (errno != 0) {
+                                        telem_perror("Failed to convert record-version number");
+                                        goto fail;
+                                }
+                                if (endptr && *endptr != '\0') {
+                                        telem_log(LOG_ERR, "Invalid record-version number. Must be an integer\n");
+                                        goto fail;
+                                }
+
+                                payload_version = (uint32_t)tmp;
+                                break;
+                        case 'e':
+                                opt_event_id = strdup(optarg);
+                                if (opt_event_id == NULL) {
+                                        goto fail;
+                                }
+                                break;
+                        case 'o':
+                                opt_echo = true;
+                                break;
+                        case 'n':
+                                opt_nopost = true;
+                                break;
+                        case 'f':
+                                if (tm_set_config_file(optarg) != 0) {
+                                    telem_log(LOG_ERR, "Configuration file path not valid\n");
+                                    exit(EXIT_FAILURE);
+                                }
+                                break;
+                }
         }
 
-        ret = 1;
-
+        ret = true;
 fail:
-        if (context) {
-                g_option_context_free(context);
-        }
-
         return ret;
 }
 
-int validate_opts(void)
+static bool validate_opts(void)
 {
-        size_t i;
-        int ret = 0;
+        size_t i, len;
+        int x, slashes = 0;
+        const char alphab[] = EVENT_ID_ALPHAB;
+        bool ret = false;
 
         /* classification */
         if (opt_class == NULL) {
                 fprintf(stderr, "Error: Classification required. See --help.\n");
                 return ret;
-
         }
-        i = strlen(opt_class);
 
-        if ((i == 0) || (i > 120)) {
+        len = strlen(opt_class);
+
+        if ((len == 0) || (len > MAX_CLASS_LENGTH)) {
                 fprintf(stderr, "Error: Valid size for classification "
-                        "is 1-120 chars\n");
+                        "is 1-%d chars\n", MAX_CLASS_LENGTH);
                 return ret;
         }
 
-        if (!g_str_is_ascii(opt_class)) {
-                fprintf(stderr, "Error: Non-ascii characters detected "
-                        "in classification - aborting\n");
+        for (int c = 0; c < len; c++) {
+                if (isascii(opt_class[c]) == 0) {
+                        fprintf(stderr, "Error: Non-ascii characters detected "
+                                "in classification - aborting\n");
+                        return ret;
+                }
+        }
+
+        for (i = 0, x = 0; i <= (len - 1); i++, x++) {
+                if (opt_class[i] == '/') {
+                        slashes++;
+                        x = 0;
+                } else {
+                        if (x > MAX_SUBCAT_LENGTH) {
+                                fprintf(stderr, "Error: Classification strings"
+                                        " between slashes should have at most"
+                                        " %d chars\n", MAX_SUBCAT_LENGTH);
+                                return ret;
+                        }
+                }
+        }
+
+        if (slashes != 2) {
+                fprintf(stderr, "Error: Classification needs to be in "
+                        "most/to/least specific format, 2 \'/\' required.\n");
                 return ret;
         }
 
+
         /* Severity */
         if ((severity) < 1 || (severity > 4)) {
                 fprintf(stderr, "Error: Valid range for severity is 1-4\n");
                 return ret;
         }
 
-        return 1;
-}
-
-int allocate_payload_buffer(char **payload)
-{
-        int ret = 0;
-
-        *payload = (char *)malloc(sizeof(gchar) * MAX_PAYLOAD_SIZE);
-
-        if (*payload == NULL) {
-                goto out1;
+        /* Event id  */
+        if (opt_event_id) {
+                size_t result = 0;
+                if ((result = strlen(opt_event_id)) != EVENT_ID_LEN) {
+                        fprintf(stderr, "Error: event_id length %zu it should have %d characters\n", result, EVENT_ID_LEN);
+                        return ret;
+                }
+                if ((result = strspn(opt_event_id, alphab)) != EVENT_ID_LEN) {
+                        fprintf(stderr, "Error: event_id contains invalid character '%c' at position %zu, valid characters are %s\n",
+                                opt_event_id[result], result, alphab);
+                        return ret;
+                }
         }
 
-        *payload = memset(*payload, 0, MAX_PAYLOAD_SIZE);
-        ret = 1;
-
-out1:
-        return ret;
+        return true;
 }
 
-int get_payload_from_file(char **payload)
+static bool get_payload_from_file(char **payload)
 {
-        int ret = 1;
+        bool ret = false;
         FILE *fp = NULL;
         size_t bytes_in = 0;
 
@@ -162,15 +247,15 @@ int get_payload_from_file(char **payload)
                 goto out;
         }
 
-        bytes_in = fread(*payload, sizeof(gchar),
-                         MAX_PAYLOAD_SIZE - 1, fp);
+        bytes_in = fread(*payload, 1,
+                         MAX_PAYLOAD_LENGTH - 1, fp);
 
         /* if fread fails */
         if (bytes_in == 0 && ferror(fp) != 0) {
-                ret = 0;
                 goto out;
         }
 
+        ret = true;
 out:
         if (fp) {
                 fclose(fp);
@@ -179,25 +264,27 @@ out:
 
 }
 
-void get_payload_from_opt(char **payload)
+static void get_payload_from_opt(char **payload)
 {
         size_t len = 0;
 
         len = strlen(opt_payload);
-        if (len >= MAX_PAYLOAD_SIZE) {
-                len = MAX_PAYLOAD_SIZE - 1;
+        if (len >= MAX_PAYLOAD_LENGTH) {
+                len = MAX_PAYLOAD_LENGTH - 1;
         }
-        strncpy(*payload, opt_payload, len);
+
+        /* "payload" is pre-allocated zeroed buffer of MAX_PAYLOAD_LENGTH */
+        memcpy(*payload, opt_payload, len);
 }
 
-void get_payload_from_stdin(char **payload)
+static void get_payload_from_stdin(char **payload)
 {
         size_t bytes_in = 0;
         int c;
 
-        bytes_in = fread(*payload, sizeof(gchar), MAX_PAYLOAD_SIZE - 1, stdin);
+        bytes_in = fread(*payload, 1, MAX_PAYLOAD_LENGTH - 1, stdin);
 
-        if (bytes_in == MAX_PAYLOAD_SIZE - 1) {
+        if (bytes_in == MAX_PAYLOAD_LENGTH - 1) {
                 /* Throw away the rest of stdin */
                 while (EOF != (c = fgetc(stdin))) {
                         continue;
@@ -206,60 +293,92 @@ void get_payload_from_stdin(char **payload)
 
 }
 
-int get_payload(char **payload)
+static bool get_payload(char **payload)
 {
-        int ret = 0;
+        bool ret = false;
 
-        if (!allocate_payload_buffer(payload)) {
-                goto out1;
-        }
+        *payload = (char *)calloc(sizeof(char), MAX_PAYLOAD_LENGTH);
+
+        if (*payload == NULL)
+                return false;
 
         if (opt_payload_file) {
 
                 if (get_payload_from_file(payload)) {
-                        ret = 1;
+                        ret = true;
                 }
 
         } else if (opt_payload) {
 
                 get_payload_from_opt(payload);
-                ret = 1;
+                ret = true;
 
         } else {
                 get_payload_from_stdin(payload);
-                ret = 1;
+                ret = true;
         }
 
-        if (ret == 0) {
+        if (ret == false) {
                 free(*payload);
                 *payload = NULL;
         }
 
+        return ret;
+}
+
+static int instanciate_record(struct telem_ref **t_ref, char *payload)
+{
+        int ret = 0;
+
+        if ((ret = tm_create_record(t_ref, (uint32_t)severity,
+                                    opt_class, payload_version)) < 0) {
+                goto out1;
+        }
+
+        if ((ret = opt_event_id && tm_set_event_id(*t_ref, opt_event_id)) < 0) {
+                goto out1;
+        }
+
+        if ((ret = tm_set_payload(*t_ref, payload)) < 0) {
+                goto out1;
+        }
 out1:
         return ret;
 }
 
-int send_record(char *payload)
+static int send_record(char *payload)
 {
-        struct telem_ref *t_ref;
+        struct telem_ref *t_ref = NULL;
         int ret = 0;
 
-        if (tm_create_record(&t_ref, (uint32_t)severity,
-                             opt_class, payload_version) < 0) {
-                goto out1;
+        if ((ret = instanciate_record(&t_ref, payload)) < 0) {
+                goto out;
         }
 
-        if (tm_set_payload(t_ref, payload) < 0) {
-                goto out2;
+        if ((ret = tm_send_record(t_ref)) < 0) {
+                goto out;
         }
 
-        if (tm_send_record(t_ref) >= 0) {
-                ret = 1;
-        }
-
-out2:
+out:
         tm_free_record(t_ref);
-out1:
+
+        return ret;
+}
+
+static int print_record(char *payload)
+{
+        struct telem_ref *t_ref = NULL;
+        int ret = 0;
+        int i = 0;
+
+        if ((ret = instanciate_record(&t_ref, payload)) == 0) {
+                for (i = 0; i < NUM_HEADERS; i++) {
+                        fprintf(stdout, "%s", t_ref->record->headers[i]);
+                }
+                fprintf(stdout, "%s\n", t_ref->record->payload);
+                tm_free_record(t_ref);
+        }
+
         return ret;
 }
 
@@ -272,11 +391,6 @@ int main(int argc, char **argv)
                 goto fail;
         }
 
-        if (version_p) {
-                g_print(PACKAGE_VERSION "\n");
-                goto success;
-        }
-
         if (!validate_opts()) {
                 goto fail;
         }
@@ -285,17 +399,22 @@ int main(int argc, char **argv)
                 goto fail;
         }
 
-        if (!send_record(payload)) {
+        if (opt_echo == true) {
+                print_record(payload);
+        }
+
+        if (opt_nopost == false && send_record(payload) != 0) {
                 goto fail;
         }
 
-success:
         ret = EXIT_SUCCESS;
 fail:
-        free_glib_strings();
+        free(opt_class);
+        free(opt_payload);
+        free(opt_event_id);
+        free(payload);
 
         return ret;
 }
 
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */
-

src/retention.c

@@ -0,0 +1,46 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+
+#include "log.h"
+#include "common.h"
+
+int delete_record_by_id(char *record_id)
+{
+        int ret = 0;
+        char *record_path = NULL;
+
+        ret = asprintf(&record_path, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (ret == -1) {
+                return 1;
+        }
+        ret = unlink(record_path);
+        if (ret == -1) {
+                telem_perror("Error deleting saved record");
+        }
+        free(record_path);
+
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/retention.h

@@ -0,0 +1,27 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+/**
+ * Delete record identified by record unique id
+ *
+ * @param record_id Unique identifier of entry
+ *
+ */
+int delete_record_by_id(char *record_id);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/spool.c

@@ -27,7 +27,7 @@
 #include <errno.h>
 
 #include "spool.h"
-#include "telemdaemon.h"
+#include "telempostdaemon.h"
 #include "log.h"
 #include "configuration.h"
 #include "util.h"
@@ -202,9 +202,15 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
         FILE *fp = NULL;
         char *headers[NUM_HEADERS];
         char *payload = NULL;
-        int i, k;
-        char line[LINE_MAX] = { 0 };
+        int num_headers = 0, k;
+#if (LINE_MAX > PATH_MAX)
+        char line[LINE_MAX+1] = { 0 };
+#else
+        char line[PATH_MAX+1] = { 0 };
+#endif
         long offset;
+        char *cfg_file = NULL;
+        uint32_t cfg_prefix = 0;
 
         fp = fopen(record_path, "r");
         if (fp == NULL) {
@@ -212,16 +218,47 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
                 return;
         }
 
-        for (i = 0; i < NUM_HEADERS; i++) {
-                const char *header_name = get_header_name(i);
-                if (!fgets(line, LINE_MAX, fp)) {
+        // First line optionally contains configuration file path
+        if (fread(&cfg_prefix, CFG_PREFIX_LENGTH, 1, fp) != 1) {
+                telem_log(LOG_ERR, "Error while parsing spooled record configuration info.\n");
+                goto read_error;
+        }
+
+        if (cfg_prefix == CFG_PREFIX_32BIT) {
+                size_t pathlen;
+                char *nl;
+
+                if (!fgets(line, sizeof(line), fp)) {
                         telem_log(LOG_ERR, "Error while parsing record file\n");
-                        fclose(fp);
-                        return;
+                        goto read_error;
+                }
+
+                if ((nl = strchr(line, '\n')) != NULL) {
+                        *nl = '\0';
+                }
+
+                pathlen = strlen(line);
+                cfg_file = malloc(pathlen + 1);
+                if (cfg_file == NULL) {
+                        telem_log(LOG_ERR, "Could not allocate memory for config file path\n");
+                        goto read_error;
+                }
+                strcpy (cfg_file, line);
+                telem_debug("DEBUG: cfg_file: %s\n", cfg_file);
+        } else {
+                cfg_file = NULL;
+                rewind(fp);
+        }
+
+        for (num_headers = 0; num_headers < NUM_HEADERS; num_headers++) {
+                const char *header_name = get_header_name(num_headers);
+                if (!fgets(line, sizeof(line), fp)) {
+                        telem_log(LOG_ERR, "Error while parsing record file\n");
+                        goto read_error;
                 }
                 //Get rid of trailing newline
                 strtok(line, "\n");
-                if (get_header(line, header_name, &headers[i], strlen(header_name))) {
+                if (get_header(line, header_name, &headers[num_headers])) {
                         continue;
                 } else {
                         telem_log(LOG_ERR, "transmit_spooled_record: Incorrect"
@@ -242,8 +279,7 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
 
         if (!payload) {
                 telem_log(LOG_ERR, "Could not allocate memory for payload\n");
-                fclose(fp);
-                return;
+                goto read_error;
         }
         memset(payload, 0, (size_t)size);
 
@@ -254,7 +290,7 @@ void transmit_spooled_record(char *record_path, bool *post_succeeded, long size)
                 goto read_error;
         }
 
-        *post_succeeded = post_record_http(headers, payload, false);
+        *post_succeeded = post_record_http(headers, payload, cfg_file);
         if (*post_succeeded) {
                 unlink(record_path);
         }
@@ -267,9 +303,13 @@ read_error:
                 fclose(fp);
         }
 
-        for (k = 0; k < i; k++) {
+        for (k = 0; k < num_headers; k++) {
                 free(headers[k]);
         }
+
+        if (cfg_file) {
+                free(cfg_file);
+        }
 }
 
 int spool_record_compare(const void *entrya, const void *entryb, void *path)

src/telemdaemon.c

@@ -27,38 +27,23 @@
 #include <fcntl.h>
 #include <time.h>
 #include <malloc.h>
+#include <sys/uio.h>
 
+#include "iorecord.h"
 #include "telemdaemon.h"
 #include "common.h"
 #include "util.h"
 #include "log.h"
 #include "configuration.h"
 
-void initialize_daemon(TelemDaemon *daemon)
+void initialize_probe_daemon(TelemDaemon *daemon)
 {
         client_list_head head;
         LIST_INIT(&head);
         daemon->nfds = 0;
         daemon->pollfds = NULL;
         daemon->client_head = head;
-        daemon->bypass_http_post_ts = 0;
-        daemon->is_spool_valid = false;
-        initialize_rate_limit(daemon);
-        daemon->current_spool_size = 0;
-}
-
-void initialize_rate_limit(TelemDaemon *daemon)
-{
-        for (int i = 0; i < TM_RATE_LIMIT_SLOTS; i++) {
-                daemon->record_burst_array[i] = 0;
-                daemon->byte_burst_array[i] = 0;
-        }
-        daemon->rate_limit_enabled = rate_limit_enabled_config();
-        daemon->record_burst_limit = record_burst_limit_config();
-        daemon->record_window_length = record_window_length_config();
-        daemon->byte_burst_limit = byte_burst_limit_config();
-        daemon->byte_window_length = byte_window_length_config();
-        daemon->rate_limit_strategy = rate_limit_strategy_config();
+        daemon->machine_id_override = NULL;
 }
 
 client *add_client(client_list_head *client_head, int fd)
@@ -180,7 +165,7 @@ bool handle_client(TelemDaemon *daemon, nfds_t ind, client *cl)
                         cl->offset = 0;
                         cl->size = RECORD_SIZE_LEN;
                         processed = true;
-                        telem_log(LOG_DEBUG, "Record processed for client %d\n", cl->fd);
+                        telem_debug("DEBUG: Record processed for client %d\n", cl->fd);
                         break;
                 }
         } while (len > 0);
@@ -262,33 +247,97 @@ void machine_id_replace(char **machine_header, char *machine_id_override)
         free(old_header);
 }
 
+static void stage_record(char *filepath, char *headers[], char *body, char *cfg_file)
+{
+        int tmpfd;
+        FILE *tmpfile = NULL;
+
+        telem_debug("DEBUG: filepath:%s\n", filepath);
+        telem_debug("DEBUG: body:%s\n", body);
+        telem_debug("DEBUG: cfg:%s\n", cfg_file);
+        // Use default path if not provided
+        if (filepath == NULL) {
+                telem_log(LOG_ERR, "filepath value must be provided, aborting\n");
+                exit(EXIT_FAILURE);
+        }
+
+        tmpfd = mkstemp(filepath);
+        if (!tmpfd) {
+                telem_perror("Error opening staging file");
+                close(tmpfd);
+                if (unlink(filepath)) {
+                        telem_perror("Error deleting staging file");
+                }
+                goto clean_exit;
+        }
+
+        // open file
+        tmpfile = fdopen(tmpfd, "a");
+        if (!tmpfile) {
+                telem_perror("Error opening temp stage file");
+                close(tmpfd);
+                if (unlink(filepath)) {
+                        telem_perror("Error deleting temp stage file");
+                }
+                goto clean_exit;
+        }
+
+        // write cfg info if exists
+        if (cfg_file != NULL) {
+                fprintf(tmpfile, "%s%s\n", CFG_PREFIX, cfg_file);
+        }
+
+        // write headers
+        for (int i = 0; i < NUM_HEADERS; i++) {
+                fprintf(tmpfile, "%s\n", headers[i]);
+        }
+
+        //write body
+        fprintf(tmpfile, "%s\n", body);
+        fflush(tmpfile);
+        fclose(tmpfile);
+
+clean_exit:
+
+        return;
+}
+
 void process_record(TelemDaemon *daemon, client *cl)
 {
         int i = 0;
+        int ret = 0;
         char *headers[NUM_HEADERS];
-        bool do_spool = false;
-        bool record_sent = false;
         char *tok = NULL;
         size_t header_size = 0;
         size_t message_size = 0;
         char *temp_headers = NULL;
         char *msg;
         char *body;
-        int current_minute = 0;
-        bool record_check_passed = true;
-        bool byte_check_passed = true;
-        bool record_burst_enabled = true;
-        bool byte_burst_enabled = true;
+        char *recordpath = NULL;
+        char *cfg_file = NULL;;
+        size_t cfg_info_size = 0;
+        uint8_t *buf;
 
-        /* Gets the current minute of time */
-        time_t temp = time(NULL);
-        struct tm *tm_s = localtime(&temp);
-        current_minute = tm_s->tm_min;
+        buf = cl->buf;
 
-        header_size = *(uint32_t *)cl->buf;
-        message_size = cl->size - header_size;
+        /* Check for an optional CFG_PREFIX in the first 32 bits */
+        if (*(uint32_t *)buf == CFG_PREFIX_32BIT) {
+                char *cfg  = (char *)cl->buf;
+
+                cfg_file = cfg + CFG_PREFIX_LENGTH;
+                cfg_info_size = CFG_PREFIX_LENGTH + strlen(cfg_file) + 1;
+                telem_debug("DEBUG: cfg_file: %s\n", cfg_file);
+        }
+
+        buf += cfg_info_size;
+        header_size = *(uint32_t *)buf;
+        message_size = cl->size - (cfg_info_size + header_size);
+        telem_debug("DEBUG: cl->size: %zu\n", cl->size);
+        telem_debug("DEBUG: header_size: %zu\n", header_size);
+        telem_debug("DEBUG: message_size: %zu\n", message_size);
+        telem_debug("DEBUG: cfg_info_size: %zu\n", cfg_info_size);
         assert(message_size > 0);      //TODO:Check for min and max limits
-        msg = (char *)cl->buf + sizeof(uint32_t);
+        msg = (char *)buf + sizeof(uint32_t);
 
         /* Copying the headers as strtok modifies the orginal buffer */
         temp_headers = strndup(msg, header_size);
@@ -297,7 +346,7 @@ void process_record(TelemDaemon *daemon, client *cl)
         for (i = 0; i < NUM_HEADERS; i++) {
                 const char *header_name = get_header_name(i);
 
-                if (get_header(tok, header_name, &headers[i], strlen(header_name))) {
+                if (get_header(tok, header_name, &headers[i])) {
                         if (strcmp(header_name, TM_MACHINE_ID_STR) == 0) {
                                 machine_id_replace(&headers[i], daemon->machine_id_override);
                         }
@@ -311,67 +360,15 @@ void process_record(TelemDaemon *daemon, client *cl)
         /* TODO : check if the body is within the limits. */
         body = msg + header_size;
 
-        if (inside_direct_spool_window(daemon, time(NULL))) {
-                telem_log(LOG_INFO, "process_record: delivering directly to spool\n");
-                spool_record(daemon, headers, body);
-                goto end;
-        }
-        if (daemon->record_window_length == -1 ||
-            daemon->byte_window_length == -1) {
-                telem_log(LOG_ERR, "Invalid value for window length\n");
+        /* Save record to stage */
+        ret = asprintf(&recordpath, "%s/XXXXXX", spool_dir_config());
+        if (ret == -1) {
+                telem_log(LOG_ERR, "Failed to allocate memory for record name in staging folder, aborting\n");
                 exit(EXIT_FAILURE);
         }
-        /* Checks if entirety of rate limiting is enabled */
-        if (daemon->rate_limit_enabled) {
-                /* Checks whether record and byte bursts are enabled individually */
-                record_burst_enabled = burst_limit_enabled(daemon->record_burst_limit);
-                byte_burst_enabled = burst_limit_enabled(daemon->byte_burst_limit);
-
-                if (record_burst_enabled) {
-                        record_check_passed = rate_limit_check(current_minute,
-                                                               daemon->record_burst_limit, daemon->record_window_length,
-                                                               daemon->record_burst_array, TM_RECORD_COUNTER);
-                }
-                if (byte_burst_enabled) {
-                        byte_check_passed = rate_limit_check(current_minute,
-                                                             daemon->byte_burst_limit, daemon->byte_window_length,
-                                                             daemon->byte_burst_array, cl->size);
-                }
-                /* If both record and byte burst disabled, rate limiting disabled */
-                if (!record_burst_enabled && !byte_burst_enabled) {
-                        daemon->rate_limit_enabled = false;
-                }
-        }
-        /* Sends record if rate limiting is disabled, or all checks passed */
-        if (!daemon->rate_limit_enabled || (record_check_passed && byte_check_passed)) {
-                /* Send the record as https post */
-                record_sent = post_record_ptr(headers, body, true);
-        }
-
-        // Get rate-limit strategy
-        do_spool = spool_strategy_selected(daemon);
-
-        // Drop record
-        if (!record_sent && !do_spool) {
-                goto end;
-        }
-        // Spool Record
-        else if (!record_sent && do_spool) {
-                start_network_bypass(daemon);
-                telem_log(LOG_INFO, "process_record: initializing direct-spool window\n");
-                spool_record(daemon, headers, body);
-        } else {
-                /* Updates rate limiting arrays if record sent */
-                if (record_burst_enabled) {
-                        rate_limit_update (current_minute, daemon->record_window_length,
-                                           daemon->record_burst_array, TM_RECORD_COUNTER);
-                }
-                if (byte_burst_enabled) {
-                        rate_limit_update (current_minute, daemon->byte_window_length,
-                                           daemon->byte_burst_array, cl->size);
-                }
-        }
 
+        stage_record(recordpath, headers, body, cfg_file);
+        free(recordpath);
 end:
         free(temp_headers);
         for (int k = 0; k < i; k++)
@@ -379,234 +376,6 @@ end:
         return;
 }
 
-bool burst_limit_enabled(int64_t burst_limit)
-{
-        return (burst_limit > -1) ? true : false;
-}
-
-bool rate_limit_check(int current_minute, int64_t burst_limit, int
-                      window_length, size_t *array, size_t incValue)
-{
-        size_t count = 0;
-
-        /* Variable to determine last minute allowed in burst window */
-        int window_start = (TM_RATE_LIMIT_SLOTS + (current_minute - window_length + 1))
-                           % TM_RATE_LIMIT_SLOTS;
-
-        /* The modulo is not placed in the for loop inself, because if
-         * the current minute is less than the window start (from wrapping)
-         * the for loop will never be entered.
-         */
-
-        /* Counts all elements within burst_window in array */
-        for (int i = window_start; i < (window_start + window_length); i++) {
-                count += array[i % TM_RATE_LIMIT_SLOTS];
-
-                if ((array[i % TM_RATE_LIMIT_SLOTS] + incValue) >= SIZE_MAX) {
-                        /* Exceeds maximum size array can hold */
-                        return false;
-                }
-        }
-        /* Include current record being processed to count */
-        count += incValue;
-
-        /* Determines whether the count has exceeded the limit or not */
-        return (count > burst_limit) ? false : true;
-}
-
-void rate_limit_update(int current_minute, int window_length, size_t *array,
-                       size_t incValue)
-{
-        /* Variable to determine the amount of zero'd out spots needed */
-        int blank_slots = (TM_RATE_LIMIT_SLOTS - window_length);
-
-        /* Update specified array depending on increment value */
-        array[current_minute] += incValue;
-
-        /* Zero out expired records for record array */
-        for (int i = current_minute + 1; i < (blank_slots + current_minute + 1); i++) {
-                array[i % TM_RATE_LIMIT_SLOTS] = 0;
-        }
-}
-
-bool spool_strategy_selected(TelemDaemon *daemon)
-{
-        /* Performs action depending on strategy choosen */
-        return (strcmp(daemon->rate_limit_strategy, "spool") == 0) ? true : false;
-}
-
-bool inside_direct_spool_window(TelemDaemon *daemon, time_t current_time)
-{
-        return (current_time < daemon->bypass_http_post_ts + 1800) ? true : false;
-}
-
-void start_network_bypass(TelemDaemon *daemon)
-{
-        daemon->bypass_http_post_ts = time(NULL);
-}
-
-size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
-{
-        telem_log(LOG_DEBUG, "Received data:\n%s\n", ptr);
-        return size * nmemb;
-}
-
-bool post_record_http(char *headers[], char *body, bool spool)
-{
-        CURL *curl;
-        int res = 0;
-        char *content = "Content-Type: application/text";
-        struct curl_slist *custom_headers = NULL;
-        char errorbuf[CURL_ERROR_SIZE];
-        long http_response = 0;
-        const char *cert_file = get_cainfo_config();
-        const char *tid_header = get_tidheader_config();
-
-        curl = curl_easy_init();
-        if (!curl) {
-                telem_log(LOG_ERR, "curl_easy_init(): Unable to start libcurl"
-                          " easy session, exiting\n");
-                exit(EXIT_FAILURE);
-                /* TODO: check if memory needs to be released */
-        }
-
-        // Errors for any curl_easy_* functions will store nice error messages
-        // in errorbuf, so send log messages with errorbuf contents
-        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf);
-
-        curl_easy_setopt(curl, CURLOPT_URL, server_addr_config());
-        curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 5L);
-        curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
-        curl_easy_setopt(curl, CURLOPT_POST, 1);
-#ifdef DEBUG
-        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
-#endif
-
-        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
-
-        for (int i = 0; i < NUM_HEADERS; i++) {
-                custom_headers = curl_slist_append(custom_headers, headers[i]);
-        }
-        custom_headers = curl_slist_append(custom_headers, tid_header);
-        // This should be set by probes/libtelemetry in the future
-        custom_headers = curl_slist_append(custom_headers, content);
-
-        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, custom_headers);
-        curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body);
-        curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strlen(body));
-        curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);
-
-        if (strlen(cert_file) > 0) {
-                if (access(cert_file, F_OK) != -1) {
-                        curl_easy_setopt(curl, CURLOPT_CAINFO, cert_file);
-                        telem_log(LOG_INFO, "cafile was set to %s\n", cert_file);
-                }
-        }
-
-        telem_log(LOG_DEBUG, "Executing curl operation...\n");
-        errorbuf[0] = 0;
-        res = curl_easy_perform(curl);
-        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_response);
-
-        if (res) {
-                size_t len = strlen(errorbuf);
-                if (len) {
-                    telem_log(LOG_ERR, "Failed sending record: %s%s", errorbuf,
-                              ((errorbuf[len - 1] != '\n') ? "\n" : ""));
-                } else {
-                    telem_log(LOG_ERR, "Failed sending record: %s\n",
-                              curl_easy_strerror(res));
-                }
-        } else if (http_response != 201 && http_response != 200) {
-                /*  201 means the record was  successfully created
-                 *  200 is a generic "ok".
-                 *  TODO: Make list of white-listed response codes configurable?
-                 */
-                telem_log(LOG_ERR, "Encountered error %ld on the server\n",
-                          http_response);
-                // We treat HTTP error codes the same as libcurl errors
-                res = 1;
-        } else {
-                telem_log(LOG_INFO, "Record sent successfully\n");
-        }
-
-        curl_slist_free_all(custom_headers);
-        curl_easy_cleanup(curl);
-        return res ? false : true;
-}
-
-void spool_record(TelemDaemon *daemon, char *headers[], char *body)
-{
-        int ret = 0;
-        struct stat stat_buf;
-        int64_t max_spool_size = 0;
-        char *tmpbuf = NULL;
-        FILE *tmpfile = NULL;
-        int tmpfd = 0;
-
-        if (!daemon->is_spool_valid) {
-                /* If spool is not valid, simply drop the record */
-                return;
-        }
-
-        //check if the size is greater than 1 MB/spool max size
-        max_spool_size = spool_max_size_config();
-        if (max_spool_size != -1) {
-                telem_log(LOG_DEBUG, "Total size of spool dir: %ld\n", daemon->current_spool_size);
-                if (daemon->current_spool_size < 0) {
-                        telem_log(LOG_ERR, "Error getting spool directory size: %s\n",
-                                  strerror(-(int)(daemon->current_spool_size)));
-                        return;
-                } else if (daemon->current_spool_size >= (max_spool_size * 1024)) {
-                        telem_log(LOG_INFO, "Spool dir full, dropping record\n");
-                        return;
-                }
-        }
-
-        // create file with record
-        ret = asprintf(&tmpbuf, "%s/XXXXXX", spool_dir_config());
-        if (ret == -1) {
-                telem_log(LOG_ERR, "Failed to allocate memory for record name, aborting\n");
-                exit(EXIT_FAILURE);
-        }
-
-        tmpfd = mkstemp(tmpbuf);
-        if (tmpfd == -1) {
-                telem_perror("Error while creating temp file");
-                goto spool_err;
-        }
-
-        //fp = fopen(spool_dir_path, const char *mode);
-        tmpfile = fdopen(tmpfd, "a");
-        if (!tmpfile) {
-                telem_perror("Error opening temp file");
-                close(tmpfd);
-                if (unlink(tmpbuf)) {
-                        telem_perror("Error deleting temp file");
-                }
-                goto spool_err;
-        }
-
-        // write headers
-        for (int i = 0; i < NUM_HEADERS; i++) {
-                fprintf(tmpfile, "%s\n", headers[i]);
-        }
-
-        //write body
-        fprintf(tmpfile, "%s\n", body);
-        fflush(tmpfile);
-        fclose(tmpfile);
-
-        /* The stat should not fail here unless it is ENOMEM */
-        if (stat(tmpbuf, &stat_buf) == 0) {
-                daemon->current_spool_size += (stat_buf.st_blocks * 512);
-        }
-
-spool_err:
-        free(tmpbuf);
-        return;
-}
-
 void add_pollfd(TelemDaemon *daemon, int fd, short events)
 {
         assert(daemon);
@@ -671,7 +440,7 @@ bool get_machine_id(char *machine_id)
         return true;
 }
 
-int machine_id_write(uint64_t upper, uint64_t lower)
+int machine_id_write(char *new_id)
 {
         FILE *fp;
         int ret = 0;
@@ -683,7 +452,7 @@ int machine_id_write(uint64_t upper, uint64_t lower)
                 goto file_error;
         }
 
-        ret = fprintf(fp, "%.16" PRIx64 "%.16" PRIx64, upper, lower);
+        ret = fprintf(fp, "%s", new_id);
         if (ret < 0) {
                 telem_perror("Unable to write to machine id file");
                 goto file_error;
@@ -700,49 +469,17 @@ file_error:
 
 int generate_machine_id(void)
 {
-        int frandom = -1;
-        struct stat stat_buf;
-        int ret = 0;
-        struct iovec random_id[2];
-        uint64_t random_id_upper;
-        uint64_t random_id_lower;
-        int result = -1;
+        int result = 0;
+        char *new_id = NULL;
 
-        ret = stat("/dev/urandom", &stat_buf);
-        if (ret == -1) {
-                telem_log(LOG_ERR, "Unable to stat urandom device\n");
+        if ((result = get_random_id(&new_id)) != 0) {
                 goto rand_err;
         }
 
-        if (!S_ISCHR(stat_buf.st_mode)) {
-                telem_log(LOG_ERR, "/dev/urandom is not a character device file\n");
-                goto rand_err;
-        }
-
-        /* TODO : check for major and minor numbers to be extra sure ??
-           ((stat_buffer.st_rdev == makedev(1, 8)) || (stat_buffer.st_rdev == makedev(1, 9))*/
-
-        frandom = open("/dev/urandom", O_RDONLY);
-        if (frandom < 0) {
-                telem_perror("Error opening /dev/urandom");
-                goto rand_err;
-        }
-
-        random_id[0].iov_base = &random_id_upper;
-        random_id[0].iov_len = 8;
-        random_id[1].iov_base = &random_id_lower;
-        random_id[1].iov_len = 8;
-
-        if (readv(frandom, random_id, 2) < 0) {
-                telem_perror("Error while reading /dev/urandom");
-                goto rand_err;
-        }
-
-        result = machine_id_write(random_id_upper, random_id_lower);
+        result = machine_id_write(new_id);
 rand_err:
-        if (frandom >= 0) {
-                close(frandom);
-        }
+        free(new_id);
+
         return result;
 }
 

src/telemdaemon.h

@@ -54,23 +54,6 @@ typedef struct TelemDaemon {
         size_t current_alloc;
         /* client list head */
         client_list_head client_head;
-        /* Time of last failed post */
-        time_t bypass_http_post_ts;
-
-        /* Rate limit record and byte arrays */
-        size_t record_burst_array[TM_RATE_LIMIT_SLOTS];
-        size_t byte_burst_array[TM_RATE_LIMIT_SLOTS];
-        /* Rate Limit Configurations */
-        bool rate_limit_enabled;
-        int64_t record_burst_limit;
-        int record_window_length;
-        int64_t byte_burst_limit;
-        int byte_window_length;
-        const char *rate_limit_strategy;
-
-        bool is_spool_valid;
-        long current_spool_size;
-
         char *machine_id_override;
 } TelemDaemon;
 
@@ -79,14 +62,7 @@ typedef struct TelemDaemon {
  *
  * @param daemon A pointer to the daemon structure.
  */
-void initialize_daemon(TelemDaemon *daemon);
-
-/**
- * Initialize rate-limiting in the daemon
- *
- * @param daemon A pointer to the daemon structure
- */
-void initialize_rate_limit(TelemDaemon *daemon);
+void initialize_probe_daemon(TelemDaemon *daemon);
 
 /**
  * Add poll fd struct to the array of pollfds.
@@ -170,105 +146,6 @@ void terminate_client(TelemDaemon *daemon, client *cl, nfds_t index);
  */
 void process_record(TelemDaemon *daemon, client *cl);
 
-/**
- * Check if burst limiting is enabled
- *
- * @param burst_limit Burst limit amount. If negative rate limiting
- * is disabled
- *
- * @return true if rate limiting for burst_limit variable is enabled
- */
-bool burst_limit_enabled(int64_t burst_limit);
-
-/**
- * Checks whether record does not exceed the burst threshold
- *
- * @param current_minute The current minute of time the record processed
- * @param burst_limit Amount of records that can be passed within a window
- * @param window_length Window of time burst limit refers to
- * @param array Pointer to array that stores record or byte count
- * @param incValue Amount to increase array slot by
- *
- * @return true if record does not exceed threshold, and false otherwise
- */
-bool rate_limit_check(int current_minute, int64_t burst_limit, int
-                      window_length, size_t *array, size_t incValue);
-
-/**
- * Updates rate limit arrays with passing record information
- *
- * @param current_minute The current minute of time the record processed
- * @param window_length Window of time burst limit refers to
- * @param array Pointer to array that stores record or byte count
- * @param incValue Amount to increase array slot by
- *
- */
-void rate_limit_update(int current_minute, int window_length,
-                       size_t *array, size_t incValue);
-
-/**
- * Checks if the spool strategy is chosen for rate-limiting
- *
- * @param daemon Pointer to the daemon
- *
- * @return true if spool is choosen false if drop is choosen
- */
-bool spool_strategy_selected(TelemDaemon *daemon);
-
-/**
- * Check if daemon should direct-spool after encountering network issues.
- *
- * @param daemon Pointer to the daemon
- * @param time The time as time_t (usually from time())
- *
- * @return true if inside direct spool window, false otherwise.
- */
-bool inside_direct_spool_window(TelemDaemon *daemon, time_t time);
-
-/**
- * Set the start time of the direct spool window. Will use the
- * current time.
- *
- * @param daemon Pointer to the daemon
- *
- * @return no return value.
- */
-void start_network_bypass(TelemDaemon *daemon);
-
-/**
- * Function pointer for posting the records to the server. Replaced by a
- * mock function in the unit tests.
- *
- * @param header_values The header data read from the socket
- * @param body The payload of the record read
- * @param spool Indicate whether the record should be spooled if not
- *    successfully delivered
- *
- * @return true if the record was successfully posted
- */
-extern bool (*post_record_ptr)(char *headers[], char *body, bool spool);
-
-/**
- * Post records on a http connection to server
- *
- * @param header_values The header data read from the socket
- * @param body The payload of the record read
- * @param spool Indicate whether the record should be spooled if not
- *    successfully delivered
- *
- * @return true if the record was successfully posted
- */
-bool post_record_http(char *header_values[], char *body, bool spool);
-
-/**
- * Spool the record in case it cannot be delivered to the server
- *
- * @param headers The headers for the record
- * @param body The payload of the record
- *
- */
-void spool_record(TelemDaemon *daemon, char *headers[], char *body);
-
 /**
  * Get random machine id stored in file
  *
@@ -290,4 +167,5 @@ int update_machine_id(void);
  * or if the file is empty
  */
 char *read_machine_id_override(void);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/telemetry.c

@@ -15,7 +15,7 @@
  */
 
 #define _GNU_SOURCE
-#include <glib.h>
+
 #include <string.h>
 #include <stdio.h>
 #include <sys/stat.h>
@@ -29,10 +29,13 @@
 #include <sys/un.h>
 #include <limits.h>
 #include <inttypes.h>
+#include <ctype.h>
 
+#include "util.h"
 #include "common.h"
 #include "configuration.h"
 #include "telemetry.h"
+#include "log.h"
 
 /**
  * Return a file descriptor to either site's version file
@@ -74,9 +77,7 @@ static int set_header(char **dest, const char *prefix, char *value, size_t *head
         rc = asprintf(dest, "%s: %s\n", prefix, value);
 
         if (rc < 0) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         } else {
                 *header_size += (size_t)rc;
@@ -137,38 +138,14 @@ static int set_severity_header(struct telem_ref *t_ref, uint32_t severity)
  */
 static int set_classification_header(struct telem_ref *t_ref, char *classification)
 {
-        size_t i, j;
-        int slashes = 0;
-        int x = 0;
 
-        j = strlen(classification);
-
-        if (j > MAX_CLASS_LENGTH) {
-                return -EINVAL;
-        }
-
-        for (i = 0, x = 0; i <= (j - 1); i++, x++) {
-                if (classification[i] == '/') {
-                        slashes++;
-                        x = 0;
-                } else {
-                        if (x > MAX_SUBCAT_LENGTH) {
-                                return -EINVAL;
-                        }
-                }
-        }
-
-        if (slashes != 2) {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Classification string should have two /s.\n");
-#endif
+        if (validate_classification(classification) == 1) {
                 return -EINVAL;
         }
 
         return set_header(&(t_ref->record->headers[TM_CLASSIFICATION]),
                           TM_CLASSIFICATION_STR, classification,
                           &(t_ref->record->header_size));
-
 }
 
 /**
@@ -255,35 +232,49 @@ static int set_system_name_header(struct telem_ref *t_ref)
         int fd;
         char buf[SMALL_LINE_BUF] = { 0 };
         char name[SMALL_LINE_BUF] = { 0 };
+        char variant[SMALL_LINE_BUF] = { 0 };
 
         fd = version_file();
         if (fd == -1) {
-#ifdef DEBUG
-                fprintf(stderr, "WARNING: Cannot find os-release file\n");
-#endif
+                telem_log(LOG_WARNING, "WARNING: Cannot find os-release file\n");
                 sprintf(name, "unknown");
         } else {
                 fs = fdopen(fd, "r");
                 while (fgets(buf, SMALL_LINE_BUF, fs)) {
-                        if (sscanf(buf, "ID=%s", name) < 1) {
-                                continue;
-                        } else {
+                        if (sscanf(buf, "ID=%s", name) > 0) {
+                                break;
+                        }
+                }
+                rewind(fs);
+                while (fgets(buf, SMALL_LINE_BUF, fs)) {
+                        if (sscanf(buf, "VARIANT_ID=%s", variant) > 0) {
                                 break;
                         }
                 }
 
+                fclose(fs);
+
                 if (strlen(name) == 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "WARNING: Cannot find os-release field: ID\n");
-#endif
+                        telem_log(LOG_WARNING, "WARNING: Cannot find os-release field: ID\n");
                         sprintf(name, "unknown");
                 }
 
-                fclose(fs);
+                if (strlen(variant) == 0) {
+                        snprintf(buf, sizeof(buf), "%s", name);
+                } else {
+                        /* We do not want the header to be greater than 80 characters, allow truncation */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-truncation"
+                        if (snprintf(buf, sizeof(buf), "%s_%s", name, variant) > sizeof(buf)) {
+                                telem_log(LOG_WARNING, "WARNING: truncating system name header to 80 characters\n");
+                        }
+                        telem_debug("DEBUG: Found variant OS information: %s\n", variant);
+#pragma GCC diagnostic pop
+                }
         }
 
         return set_header(&(t_ref->record->headers[TM_SYSTEM_NAME]),
-                          TM_SYSTEM_NAME_STR, name,
+                          TM_SYSTEM_NAME_STR, buf,
                           &(t_ref->record->header_size));
 
 }
@@ -307,9 +298,7 @@ static int set_system_build_header(struct telem_ref *t_ref)
 
         fd = version_file();
         if (fd == -1) {
-#ifdef DEBUG
-                fprintf(stderr, "WARNING: Cannot find build version file\n");
-#endif
+                telem_log(LOG_WARNING, "WARNING: Cannot find build version file\n");
                 sprintf(version, "0");
         } else {
                 fs = fdopen(fd, "r");
@@ -322,9 +311,7 @@ static int set_system_build_header(struct telem_ref *t_ref)
                 }
 
                 if (strlen(version) == 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "WARNING: Cannot find build version number\n");
-#endif
+                        telem_log(LOG_WARNING, "WARNING: Cannot find build version number\n");
                         sprintf(version, "0");
                 }
 
@@ -338,7 +325,7 @@ static int set_system_build_header(struct telem_ref *t_ref)
 
 /**
  * Sets the header reporting the machine_id. Note, the machine_id is really
- * managed by the telemetrics daemon (telemd), and populated by the daemon
+ * managed by the telemetrics daemon (telemprobd), and populated by the daemon
  * as it delivers the records. Consequently, the header is simply set to a
  * dummy value of 0xFFFFFFF.
  *
@@ -400,6 +387,60 @@ static int set_timestamp_header(struct telem_ref *t_ref)
         return status;
 }
 
+/**
+ * Sets cpu model for telemetry record. The information from cpu is extracted
+ * from /proc/cpuinfo, specifically "model name" attribute.
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_cpu_model_header(struct telem_ref *t_ref)
+{
+        FILE *fs = NULL;
+        char buf[SMALL_LINE_BUF] = { 0 };
+        char *model_name = NULL;
+        const char *attr_name = "model name";
+        int status = 0;
+        size_t attr_len = strlen(attr_name);
+        size_t model_str_len = 0;
+
+        fs = fopen("/proc/cpuinfo", "r");
+        if (fs != NULL) {
+                while (fgets(buf, SMALL_LINE_BUF, fs)) {
+                        if (strncmp(attr_name, buf, attr_len) == 0) {
+                                model_name = strchr(buf, ':');
+                                break;
+                        }
+                }
+                fclose(fs);
+                if (model_name != NULL) {
+                        model_str_len = strlen(model_name);
+                        if (model_str_len > 2) {
+                                model_name = (char *)model_name + 2 * sizeof(char);
+                                model_name[model_str_len - 3] = '\0';
+                        } else {
+                                model_name = "blank";
+                        }
+                } else {
+                        model_name = "blank";
+                        telem_log(LOG_NOTICE, "NOTICE: Unable to find attribute:%s\n", attr_name);
+                }
+
+                status = set_header(
+                        &(t_ref->record->headers[TM_CPU_MODEL]),
+                        TM_CPU_MODEL_STR, model_name,
+                        &(t_ref->record->header_size));
+
+        } else {
+                telem_log(LOG_NOTICE, "NOTICE: Unable to open /proc/cpuinfo\n");
+                status = -1;
+        }
+
+        return status;
+}
+
 /**
  * A healper function for set_host_type_header that reads the first line out of
  * a file and chomps the newline if necessary. If the file does not exist the
@@ -427,9 +468,7 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
         *buf = (char *)malloc(sizeof(char) * SMALL_LINE_BUF);
 
         if (*buf == NULL) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         }
 
@@ -452,9 +491,7 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
                 if (new_size == 0) {
                         old_value = *buf;
                         if (asprintf(buf, "blank") < 0) {
-#ifdef DEBUG
-                                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                                 ret = -ENOMEM;
                         }
                         free(old_value);
@@ -471,24 +508,18 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
                         if (j == new_size) {
                                 old_value = *buf;
                                 if (asprintf(buf, "blank") < 0) {
-#ifdef DEBUG
-                                        fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                                        telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                                         ret = -ENOMEM;
                                 }
                                 free(old_value);
                         }
                 }
         } else {
-#ifdef DEBUG
-                fprintf(stderr, "NOTICE: Dmi file %s does not exist\n", source);
-#endif
+                telem_debug("DEBUG: Dmi file %s does not exist\n", source);
 
                 old_value = *buf;
                 if (asprintf(buf, "no_%s_file", key) < 0) {
-#ifdef DEBUG
-                        fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                        telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                         ret = -ENOMEM;
                 }
                 free(old_value);
@@ -497,6 +528,118 @@ static int get_dmi_value(const char *source, const char *key, char **buf)
         return ret;
 }
 
+/**
+ * Sets the board name for telemetry record, this record is a combination of
+ * board name and board vendor. This information is read from dmi filesystem
+ * Board Name (board_name) and Board Vendor (board_vendor).
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_board_name_header(struct telem_ref *t_ref)
+{
+        int status = 0;
+        int rc;
+        char *buf = NULL;
+        char *bn = NULL;
+        char *bv = NULL;
+
+        rc  = get_dmi_value("/sys/class/dmi/id/board_name", "bn", &bn);
+        if (rc < 0) {
+                status = rc;
+                goto cleanup;
+        }
+
+        rc  = get_dmi_value("/sys/class/dmi/id/board_vendor", "bv", &bv);
+        if (rc < 0) {
+                status = rc;
+                goto cleanup;
+        }
+
+        rc = asprintf(&buf, "%s|%s", bn, bv);
+
+        if (rc < 0) {
+                status = -ENOMEM;
+                goto cleanup;
+        } else {
+                status = set_header(
+                        &(t_ref->record->headers[TM_BOARD_NAME]),
+                        TM_BOARD_NAME_STR, buf,
+                        &(t_ref->record->header_size));
+                free(buf);
+        }
+
+cleanup:
+        if (bn != NULL) {
+                free(bn);
+        }
+
+        if (bv != NULL) {
+                free(bv);
+        }
+
+        return status;
+}
+
+/**
+ * Sets BIOS version header for telemetry record, this information is
+ * read from dmi filesystem BIOS Version (bios_version).
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, or a negative errno-style value if not.
+ *
+ */
+static int set_bios_version_header(struct telem_ref *t_ref)
+{
+        int status = 0;
+        int rc = 0;
+        char *bios_version  = NULL;
+
+        rc  = get_dmi_value("/sys/class/dmi/id/bios_version", "bv", &bios_version);
+        if (rc < 0) {
+                status = rc;
+        } else {
+                status = set_header(
+                        &(t_ref->record->headers[TM_BIOS_VERSION]),
+                        TM_BIOS_VERSION_STR, bios_version,
+                        &(t_ref->record->header_size));
+                free(bios_version);
+        }
+
+        return status;
+}
+
+/**
+ * Sets the event_id header, this id is an identifier that multiple
+ * records can share. This means that one event can lead to multiple
+ * records.
+ *
+ * @param t_ref Telemetry Record reference obtained from tm_create_record.
+ *
+ * @return 0 if successful, -1 on failure
+ *
+ */
+static int set_event_id_header(struct telem_ref *t_ref)
+{
+        int rc = 0;
+        char *buff = NULL;
+
+        rc = get_random_id(&buff);
+
+        if (rc == 0) {
+                rc = set_header(
+                        &(t_ref->record->headers[TM_EVENT_ID]),
+                        TM_EVENT_ID_STR, buff,
+                        &(t_ref->record->header_size));
+        }
+        free(buff);
+
+        return rc;
+}
+
 /**
  * Sets the hosttype header, which is a tuple of three values looked for
  * in the dmi filesystem.  System Vendor (sys_vendor), Product Name
@@ -631,9 +774,9 @@ static int set_payload_format_header(struct telem_ref *t_ref, uint32_t payload_v
         return status;
 }
 
-void tm_set_config_file(char *c_file)
+int tm_set_config_file(const char *c_file)
 {
-        set_config_file(c_file);
+        return set_config_file(c_file);
 }
 
 /**
@@ -730,6 +873,30 @@ int allocate_header(struct telem_ref *t_ref, uint32_t severity,
                 goto free_and_fail;
         }
 
+        i++;
+
+        if ((ret = set_board_name_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_cpu_model_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_bios_version_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
+        i++;
+
+        if ((ret = set_event_id_header(t_ref)) < 0) {
+                goto free_and_fail;
+        }
+
         i++; /* Not necessary, but including for future expansion */
 
         return ret;
@@ -757,17 +924,13 @@ int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
 
         *t_ref = (struct telem_ref *)malloc(sizeof(struct telem_ref));
         if (*t_ref == NULL) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         }
 
         (*t_ref)->record = (struct telem_record *)malloc(sizeof(struct telem_record));
         if ((*t_ref)->record == NULL) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 free(*t_ref);
                 return -ENOMEM;
         }
@@ -806,8 +969,8 @@ int payload_is_ascii(char *payload, size_t len)
         int ret = 0;
 
         for (i = 0; i < len; i++) {
-                if (!g_ascii_isprint((gchar)payload[i]) &&
-                    !g_ascii_isspace((gchar)payload[i])) {
+                if (!isprint(payload[i]) &&
+                    !isspace(payload[i])) {
                         ret = -EINVAL;
                         break;
                 }
@@ -840,26 +1003,72 @@ int tm_set_payload(struct telem_ref *t_ref, char *payload)
                 return -EINVAL;
         }
 
-        t_ref->record->payload = (char *)malloc(sizeof(char) * payload_len + 1);
+        t_ref->record->payload = strdup(payload);
 
         if (!t_ref->record->payload) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 return -ENOMEM;
         }
 
-        memset(t_ref->record->payload, 0, sizeof(char) * payload_len + 1);
-
-        strncpy((char *)(t_ref->record->payload), (char *)payload, payload_len);
-
         t_ref->record->payload_size = payload_len;
 
         return ret;
 }
 
 /**
- * Write nbytes from buf to fd. Used to send records to telemd.
+ * Checks for id to have length = 32 characters
+ * and hexadecimal characters only.
+ *
+ * @param id A pointer to string to be checked
+ *
+ * @return 0 on success, 1 on failure
+ *
+ */
+static int validate_event_id(char *id)
+{
+        int rc = 0;
+        const char alphab[] = EVENT_ID_ALPHAB;
+
+        // make sure is not null
+        if (!id) {
+                return 1;
+        }
+
+        // 32 - 32 = 0
+        if ((rc = (int)(strlen(id) - EVENT_ID_LEN)) != 0) {
+                return 1;
+        }
+
+        // verify alphabet
+        // strspn checks for characters in alphab
+        if (strspn(id, alphab) != EVENT_ID_LEN) {
+                return 1;
+        }
+
+        return rc;
+}
+
+int tm_set_event_id(struct telem_ref *t_ref, char *event_id)
+{
+        int rc = -1;
+
+        if (!validate_event_id(event_id)) {
+                if (t_ref && t_ref->record && t_ref->record->headers) {
+                        // free default id before overriding
+                        free(t_ref->record->headers[TM_EVENT_ID]);
+                        // set new event_id
+                        if (asprintf(&(t_ref->record->headers[TM_EVENT_ID]),
+                                     "%s: %s\n", TM_EVENT_ID_STR, event_id) > 0) {
+                                rc = 0;
+                        }
+                }
+        }
+
+        return rc;
+}
+
+/**
+ * Write nbytes from buf to fd. Used to send records to telemprobd.
  *
  * @param fd Socket fd obtained from tm_get_socket.
  * @param buf Data to be written to the socket.
@@ -880,10 +1089,7 @@ static int tm_write_socket(int fd, char *buf, size_t nbytes)
 
                 if (b == -1 && errno != EAGAIN) {
                         ret = -errno;
-#ifdef DEBUG
-                        fprintf(stderr, "ERR: Write to daemon socket with"
-                                " system error: %s\n", strerror(errno));
-#endif
+                        telem_perror("Error writing to daemon socket");
                         return ret;
                 } else if (b == -1 &&
                            (errno == EAGAIN || errno == EWOULDBLOCK)) {
@@ -923,10 +1129,7 @@ static int tm_get_socket(void)
 
         if (sfd == -1) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Attempt to allocate socket fd failed:"
-                        " %s\n", strerror(errno));
-#endif
+                telem_perror("Attempt to allocate socket fd failed");
                 return ret;
         }
 
@@ -937,10 +1140,7 @@ static int tm_get_socket(void)
         if (setsockopt(sfd, SOL_SOCKET, SO_RCVTIMEO, &tv,
                        sizeof(tv)) < 0) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to set socket timeout:"
-                        " %s\n", strerror(errno));
-#endif
+                telem_perror("Failed to set socket timeout");
                 goto out1;
         }
 
@@ -996,9 +1196,7 @@ static int tm_get_socket(void)
         // Set non-blocking after the connect() succeeded
         if ((sflags = fcntl(sfd, F_GETFL, NULL)) < 0) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to get socket flags\n");
-#endif
+                telem_log(LOG_ERR, "Failed to get socket flags\n");
                 goto out1;
         }
 
@@ -1006,9 +1204,7 @@ static int tm_get_socket(void)
 
         if (fcntl(sfd, F_SETFL, sflags) < 0) {
                 ret = -errno;
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to set socket flags\n");
-#endif
+                telem_log(LOG_ERR, "Failed to set socket flags\n");
                 goto out1;
         }
 
@@ -1030,6 +1226,8 @@ int tm_send_record(struct telem_ref *t_ref)
         int ret = 0;
         int k = 0;
         struct stat unused;
+        size_t cfg_file_name_size = 0;
+        const char *cfg_file_name = NULL;
 
         k = stat(TM_OPT_OUT_FILE, &unused);
         if (k == 0) {
@@ -1040,24 +1238,44 @@ int tm_send_record(struct telem_ref *t_ref)
         sfd = tm_get_socket();
 
         if (sfd < 0) {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Failed to get socket fd: %s\n",
-                        strerror(-sfd));
-#endif
+                telem_log(LOG_ERR, "Failed to get socket fd: %s\n",
+                          strerror(-sfd));
                 return sfd;
         }
 
         total_size = t_ref->record->header_size + t_ref->record->payload_size;
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Header size : %zu\n", t_ref->record->header_size);
-        fprintf(stderr, "DEBUG: Payload size : %zu\n", t_ref->record->payload_size);
 
-        fprintf(stderr, "DEBUG: Total size : %zu\n", total_size);
-#endif
+        /*
+         * The user may want to use a custom (cmd line specified) config file.
+         * If that is the case, we need to include the absolute path in the data packet.
+         * We optionally add the string CFG:<absolute path> as the first string
+         * in the data. If the receiving end sees this in the data, it should
+         * use the config file specified to override any default telemetrics.conf
+         * files.
+         */
+
+        cfg_file_name = get_cmd_line_config_file();
+        if (cfg_file_name != NULL) {
+                cfg_file_name_size = strlen(cfg_file_name) + 1;
+                total_size += (cfg_file_name_size + CFG_PREFIX_LENGTH);
+        }
+
+        if (cfg_file_name != NULL) {
+                telem_debug("DEBUG: CFG field size : %zu\n", cfg_file_name_size + CFG_PREFIX_LENGTH);
+                telem_debug("DEBUG: CFG file name : %s\n", cfg_file_name);
+        }
+
+        telem_debug("DEBUG: Header size : %zu\n", t_ref->record->header_size);
+        telem_debug("DEBUG: Payload size : %zu\n", t_ref->record->payload_size);
+
+        telem_debug("DEBUG: Total size : %zu\n", total_size);
 
         /*
          * Allocating buffer for what we intend to send.  Buffer layout is:
-         * <uint32_t total_size><uint32_t header_size><headers + Payload>
+         * <uint32_t total_size>     : so recv knows how much to read
+         * <custom cfg file field>   : optional
+         * <uint32_t header_size>
+         * <headers + Payload>
          * <null-byte>
          * The additional char at the end ensures null termination
          */
@@ -1065,9 +1283,7 @@ int tm_send_record(struct telem_ref *t_ref)
 
         data = malloc(record_size);
         if (!data) {
-#ifdef DEBUG
-                fprintf(stderr, "CRIT: Out of memory\n");
-#endif
+                telem_log(LOG_CRIT, "CRIT: Out of memory\n");
                 close(sfd);
                 return -ENOMEM;
         }
@@ -1077,6 +1293,13 @@ int tm_send_record(struct telem_ref *t_ref)
         memcpy(data, &total_size, sizeof(uint32_t));
         offset += sizeof(uint32_t);
 
+        if (cfg_file_name != NULL) {
+                memcpy(data + offset, CFG_PREFIX, CFG_PREFIX_LENGTH);
+                offset += CFG_PREFIX_LENGTH;
+                memcpy(data + offset, cfg_file_name, cfg_file_name_size);
+                offset += cfg_file_name_size;
+        }
+
         memcpy(data + offset, &t_ref->record->header_size, sizeof(uint32_t));
         offset += sizeof(uint32_t);
 
@@ -1089,17 +1312,11 @@ int tm_send_record(struct telem_ref *t_ref)
 
         memcpy(data + offset, t_ref->record->payload, t_ref->record->payload_size);
 
-#ifdef DEBUG
-        fprintf(stderr, "DEBUG: Data to be sent :\n\n%s\n", data + 2 * sizeof(uint32_t));
-#endif
+        telem_debug("DEBUG: Data to be sent :\n\n%s\n", data + 2 * sizeof(uint32_t));
         if ((ret = tm_write_socket(sfd, data, record_size)) == 0) {
-#ifdef DEBUG
-                fprintf(stderr, "INFO: Successfully sent record over the socket\n");
-#endif
+                telem_log(LOG_INFO, "INFO: Successfully sent record over the socket\n");
         } else {
-#ifdef DEBUG
-                fprintf(stderr, "ERR: Error while writing data to socket\n");
-#endif
+                telem_log(LOG_ERR, "Error while writing data to socket\n");
         }
 
         close(sfd);

src/telemetry.h

@@ -31,7 +31,10 @@
  * - a library, libtelemetry, that telemetrics probes use to create telemetrics
  *   records and send them to the daemon for further processing.
  *
- * - a daemon, telemd, that prepares the records to send to a telemetrics server
+ * - telemprobd, a daemon that handles communication with probes and forwards
+ *   records to a second daemon.
+ *
+ * - telempostd, this daemon sends telemetry records to a telemetrics server
  *   (not included in this source tree), or spools the records on disk in case
  *   it's unable to successfully deliver them.
  *
@@ -55,11 +58,11 @@ struct telem_ref {
 /**
  * Set the configuration file name to use
  *
- * @param c_file Absolute file name of the configuration file. If the path name
- * provided is not an absolute path, or the path given cannot be successfuly
- * stat'd, then the effective configuration file is left unchanged.
+ * @param c_file Absolute file name of the configuration file.
+ * @return 0 on success, or a negative errno-style value on error. In case of 
+ *     an error return the effective configuration file is left unchanged.
  */
-void tm_set_config_file(char *c_file);
+int tm_set_config_file(const char *c_file);
 
 /**
  * Create a new telemetrics record
@@ -85,6 +88,19 @@ void tm_set_config_file(char *c_file);
 int tm_create_record(struct telem_ref **t_ref, uint32_t severity,
                      char *classification, uint32_t payload_version);
 
+/**
+ * Sets the event_id to a user defined id
+ *
+ * @param t_ref A pointer to a telem_ref struct that was previously initialized
+ *        this struct will have an event_id already initialized and it will be
+ *        overriden by a user provided event_id
+ * @param event_id A pointer to a string that contains the 32 character length
+ *        event_id
+ *
+ * @return 0 on success, or a negative errno-style value on error
+ */
+int tm_set_event_id(struct telem_ref *t_ref, char *event_id);
+
 /**
  * Set the payload field of a telemetrics record
  *