Release v1.16.0

This release contains:

* record_retention_enabled configuration key: when configuration value
  is enabled copies of the last 100 telemetry record payload are kept
  locally.

* record_server_delivery_enabled configuration key: when this key is
  set to true (default value) records are reported to server. When key
  is set to false records are not sent. This configuration can be used
  with record_retention_enabled to retain debug information locally
  only.

* A new switch (-i/--include_record) for 'telemctl journal' command,
  this switch allows the inclusion of record contents when printing
  journal to standard output.

Signed-off-by: avjarami <alex.v.jaramillo@intel.com>

.gitignore

@@ -38,6 +38,7 @@ autoscan.log
 configure.scan
 tprobe
 hprobe
+bertprobe
 crashprobe
 journalprobe
 telem-record-gen
@@ -45,6 +46,7 @@ klogscanner
 oopsprobe
 pstoreclean
 pstoreprobe
+telem_journal
 make.out
 types_c.taghl
 src/recomp.sh

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([telemetrics-client], [1.15.0], [https://clearlinux.org/])
+AC_INIT([telemetrics-client], [1.16.0], [https://clearlinux.org/])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([1.14 -Wall -Werror -Wno-extra-portability foreign subdir-objects])
 AM_SILENT_RULES([yes])

src/common.h

@@ -64,6 +64,7 @@
 /* Journal */
 #define JOURNAL_PATH "/var/log/telemetry/journal"
 #define JOURNAL_TMPDIR  "/var/log/telemetry/"
+#define RECORD_RETENTION_DIR "/var/log/telemetry/records"
 
 /* For internal library usage. Bump the version whenever we change the record
  * structure (e.g. adding or removing a header field). Note that the value

src/configuration.c

@@ -47,8 +47,8 @@ const char *config_key_int[] = { NULL, "record_expiry", "spool_max_size",
                                  "byte_window_length", "record_burst_limit",
                                  "byte_burst_limit", NULL };
 
-const char *config_key_bool[] = { NULL, "rate_limit_enabled",
-                                  "daemon_recycling_enabled", NULL };
+const char *config_key_bool[] = { NULL, "rate_limit_enabled", "daemon_recycling_enabled",
+                                  "record_retention_enabled", "record_server_delivery_enabled", NULL };
 
 static struct configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
 
@@ -132,8 +132,22 @@ bool read_config_from_file(char *config_file, struct configuration *config)
                                         config->boolValues[i] = true;
                                 }
                         } else {
-                                fprintf(stderr, "ERR: missing key with boolean value: %s\n", config_key_bool[i]);
-                                return false;
+                                /* New configuration keys, CONF_RECORD_RETENTION_ENABLED and CONF_RECORD_SERVER_DELIVERY_ENABLED
+                                 * default values, otherwise update will require changes in custom configurations */
+                                if (i == CONF_RECORD_RETENTION_ENABLED) {
+#ifdef DEBUG
+                                        fprintf(stderr, "WARN: missing boolean optional key: %s\n", config_key_bool[i]);
+#endif
+                                        config->boolValues[i] = RECORD_RETENTION_ENABLED_DEFAULT;
+                                } else if (i == CONF_RECORD_SERVER_DELIVERY_ENABLED) {
+#ifdef DEBUG
+                                        fprintf(stderr, "WARN: missing boolean optional key: %s\n", config_key_bool[i]);
+#endif
+                                        config->boolValues[i] = RECORD_SERVER_DELIVERY_ENABLED_DEFAULT;
+                                } else {
+                                        fprintf(stderr, "ERR: missing key with boolean value: %s\n", config_key_bool[i]);
+                                        return false;
+                                }
                         }
                 }
         }
@@ -356,4 +370,16 @@ bool daemon_recycling_enabled_config(void)
         initialize_config();
         return config.boolValues[CONF_DAEMON_RECYCLING_ENABLED];
 }
+
+bool record_retention_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_RETENTION_ENABLED];
+}
+
+bool record_server_delivery_enabled_config(void)
+{
+        initialize_config();
+        return config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED];
+}
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/configuration.h

@@ -19,6 +19,8 @@
 #include <stdint.h>
 
 #define TM_MAX_WINDOW_LENGTH (1 /*h*/ * 60 /*m*/)
+#define RECORD_RETENTION_ENABLED_DEFAULT false
+#define RECORD_SERVER_DELIVERY_ENABLED_DEFAULT true
 
 enum config_str_keys {
         CONF_STR_MIN = 0,
@@ -47,6 +49,8 @@ enum config_bool_keys {
         CONF_BOOL_MIN = 0,
         CONF_RATE_LIMIT_ENABLED,
         CONF_DAEMON_RECYCLING_ENABLED,
+        CONF_RECORD_RETENTION_ENABLED,
+        CONF_RECORD_SERVER_DELIVERY_ENABLED,
         CONF_BOOL_MAX
 };
 
@@ -117,4 +121,10 @@ const char *get_tidheader_config(void);
 /* Gets whether recycling is enabled */
 bool daemon_recycling_enabled_config(void);
 
+/* Gets whether local retention of records is enabled */
+bool record_retention_enabled_config(void);
+
+/* Gets whether records should be sent to server_addr */
+bool record_server_delivery_enabled_config(void);
+
 /* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/data/example.1.conf

@@ -0,0 +1,46 @@
+[settings]
+server=http://127.0.0.1
+
+cainfo=/tmp/cacert.crt
+
+tidheader=X-Telemetry-TID:\s6907c830-eed9-4ce9-81ae-76daf8d88f0f
+
+socket_path=/tmp/test_telem_socket
+
+#record expiry time in minutes
+record_expiry=1200
+
+spool_dir=/tmp/spool
+
+#maximum size of the spool directory in KB
+spool_max_size=1024
+
+#time in seconds for processing spool
+spool_process_time=900
+
+#rate limiting record burst limit
+record_burst_limit=100
+
+#rate limiting record window length
+record_window_length = 15
+
+#rate limiting byte burst limit
+byte_burst_limit = 1000
+
+#rate limiting byte window length
+byte_window_length=20
+
+#rate limit enabled
+rate_limit_enabled=true
+
+#rate limit strategy if record not send
+rate_limit_strategy=spool
+
+#daemon recycling enabled
+daemon_recycling_enabled=true
+
+#server delivery disable
+record_server_delivery_enabled=false
+
+#local copy enabled
+record_retention_enabled=true

src/data/local.mk

@@ -11,6 +11,7 @@ EXTRA_DIST += \
 	%D%/40-core-ulimit.conf \
 	%D%/40-crash-probe.conf.in \
 	%D%/example.conf \
+	%D%/example.1.conf \
 	%D%/hprobe.service.in \
 	%D%/hprobe.timer \
 	%D%/bert-probe.service.in \

src/data/telemetrics-dirs.conf.in

@@ -1,6 +1,7 @@
 d @localstatedir@/lib/telemetry 0755 telemetry telemetry -
 d @localstatedir@/spool/telemetry 0750 telemetry telemetry -
 d @localstatedir@/log/telemetry 0750 telemetry telemetry -
+d @localstatedir@/log/telemetry/records 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry/oops 0750 telemetry telemetry -
 d @localstatedir@/cache/telemetry/pstore 0750 telemetry telemetry -

src/data/telemetrics.conf.in

@@ -55,3 +55,13 @@ rate_limit_strategy=spool
 # has not any client nor spool data, then it exits.
 # this is to ensure that latest code runs.
 daemon_recycling_enabled=true
+
+# record server delivery enabled - when enabled records will be delivered
+# to server otherwise records will be ignored. This configuration can be used
+# with 'record_retention_enable' configuration value to keep records local only.
+record_server_delivery_enabled=true
+
+# record retention enabled - when enabled a copy of reported telemetry records
+# will be kept locally. This configuration combined with 'record_server_delivery_enabled'
+# value can be used to keep records local only.
+record_retention_enabled=false

src/journal/cli.c

@@ -27,12 +27,13 @@
 void print_usage(void)
 {
         printf(" Usage\n");
-        printf("  -r,  --record_id       Print record to stdout\n");
-        printf("  -e,  --event_id        List records with specific event_id\n");
-        printf("  -c,  --classification  List records with specific classification\n");
-        printf("  -b,  --boot_id         List records with specific boot_id\n");
-        printf("  -V,  --verbose         Verbose output\n");
-        printf("  -h,  --help            Display this help message\n");
+        printf("  -r,  --record_id        Print record with specific record_id\n");
+        printf("  -e,  --event_id         Print records with specific event_id\n");
+        printf("  -c,  --classification   Print records with specific classification\n");
+        printf("  -b,  --boot_id          Print records with specific boot_id\n");
+        printf("  -i,  --include_record   Include record content\n");
+        printf("  -V,  --verbose          Verbose output\n");
+        printf("  -h,  --help             Display this help message\n");
 }
 
 int main(int argc, char **argv)
@@ -41,6 +42,7 @@ int main(int argc, char **argv)
         int rc = EXIT_SUCCESS;
         int count = 0;
         int verbose_output = 0;
+        int record = 0;
         char *boot_id = NULL;
         char *record_id = NULL;
         char *event_id = NULL;
@@ -56,11 +58,12 @@ int main(int argc, char **argv)
                 { "classification", 1, NULL, 'c' },
                 { "boot_id", 1, NULL, 'b' },
                 { "verbose", 0, NULL, 'V' },
+                { "include_record", 0, NULL, 'i' },
                 { "help", 0, NULL, 'h' },
                 { NULL, 0, NULL, 0 }
         };
 
-        while ((c = getopt_long(argc, argv, "r:e:c:b:Vh", opts, &opt_index)) != -1) {
+        while ((c = getopt_long(argc, argv, "r:e:c:b:Vih", opts, &opt_index)) != -1) {
                 switch (c) {
                         case 'r':
                                 record_id = optarg;
@@ -77,6 +80,9 @@ int main(int argc, char **argv)
                         case 'V':
                                 verbose_output = 1;
                                 break;
+                        case 'i':
+                                record = 1;
+                                break;
                         case 'h':
                                 print_usage();
                                 exit(EXIT_SUCCESS);
@@ -92,7 +98,7 @@ int main(int argc, char **argv)
                         fprintf(stdout, "%-30s %-27s %-32s %-32s %-36s\n", "Classification", "Time stamp",
                                 "Record ID", "Event ID", "Boot ID");
                 }
-                count = print_journal(telem_journal, classification, record_id, event_id, boot_id);
+                count = print_journal(telem_journal, classification, record_id, event_id, boot_id, record);
                 if (verbose_output) {
                         fprintf(stdout, "Total records: %d\n", count);
                 }
@@ -102,10 +108,5 @@ int main(int argc, char **argv)
                 rc = EXIT_FAILURE;
         }
 
-        free(classification);
-        free(record_id);
-        free(event_id);
-        free(boot_id);
-
         return rc;
 }

src/journal/journal.c

@@ -19,6 +19,7 @@
 #define ID_LEN 32
 #define BOOTID_LEN 37 // Includes the \n character at the end
 #define BOOTID_FILE "/proc/sys/kernel/random/boot_id"
+#define MID_BUFF 1024
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -31,6 +32,7 @@
 #include <unistd.h>
 #include <assert.h>
 
+#include "log.h"
 #include "util.h"
 #include "common.h"
 #include "journal.h"
@@ -246,17 +248,20 @@ static int read_boot_id(char buff[])
  *
  * @param n Number of lines to skip.
  * @param fptr A file pointer.
+ * @param found_record A pointer to a function to be called
+ *                     every time a valid record is found, if
+ *                     pointer is not set to null.
  *
  * @returns 0 on success, on failure -1 if n > existing
  *            lines in file or errno if failure happens
  *            during file operation.
- *
  */
-static int skip_n_lines(int n, FILE *fptr)
+static int skip_n_lines(int n, FILE *fptr, int (*found_record)(char *))
 {
         int rc = 0;
         size_t len = 0;
         char *line = NULL;
+        struct JournalEntry *entry = NULL;
 
         if (fptr == NULL) {
 #ifdef DEBUG
@@ -275,6 +280,14 @@ static int skip_n_lines(int n, FILE *fptr)
                         rc = -1;
                         break;
                 }
+                if (found_record != NULL) {
+                        deserialize_journal_entry(line, &entry);
+                        if (entry) {
+                                found_record(entry->record_id);
+                                free_journal_entry(entry);
+                                entry = NULL;
+                        }
+                }
         }
         free(line);
 
@@ -361,6 +374,8 @@ TelemJournal *open_journal(const char *journal_file)
         telem_journal->boot_id = strndup(boot_id, BOOTID_LEN - 1);
         telem_journal->record_count = get_record_count(telem_journal);
         telem_journal->record_count_limit = RECORD_LIMIT;
+        telem_journal->latest_record_id = NULL;
+        telem_journal->prune_entry_callback = NULL;
 
 #ifdef DEBUG
         printf("Records in db: %d\n", telem_journal->record_count);
@@ -375,6 +390,7 @@ void close_journal(TelemJournal *telem_journal)
         if (telem_journal) {
                 free(telem_journal->boot_id);
                 free(telem_journal->journal_file);
+                free(telem_journal->latest_record_id);
                 fclose(telem_journal->fptr);
                 free(telem_journal);
         }
@@ -393,9 +409,44 @@ static int is_class_prefix(char *class)
         return (strcmp((char *)(class + class_len - 2), "/*") == 0) ? 1 : 0;
 }
 
+/**
+ * Print records content
+ *
+ * @param record_id Unique record identifier
+ *
+ */
+static void print_record(char *record_id)
+{
+        int rc = 0;
+        size_t read = 0;
+        char buff[MID_BUFF] = { '\0' };
+        char *filepath = NULL;
+        FILE *recordfp = NULL;
+
+        rc = asprintf(&filepath, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (rc == -1) {
+                // just bail out, there are worse problems
+                return;
+        }
+
+        recordfp = fopen(filepath, "r");
+        if (!recordfp) {
+                telem_perror("Error when opening a record to print");
+                return;
+        }
+
+        while ((read = fread(buff, 1, sizeof(buff), recordfp)) > 0) {
+                fwrite(buff, 1, read, stdout);
+        }
+
+        free(filepath);
+        fclose(recordfp);
+}
+
 /* Exported function */
 int print_journal(TelemJournal *telem_journal, char *classification,
-                  char *record_id, char *event_id, char *boot_id)
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record)
 {
         int n = 0;
         int rc = 0;
@@ -419,7 +470,7 @@ int print_journal(TelemJournal *telem_journal, char *classification,
         // Advance file pointer to line n
         n = telem_journal->record_count - telem_journal->record_count_limit;
         if (n > 0) {
-                rc = skip_n_lines(n, journal_fileptr);
+                rc = skip_n_lines(n, journal_fileptr, NULL);
                 if (rc == -1) {
                         return rc;
                 } else if (rc != 0) {
@@ -432,13 +483,13 @@ int print_journal(TelemJournal *telem_journal, char *classification,
                 if (entry) {
                         /* filter entry out if one is provided */
                         if (record_id != NULL && strcmp(entry->record_id, record_id) != 0) {
-                                continue;
+                                goto skip_print;
                         }
                         if (boot_id != NULL && strcmp(entry->boot_id, boot_id) != 0) {
-                                continue;
+                                goto skip_print;
                         }
                         if (event_id != NULL && strcmp(entry->event_id, event_id) != 0) {
-                                continue;
+                                goto skip_print;
                         }
                         // In the case of class checking prefixes is an option
                         if (classification != NULL) {
@@ -456,10 +507,15 @@ int print_journal(TelemJournal *telem_journal, char *classification,
                         if (strftime(str_time, sizeof(str_time), "%a %Y-%m-%d %H:%M:%S %Z", &ts) == 0) {
                                 continue;
                         }
-
+                        /* print record metadata */
                         fprintf(stdout, "%-30s %s %s %s %s\n", entry->classification, str_time, entry->record_id, entry->event_id, entry->boot_id);
+                        /* print record content */
+                        if (include_record) {
+                                print_record(entry->record_id);
+                        }
                         count++;
                 }
+skip_print:
                 free_journal_entry(entry);
         }
         free(line);
@@ -562,6 +618,9 @@ int new_journal_entry(TelemJournal *telem_journal, char *classification,
 #endif
         }
 
+        free(telem_journal->latest_record_id);
+        telem_journal->latest_record_id = strdup(entry->record_id);
+
 quit:
         free_journal_entry(entry);
 
@@ -585,7 +644,7 @@ int prune_journal(struct TelemJournal *telem_journal, char *tmp_dir)
                 count = telem_journal->record_count - telem_journal->record_count_limit;
 
                 // jump to line# count
-                if ((rc = skip_n_lines(count, telem_journal->fptr)) != 0) {
+                if ((rc = skip_n_lines(count, telem_journal->fptr, telem_journal->prune_entry_callback)) != 0) {
 #ifdef DEBUG
                         fprintf(stderr, "Error: skipping %d journal lines\n", count);
 #endif

src/journal/journal.h

@@ -22,6 +22,7 @@
 
 #include <time.h>
 #include <stdio.h>
+#include <stdbool.h>
 
 /* Journal entry type */
 typedef struct JournalEntry {
@@ -37,8 +38,10 @@ typedef struct TelemJournal {
         FILE *fptr;
         char *journal_file;
         char *boot_id;
+        char *latest_record_id;
         int record_count;
         int record_count_limit;
+        int (*prune_entry_callback)(char *);
 } TelemJournal;
 
 /**
@@ -75,12 +78,14 @@ void close_journal(TelemJournal *telem_journal);
  * @param record_id A pointer to string used as record_id filter.
  * @param event_id A pointer to string used as event_id filter.
  * @param boot_id A pointer to string used as boor_id filter.
+ * @param include_record A flag to control record content print.
  *
  * @return the number of lines printed to stdout on success, -1
  *         on failure.
  */
 int print_journal(TelemJournal *telem_journal, char *classification,
-                  char *record_id, char *event_id, char *boot_id);
+                  char *record_id, char *event_id, char *boot_id,
+                  bool include_record);
 
 /**
  * Creates a new entry in journal.

src/local.mk

@@ -7,7 +7,9 @@ bin_PROGRAMS = %D%/telemd
 	%D%/spool.c \
 	%D%/spool.h \
 	%D%/journal/journal.c \
-	%D%/journal/journal.h
+	%D%/journal/journal.h \
+	%D%/retention.c \
+	%D%/retention.h
 
 %C%_telemd_LDADD = $(CURL_LIBS) \
 	%D%/libtelem-shared.la

src/main.c

@@ -49,6 +49,7 @@
 #include "log.h"
 #include "telemdaemon.h"
 #include "configuration.h"
+#include "retention.h"
 #include "spool.h"
 
 /*
@@ -122,6 +123,11 @@ int main(int argc, char **argv)
         }
         initialize_daemon(&daemon);
 
+        /* Register record retention delete action as a callback to prune entry */
+        if (daemon.record_journal) {
+                daemon.record_journal->prune_entry_callback = &delete_record_by_id;
+        }
+
         sigemptyset(&mask);
 
         if (sigaddset(&mask, SIGHUP) != 0) {

src/probes/telem_record_gen.c

@@ -69,7 +69,7 @@ static void print_help(void)
         printf("  -p, --payload         Record body (max size = 8k)\n");
         printf("  -P, --payload-file    File to read payload from\n");
         printf("  -R, --record-version  Version number for format of payload (default 1)\n");
-        printf("  -e,  --event-id        Event id to use in the record\n");
+        printf("  -e, --event-id        Event id to use in the record\n");
         printf("\n");
 }
 

src/retention.c

@@ -0,0 +1,46 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+
+#include "log.h"
+#include "common.h"
+
+int delete_record_by_id(char *record_id)
+{
+        int ret = 0;
+        char *record_path = NULL;
+
+        ret = asprintf(&record_path, "%s/%s", RECORD_RETENTION_DIR, record_id);
+        if (ret == -1) {
+                return 1;
+        }
+        ret = unlink(record_path);
+        if (ret == -1) {
+                telem_perror("Error deleting saved record");
+        }
+        free(record_path);
+
+        return 0;
+}
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/retention.h

@@ -0,0 +1,27 @@
+/*
+ * This program is part of the Clear Linux Project
+ *
+ * Copyright 2018 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms and conditions of the GNU Lesser General Public License, as
+ * published by the Free Software Foundation; either version 2.1 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+#define _GNU_SOURCE
+
+/**
+ * Delete record identified by record unique id
+ *
+ * @param record_id Unique identifier of entry
+ *
+ */
+int delete_record_by_id(char *record_id);
+
+/* vi: set ts=8 sw=8 sts=4 et tw=80 cino=(0: */

src/telemdaemon.c

@@ -34,6 +34,7 @@
 #include "util.h"
 #include "log.h"
 #include "configuration.h"
+#include "retention.h"
 
 void initialize_daemon(TelemDaemon *daemon)
 {
@@ -46,6 +47,7 @@ void initialize_daemon(TelemDaemon *daemon)
         daemon->is_spool_valid = false;
         daemon->record_journal = open_journal(JOURNAL_PATH);
         initialize_rate_limit(daemon);
+        initialize_record_delivery(daemon);
         daemon->current_spool_size = 0;
 }
 
@@ -63,6 +65,12 @@ void initialize_rate_limit(TelemDaemon *daemon)
         daemon->rate_limit_strategy = rate_limit_strategy_config();
 }
 
+void initialize_record_delivery(TelemDaemon *daemon)
+{
+        daemon->record_retention_enabled = record_retention_enabled_config();
+        daemon->record_server_delivery_enabled = record_server_delivery_enabled_config();
+}
+
 client *add_client(client_list_head *client_head, int fd)
 {
         client *cl;
@@ -264,6 +272,21 @@ void machine_id_replace(char **machine_header, char *machine_id_override)
         free(old_header);
 }
 
+void save_entry_to_journal(TelemDaemon *daemon, time_t t_stamp, char *headers[])
+{
+        char *classification_value;
+        char *event_id_value;
+
+        if (get_header_value(headers[TM_CLASSIFICATION], &classification_value) &&
+            get_header_value(headers[TM_EVENT_ID], &event_id_value)) {
+                if (new_journal_entry(daemon->record_journal, classification_value, t_stamp, event_id_value) != 0) {
+                        telem_log(LOG_INFO, "new_journal_entry in process_record: failed saving record entry\n");
+                }   
+        }   
+        free(classification_value);
+        free(event_id_value);
+}
+
 void process_record(TelemDaemon *daemon, client *cl)
 {
         int i = 0;
@@ -281,10 +304,6 @@ void process_record(TelemDaemon *daemon, client *cl)
         bool byte_check_passed = true;
         bool record_burst_enabled = true;
         bool byte_burst_enabled = true;
-        /* values for journal */
-        char *classification_value;
-        char *event_id_value;
-
         /* Gets the current minute of time */
         time_t temp = time(NULL);
         struct tm *tm_s = localtime(&temp);
@@ -317,14 +336,20 @@ void process_record(TelemDaemon *daemon, client *cl)
         body = msg + header_size;
 
         /* Save record in journal */
-        if (get_header_value(headers[TM_CLASSIFICATION], &classification_value) &&
-            get_header_value(headers[TM_EVENT_ID], &event_id_value)) {
-                if (new_journal_entry(daemon->record_journal, classification_value, temp, event_id_value) != 0) {
-                        telem_log(LOG_INFO, "new_journal_entry in process_record: failed saving record entry\n");
-                }
+        save_entry_to_journal(daemon, temp, headers);
+
+        /* Save local copy */
+        if (daemon->record_retention_enabled) {
+                save_local_copy(daemon, body);
+        }
+
+        /* Bail out if server delivery is not enabled */
+        if (!daemon->record_server_delivery_enabled) {
+#ifdef DEBUG
+                fprintf(stdout, "process_record: record server delivery disabled\n");
+#endif
+                goto end;
         }
-        free(classification_value);
-        free(event_id_value);
 
         if (inside_direct_spool_window(daemon, time(NULL))) {
                 telem_log(LOG_INFO, "process_record: delivering directly to spool\n");
@@ -357,12 +382,12 @@ void process_record(TelemDaemon *daemon, client *cl)
                         daemon->rate_limit_enabled = false;
                 }
         }
+
         /* Sends record if rate limiting is disabled, or all checks passed */
         if (!daemon->rate_limit_enabled || (record_check_passed && byte_check_passed)) {
                 /* Send the record as https post */
                 record_sent = post_record_ptr(headers, body, true);
         }
-
         // Get rate-limit strategy
         do_spool = spool_strategy_selected(daemon);
 
@@ -558,6 +583,39 @@ bool post_record_http(char *headers[], char *body, bool spool)
         return res ? false : true;
 }
 
+void save_local_copy(TelemDaemon *daemon, char *body)
+{
+        int ret = 0;
+        char *tmpbuf = NULL;
+        FILE *tmpfile = NULL;
+
+        if (daemon == NULL || daemon->record_journal == NULL || 
+            daemon->record_journal->latest_record_id == NULL) {
+                return;
+        }
+
+        ret = asprintf(&tmpbuf, "%s/%s", RECORD_RETENTION_DIR,
+                       daemon->record_journal->latest_record_id);
+        if (ret == -1) {
+                telem_log(LOG_ERR, "Failed to allocate memory for record full path, aborting\n");
+                return;
+        }
+
+        tmpfile = fopen(tmpbuf, "w");
+        if (!tmpfile) {
+                telem_perror("Error opening local record copy temp file");
+                goto save_err;
+        }
+
+        // Save body
+        fprintf(tmpfile, "%s\n", body);
+        fclose(tmpfile);
+
+save_err:
+        free(tmpbuf);
+        return;
+}
+
 void spool_record(TelemDaemon *daemon, char *headers[], char *body)
 {
         int ret = 0;
@@ -599,7 +657,6 @@ void spool_record(TelemDaemon *daemon, char *headers[], char *body)
                 goto spool_err;
         }
 
-        //fp = fopen(spool_dir_path, const char *mode);
         tmpfile = fdopen(tmpfd, "a");
         if (!tmpfile) {
                 telem_perror("Error opening temp file");

src/telemdaemon.h

@@ -69,10 +69,12 @@ typedef struct TelemDaemon {
         int64_t byte_burst_limit;
         int byte_window_length;
         const char *rate_limit_strategy;
-
+        /* Spool configuration */
         bool is_spool_valid;
         long current_spool_size;
-
+        /* Record local copy and delivery  */
+        bool record_retention_enabled;
+        bool record_server_delivery_enabled;
         char *machine_id_override;
 } TelemDaemon;
 
@@ -90,6 +92,14 @@ void initialize_daemon(TelemDaemon *daemon);
  */
 void initialize_rate_limit(TelemDaemon *daemon);
 
+/**
+ * Initialize record delivery to remote and locally
+ * in the daemon
+ *
+ * @param daemon A pointer to the daemon structure
+ */
+void initialize_record_delivery(TelemDaemon *daemon);
+
 /**
  * Add poll fd struct to the array of pollfds.
  *
@@ -271,6 +281,14 @@ bool post_record_http(char *header_values[], char *body, bool spool);
  */
 void spool_record(TelemDaemon * daemon, char *headers[], char *body);
 
+/**
+ * Save a copy of the record payload locally
+ *
+ * @param body The payload of the record
+ *
+ */
+void save_local_copy(TelemDaemon *daemon, char *body);
+
 /**
  * Get random machine id stored in file
  *

src/telemetry.c

@@ -615,37 +615,6 @@ static int set_bios_version_header(struct telem_ref *t_ref)
         return status;
 }
 
-/**
- * Generates an alphanumeric id of length 32
- *
- * @param buff pointer to memory buffer, buffer should have
- *        at least 33 bytes of memory
- *
- * @return 0 if successful, or -1 if there's a problem
- *
- */
-static int gen_event_id(char *buff)
-{
-        int frandom = -1;
-        int result = -1;
-        uint64_t random_id[2] = { 0 };
-
-        frandom = open("/dev/urandom", O_RDONLY);
-        if (frandom < 0) {
-                return -1;
-        }
-
-        if (read(frandom, &random_id, sizeof(random_id)) == sizeof(random_id)) {
-                if (sprintf(buff, "%.16" PRIx64 "%.16" PRIx64, random_id[0], random_id[1]) == 32) {
-                        result = 0;
-                }
-        }
-
-        close(frandom);
-
-        return result;
-}
-
 /**
  * Sets the event_id header, this id is an identifier that multiple
  * records can share. This means that one event can lead to multiple
@@ -659,9 +628,9 @@ static int gen_event_id(char *buff)
 static int set_event_id_header(struct telem_ref *t_ref)
 {
         int rc = 0;
-        char buff[33];
+        char *buff = NULL;
 
-        rc = gen_event_id(buff);
+        rc = get_random_id(&buff);
 
         if (rc == 0) {
                 rc = set_header(
@@ -669,6 +638,7 @@ static int set_event_id_header(struct telem_ref *t_ref)
                         TM_EVENT_ID_STR, buff,
                         &(t_ref->record->header_size));
         }
+        free(buff);
 
         return rc;
 }

tests/check_config.c

@@ -55,6 +55,34 @@ START_TEST(check_read_valid_config)
 }
 END_TEST
 
+START_TEST(check_read_valid_config_defaults)
+{
+        char *config_file = TOPSRCDIR "/src/data/example.conf";
+        configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
+
+        int ret = read_config_from_file(config_file, &config);
+        ck_assert(ret == true);
+
+        // RECORD_RETENTION_ENABLED_DEFAULT = false
+        ck_assert(config.boolValues[CONF_RECORD_RETENTION_ENABLED] == RECORD_RETENTION_ENABLED_DEFAULT);
+        // RECORD_SERVER_DELIVERY_ENABLED_DEFAULT = true
+        ck_assert(config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED] == RECORD_SERVER_DELIVERY_ENABLED_DEFAULT);
+}
+END_TEST
+
+START_TEST(check_read_valid_config_record_retention_delivery)
+{
+        char *config_file = TOPSRCDIR "/src/data/example.1.conf";
+        configuration config = { { 0 }, { 0 }, { 0 }, false, NULL };
+
+        int ret = read_config_from_file(config_file, &config);
+        ck_assert(ret == true);
+
+        ck_assert(config.boolValues[CONF_RECORD_RETENTION_ENABLED] == true);
+        ck_assert(config.boolValues[CONF_RECORD_SERVER_DELIVERY_ENABLED] == false);
+}
+END_TEST
+
 START_TEST(check_config_initialised)
 {
         char *config_file = ABSTOPSRCDIR "/src/data/example.conf";
@@ -89,6 +117,8 @@ Suite *config_suite(void)
         TCase *t = tcase_create("config");
         tcase_add_test(t, check_read_config_for_invalid_file);
         tcase_add_test(t, check_read_valid_config);
+        tcase_add_test(t, check_read_valid_config_defaults);
+        tcase_add_test(t, check_read_valid_config_record_retention_delivery);
         tcase_add_test(t, check_config_initialised);
 
         // add more TCases here

tests/check_journal.c

@@ -159,28 +159,28 @@ void journal_entry_setup(void)
 START_TEST(check_journal_print)
 {
         int count = 0;
-        count = print_journal(journal, NULL, NULL, NULL, NULL);
+        count = print_journal(journal, NULL, NULL, NULL, NULL, 0);
         ck_assert_int_eq(count, K + 2);
 }
 END_TEST
 
 START_TEST(check_journal_filter_by_class)
 {
-        int result = print_journal(journal, "a/b/c", NULL, NULL, NULL);
+        int result = print_journal(journal, "a/b/c", NULL, NULL, NULL, 0);
         ck_assert_int_eq(result, 1);
 }
 END_TEST
 
 START_TEST(check_journal_filter_by_class_prefix)
 {
-        int result = print_journal(journal, "a/b/*", NULL, NULL, NULL);
+        int result = print_journal(journal, "a/b/*", NULL, NULL, NULL, 0);
         ck_assert_int_eq(result, 2);
 }
 END_TEST
 
 START_TEST(check_journal_filter_by_event_id)
 {
-        int result = print_journal(journal, NULL, NULL, eid, NULL);
+        int result = print_journal(journal, NULL, NULL, eid, NULL, 0);
         ck_assert(result == 2);
 }
 END_TEST