Convert patterns to JSON input files.

Tallow will now read JSON files from /usr/share/tallow/ and /etc/tallow
and parse them to retrieve filters and patterns. The sshd patterns
are converted to JSON and used to test this change.

If a file exists in /etc/tallow with the same name as a file in
/usr/share/tallow, only the file in /etc/tallow will be parsed.

This change allows much more dynamic insertion of rules and people
to create custom patterns and filters and monitor the logs of other
daemons besides sshd that may be subject to brutefoce login attempts.

Potential use cases:
- IMAP/POP services
- SMTP
- HTTP services permitted they log to syslog
- DNS servers logging malformed requests
- etc.

.gitignore

@@ -12,7 +12,7 @@ missing
 tallow
 tallow-*.tar.gz
 tallow-*/
-tallow.o
+*.o
 tallow.service
 *~
 DEADJOE

Makefile.am

@@ -1,14 +1,26 @@
 
-AM_CFLAGS = -g $(PCRE_CFLAGS) $(LIBSYSTEMD_CFLAGS) -Wall -Wno-uninitialized -W -D_FORTIFY_SOURCE=2
+AM_CFLAGS = -g \
+	$(JSON_C_CFLAGS) $(PCRE_CFLAGS) $(LIBSYSTEMD_CFLAGS) \
+	-Wall -Wno-uninitialized -W -D_FORTIFY_SOURCE=2
+
+AM_CPPFLAGS = \
+	-DDATADIR='"$(datadir)"' -DSYSCONFDIR='"$(sysconfdir)"'
 
 systemdsystemunitdir = @SYSTEMD_SYSTEMUNITDIR@
-systemdsystemunit_DATA = tallow.service
+systemdsystemunit_DATA = data/tallow.service
 
 sbin_PROGRAMS = tallow
-tallow_SOURCES = tallow.c
-tallow_LDADD = $(PCRE_LIBS) $(LIBSYSTEMD_LIBS)
+tallow_SOURCES = tallow.c json.c json.h data.c data.h
+tallow_LDADD = $(JSON_C_LIBS) $(PCRE_LIBS) $(LIBSYSTEMD_LIBS)
 
-EXTRA_DIST = AUTHORS COPYING INSTALL README.md tallow.service.in tallow.conf.5.md tallow.1.md
+pkgdata_DATA = data/sshd.json
+
+EXTRA_DIST = \
+	AUTHORS COPYING INSTALL README.md \
+	data/tallow.service.in \
+	data/sshd.json \
+	tallow.conf.5.md \
+	tallow.1.md
 
 dist_man_MANS = tallow.1 tallow.conf.5
 
@@ -23,4 +35,3 @@ tallow.conf.5: tallow.conf.5.md
 
 tallow.1: tallow.1.md
 	ronn -r tallow.1.md --pipe > tallow.1
-

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.64])
-AC_INIT([tallow], [10], [auke-jan.h.kok@intel.com])
+AC_INIT([tallow], [15], [auke-jan.h.kok@intel.com])
 AM_INIT_AUTOMAKE([foreign])
 AC_CONFIG_FILES([Makefile])
 
@@ -11,6 +11,7 @@ AC_PROG_CC
 AC_PROG_INSTALL
 
 PKG_CHECK_MODULES(PCRE, libpcre)
+PKG_CHECK_MODULES(JSON_C, json-c)
 PKG_CHECK_MODULES(LIBSYSTEMD, libsystemd,, [PKG_CHECK_MODULES(LIBSYSTEMD, libsystemd-journal)])
 AC_SUBST(LIBSYSTEMD_CFLAGS)
 AC_SUBST(LIBSYSTEMD_LIBS)
@@ -28,5 +29,5 @@ fi
 AC_CHECK_HEADERS([stdlib.h stdio.h string.h stdarg.h limits.h sys/time.h])
 
 AC_OUTPUT([
-tallow.service
+data/tallow.service
 ])

data.c

@@ -0,0 +1,171 @@
+/*
+ * data.h - IP block sshd login abuse
+ *
+ * (C) Copyright 2019 Intel Corporation
+ * Authors:
+ *     Auke Kok <auke-jan.h.kok@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 3
+ * of the License.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/time.h>
+
+#include <pcre.h>
+
+#include "data.h"
+
+struct block_struct *blocks;
+struct pattern_struct *patterns;
+struct filter_struct *filters;
+struct whitelist_struct *whitelist;
+
+void filter_add(const char *filter)
+{
+	struct filter_struct *h = filters;
+
+	/* filter duplicates */
+	while (h) {
+		if (strcmp(h->filter, filter) == 0)
+			return;
+
+		h = h->next;
+	}
+
+	struct filter_struct *f = calloc(1, sizeof(struct filter_struct));
+	if (!f) {
+		perror("calloc()");
+		exit(EXIT_FAILURE);
+	}
+
+	f->filter = strdup(filter);
+
+	h = filters;
+	if (!h) {
+		filters = f;
+	} else {
+		while (h->next)
+			h = h->next;
+		h->next = f;
+	}
+}
+
+void pattern_add(const char *pattern, int ban, double score)
+{
+	struct pattern_struct *p = calloc(1, sizeof(struct pattern_struct));
+	if (!p) {
+		perror("calloc()");
+		exit(EXIT_FAILURE);
+	}
+
+	p->pattern = strdup(pattern);
+	p->instant_block = ban;
+	p->weight = score;
+
+	const char *pcre_err;
+	int err;
+	p->re = pcre_compile(pattern, 0, &pcre_err, &err, NULL);
+	if (!p->re) {
+		fprintf(stderr, "PCRE compilation failed. Offset %d: %s\n",
+			err, pcre_err);
+		exit(EXIT_FAILURE);
+	}
+
+	struct pattern_struct *h = patterns;
+	if (!h) {
+		patterns = p;
+	} else {
+		while (h->next)
+			h = h->next;
+		h->next = p;
+	}
+}
+
+void whitelist_add(const char *ip)
+{
+	struct whitelist_struct *w = calloc(1, sizeof(struct whitelist_struct));
+
+	if (!w) {
+		perror("calloc()");
+		exit(EXIT_FAILURE);
+	}
+
+	w->ip = strdup(ip);
+
+	size_t l = strlen(ip);
+	if ((ip[l-1] == '.') || (ip[l-1] == ':'))
+		w->len = l;
+	else
+		w->len = -1;
+
+	struct whitelist_struct *h = whitelist;
+	if (!h) {
+		whitelist = w;
+	} else {
+		while (h->next)
+			h = h->next;
+		h->next = w;
+	}
+}
+
+bool whitelist_find(const char *ip)
+{
+	struct whitelist_struct *w = whitelist;
+	while (w) {
+		if (w->len > 0) {
+			if (!strncmp(w->ip, ip, w->len))
+				return (true);
+		} else {
+			if (!strcmp(w->ip, ip))
+				return (true);
+		}
+		w = w->next;
+	}
+
+	return (false);
+}
+
+void prune(int expires)
+{
+	struct block_struct *s = blocks;
+	struct block_struct *p;
+	struct timeval tv;
+
+	(void) gettimeofday(&tv, NULL);
+	p = NULL;
+
+	while (s) {
+		/*
+		 * Expire all records, but if they are blocked, make sure to
+		 * expire them *before* the ipset rule expires, otherwise
+		 * you might get an IP to bypass checks.
+		 */
+		time_t age = tv.tv_sec - s->time.tv_sec;
+		if ((age > expires) ||
+		    ((s->blocked) && (age > expires / 2))) {
+			dbg("Expired record for %s\n", s->ip);
+			if (p) {
+				p->next = s->next;
+				free(s->ip);
+				free(s);
+				s = p->next;
+				continue;
+			} else {
+				blocks = s->next;
+				free(s->ip);
+				free(s);
+				s = blocks;
+				p = NULL;
+				continue;
+			}
+		}
+		p = s;
+		s = s->next;
+	}
+}

data.h

@@ -0,0 +1,60 @@
+/*
+ * data.h - IP block sshd login abuse
+ *
+ * (C) Copyright 2019 Intel Corporation
+ * Authors:
+ *     Auke Kok <auke-jan.h.kok@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 3
+ * of the License.
+ */
+
+#pragma once
+
+#include <pcre.h>
+
+#ifdef DEBUG
+#define dbg(args...) fprintf(stderr, ##args)
+#else
+#define dbg(args...) do {} while (0)
+#endif
+
+struct block_struct {
+	char *ip;
+	float score;
+	struct timeval time;
+	struct block_struct *next;
+	bool blocked;
+};
+
+struct whitelist_struct {
+	char *ip;
+	size_t len;
+	struct whitelist_struct *next;
+};
+
+struct pattern_struct {
+	int instant_block;
+	float weight;
+	char *pattern;
+	pcre *re;
+	struct pattern_struct *next;
+};
+
+struct filter_struct {
+	char *filter;
+	struct filter_struct *next;
+};
+
+extern struct block_struct *blocks;
+extern struct pattern_struct *patterns;
+extern struct filter_struct *filters;
+extern struct whitelist_struct *whitelist;
+
+void filter_add(const char *filter);
+void pattern_add(const char *pattern, int ban, double score);
+void whitelist_add(const char *ip);
+bool whitelist_find(const char *ip);
+void prune(int expires);

data/sshd.json

@@ -0,0 +1,57 @@
+[
+  {
+    "filter": "SYSLOG_IDENTIFIER=sshd",
+    "items": [
+      {
+        "ban": 0,
+        "score": 0.2,
+        "pattern": "MESSAGE=Failed .* for .* from ([0-9a-z:.]+) port \\d+ ssh2"
+      },
+      {
+        "ban": 0,
+        "score": 0.2,
+        "pattern": "MESSAGE=error: PAM: Authentication failure for .* from ([0-9a-z:.]+)"
+      },
+      {
+        "ban": 10,
+        "score": 0.2,
+        "pattern": "MESSAGE=Invalid user .* from ([0-9a-z:.]+) port \\d+"
+      },
+      {
+        "ban": 10,
+        "score": 0.3,
+        "pattern": "MESSAGE=Did not receive identification string from ([0-9a-z:.]+) port \\d+"
+      },
+      {
+        "ban": 15,
+        "score": 0.4,
+        "pattern": "MESSAGE=Bad protocol version identification .* from ([0-9a-z:.]+)"
+      },
+      {
+        "ban": 15,
+        "score": 0.4,
+        "pattern": "MESSAGE=Connection closed by authenticating user .* ([0-9a-z:.]+) port \\d+"
+      },
+      {
+        "ban": 10,
+        "score": 0.3,
+        "pattern": "MESSAGE=Received disconnect from ([0-9a-z:.]+) port .*\\[preauth\\]"
+      },
+      {
+        "ban": 10,
+        "score": 0.3,
+        "pattern": "MESSAGE=Connection closed by ([0-9a-z:.]+) port .*\\[preauth\\]"
+      },
+      {
+        "ban": 30,
+        "score": 0.5,
+        "pattern": "MESSAGE=Failed .* for root from ([0-9a-z:.]+) port \\d+ ssh2"
+      },
+      {
+        "ban": 60,
+        "score": 0.6,
+        "pattern": "MESSAGE=Unable to negotiate with ([0-9a-z:.]+) port \\d+: no matching key exchange method found."
+      }
+    ]
+  }
+]

json.c

@@ -0,0 +1,218 @@
+/*
+ * json.c - IP block sshd login abuse
+ *
+ * (C) Copyright 2019 Intel Corporation
+ * Authors:
+ *     Auke Kok <auke-jan.h.kok@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 3
+ * of the License.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <dirent.h>
+#include <fcntl.h>
+
+#include <json-c/json.h>
+
+#include "data.h"
+
+static const char *s_pattern;
+static const char *s_filter;
+static int s_ban;
+static double s_score;
+static bool g_filter, g_ban, g_score, g_pattern = false;
+
+static int json_parse(json_object *ob)
+{
+	int count = 0;
+	bool l_filter, l_ban, l_score, l_pattern = false;
+
+	json_object_object_foreach(ob, key, val) {
+		enum json_type type;
+		type = json_object_get_type(val);
+		switch (type) {
+		case json_type_null:
+			break;
+		case json_type_boolean:
+			break;
+		case json_type_double:
+			if (strcmp(key, "score") == 0) {
+				l_score = g_score = true;
+				s_score = json_object_get_double(val);
+			} else {
+				fprintf(stderr, "Invalid JSON key \"%s\"\n", key);
+				return (0);
+			}
+			break;
+		case json_type_int:
+			if (strcmp(key, "ban") == 0) {
+				l_ban = g_ban = true;
+				s_ban = json_object_get_int(val);
+			} else {
+				fprintf(stderr, "Invalid JSON key \"%s\"\n", key);
+				return (0);
+			}
+			break;
+		case json_type_string:
+			if (strcmp(key, "filter") == 0) {
+				l_filter = g_filter = true;
+				s_filter = json_object_get_string(val);
+			} else if (strcmp(key, "pattern") == 0) {
+				l_pattern = g_pattern = true;
+				s_pattern = json_object_get_string(val);
+			} else {
+				fprintf(stderr, "Invalid JSON key \"%s\"\n", key);
+				return (0);
+			}
+			break;
+		case json_type_array:
+			ob = json_object_object_get(ob, key);
+			int len = json_object_array_length(ob);
+			json_object *val;
+			for (int i = 0; i < len; i++) {
+				val = json_object_array_get_idx(ob, i);
+				count += json_parse(val);
+			}
+			break;
+		case json_type_object:
+			ob = json_object_object_get(ob, key);
+			count += json_parse(ob);
+			break;
+		}
+	}
+
+	/* check and finish if can */
+	if (g_score && g_ban && g_pattern && g_filter) {
+#ifdef DEBUG
+		fprintf(stderr, "Adding: %s %s %d %lf\n", s_filter, s_pattern, s_ban, s_score);
+#endif
+		filter_add(s_filter);
+		pattern_add(s_pattern, s_ban, s_score);
+		count++;
+	}
+
+	/* cleanup */
+	if (l_score) {
+		l_score = g_score = false;
+	} else if (l_ban) {
+		l_ban = g_ban = false;
+	} else if (l_pattern) {
+		l_pattern = g_pattern = false;
+	} else if (l_filter) {
+		l_filter = g_filter = false;
+	}
+
+	return (count);
+}
+
+static int json_load_file(const char* file)
+{
+	int count = 0;
+	char *json;
+	int fd;
+	struct stat st;
+	fd = open(file, O_RDONLY);
+	fstat(fd, &st);
+	json = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	close(fd);
+
+	struct json_object *obj;
+
+	obj = json_tokener_parse(json);
+	if (json_object_is_type(obj,json_type_array)) {
+		int len = json_object_array_length(obj);
+		json_object *val;
+		for (int i = 0; i < len; i++) {
+			val = json_object_array_get_idx(obj, i);
+			count += json_parse(val);
+		}
+	} else if (json_object_is_type(obj, json_type_object)) {
+		count += json_parse(obj);
+	} else {
+		fprintf(stderr, "This does not look like JSON: %s\n", file);
+		return (0);
+	}
+
+	json_object_put(obj);
+
+	munmap(json, st.st_size);
+	return (count);
+}
+
+static int json_load_dir(const char *dir)
+{
+	int count = 0;
+
+	DIR *d = opendir(dir);
+	if (!d) {
+		fprintf(stderr, "Skipped reading %s: %s\n", dir, strerror(errno));
+		return (count);
+	}
+	struct dirent *entry;
+	for (;;) {
+		entry = readdir(d);
+		if (!entry)
+			break;
+		size_t l = strlen(entry->d_name);
+		if (l < strlen(".json"))
+			continue;
+		if (strcmp(entry->d_name + l - strlen(".json"), ".json") == 0) {
+			char *p;
+			if (!asprintf(&p, "%s/%s", dir, entry->d_name)) {
+				fprintf(stderr, "asprintf: %s\n", strerror(errno));
+				exit(EXIT_FAILURE);
+			}
+
+			/* allow /etc/tallow files to override /usr/share/tallow files */
+			if (strcmp(dir, DATADIR "/" PACKAGE_NAME) == 0) {
+				char *sp;
+				if (!asprintf(&sp, SYSCONFDIR "/" PACKAGE_NAME "/%s", entry->d_name)) {
+					fprintf(stderr, "asprintf: %s\n", strerror(errno));
+					exit(EXIT_FAILURE);
+				}
+
+				struct stat st;
+				if (stat(sp, &st) == 0) {
+					dbg("Skipped %s\n", sp);
+					free(sp);
+					continue;
+				}
+				free(sp);
+			}
+
+			int c = json_load_file(p);
+			fprintf(stderr, "%s: %d patterns\n", p, c);
+			count += c;
+			free(p);
+		}
+
+	}
+	closedir(d);
+
+	return (count);
+}
+
+void json_load_patterns(void)
+{
+	int count = 0;
+	count += json_load_dir(DATADIR "/" PACKAGE_NAME);
+	count += json_load_dir(SYSCONFDIR "/" PACKAGE_NAME);
+	fprintf(stderr, "Loaded %d patterns total\n", count);
+	if (count < 1) {
+		/* consider sending a special exit code here */
+		fprintf(stderr, "No patterns loaded, nothing to do!\n");
+		exit(EXIT_SUCCESS);
+	}
+}

json.h

@@ -0,0 +1,16 @@
+/*
+ * json.h - IP block sshd login abuse
+ *
+ * (C) Copyright 2019 Intel Corporation
+ * Authors:
+ *     Auke Kok <auke-jan.h.kok@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 3
+ * of the License.
+ */
+
+#pragma once
+
+void json_load_patterns(void);

tallow.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "TALLOW" "1" "March 2018" "" ""
+.TH "TALLOW" "1" "October 2018" "" ""
 .
 .SH "NAME"
 \fBtallow\fR

tallow.c

@@ -1,7 +1,7 @@
 /*
  * tallow.c - IP block sshd login abuse
  *
- * (C) Copyright 2015 Intel Corporation
+ * (C) Copyright 2015-2019 Intel Corporation
  * Authors:
  *     Auke Kok <auke-jan.h.kok@intel.com>
  *
@@ -20,55 +20,15 @@
 #include <unistd.h>
 #include <limits.h>
 #include <sys/time.h>
-#include <pcre.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <linux/limits.h>
 
+#include <pcre.h>
 #include <systemd/sd-journal.h>
 
-#ifdef DEBUG
-#define dbg(args...) fprintf(stderr, ##args)
-#else
-#define dbg(args...) do {} while (0)
-#endif
-
-struct tallow_struct {
-	char *ip;
-	float score;
-	struct timeval time;
-	struct tallow_struct *next;
-	bool blocked;
-};
-
-static struct tallow_struct *head;
-
-struct whitelist_struct {
-	char *ip;
-	size_t len;
-	struct whitelist_struct *next;
-};
-
-static struct whitelist_struct *whitelist;
-
-#define FILTER_STRING "SYSLOG_IDENTIFIER=sshd"
-struct pattern_struct {
-	int instant_block;
-	float weight;
-	char *pattern;
-	pcre *re;
-};
-
-#define PATTERN_COUNT 10
-static struct pattern_struct patterns[PATTERN_COUNT] = {
-	{ 0, 0.3, "MESSAGE=Failed .* for .* from ([0-9a-z:.]+) port \\d+ ssh2", NULL},
-	{ 0, 0.3, "MESSAGE=error: PAM: Authentication failure for .* from ([0-9a-z:.]+)", NULL},
-	{15, 0.3, "MESSAGE=Invalid user .* from ([0-9a-z:.]+) port \\d+", NULL},
-	{15, 0.3, "MESSAGE=Did not receive identification string from ([0-9a-z:.]+) port \\d+", NULL},
-	{15, 0.4, "MESSAGE=Bad protocol version identification .* from ([0-9a-z:.]+)", NULL},
-	{15, 0.4, "MESSAGE=Connection closed by authenticating user .* ([0-9a-z:.]+) port \\d+", NULL},
-	{15, 0.4, "MESSAGE=Received disconnect from ([0-9a-z:.]+) port .*\\[preauth\\]", NULL},
-	{15, 0.4, "MESSAGE=Connection closed by ([0-9a-z:.]+) port .*\\[preauth\\]", NULL},
-	{30, 0.5, "MESSAGE=Failed .* for root from ([0-9a-z:.]+) port \\d+ ssh2", NULL},
-	{60, 0.6, "MESSAGE=Unable to negotiate with ([0-9a-z:.]+) port \\d+: no matching key exchange method found.", NULL}
-};
+#include "json.h"
+#include "data.h"
 
 #define MAX_OFFSETS 30
 
@@ -143,11 +103,10 @@ static void setup(void)
 	}
 }
 
-static void block(struct tallow_struct *s, int instant_block)
+static void block(struct block_struct *s, int instant_block)
 {
 	setup();
 
-
 	if (strchr(s->ip, ':')) {
 		if (has_ipv6) {
 			if (instant_block > 0) {
@@ -175,38 +134,10 @@ static void block(struct tallow_struct *s, int instant_block)
 	}
 }
 
-static void whitelist_add(char *ip)
+void find(const char *ip, float weight, int instant_block)
 {
-	struct whitelist_struct *w = whitelist;
-	struct whitelist_struct *n;
-
-	while (w && w->next)
-		w = w->next;
-
-	n = calloc(1, sizeof(struct whitelist_struct));
-	if (!n) {
-		fprintf(stderr, "Out of memory.\n");
-		exit(EXIT_FAILURE);
-	}
-
-	n->ip = strdup(ip);
-	size_t l = strlen(ip);
-	if ((ip[l-1] == '.') || (ip[l-1] == ':'))
-		n->len = l;
-	else
-		n->len = -1;
-
-	if (!whitelist)
-		whitelist = n;
-	else
-		w->next = n;
-}
-
-static void find(const char *ip, float weight, int instant_block)
-{
-	struct tallow_struct *s = head;
-	struct tallow_struct *n;
-	struct whitelist_struct *w = whitelist;
+	struct block_struct *s = blocks;
+	struct block_struct *n;
 
 	if (!ip)
 		return;
@@ -219,17 +150,8 @@ static void find(const char *ip, float weight, int instant_block)
 	if (strspn(ip, "0123456789abcdef:.") != strlen(ip))
 		return;
 
-	/* whitelist */
-	while (w) {
-		if (w->len > 0) {
-			if (!strncmp(w->ip, ip, w->len))
-				return;
-		} else {
-			if (!strcmp(w->ip, ip))
-				return;
-		}
-		w = w->next;
-	}
+	if (whitelist_find(ip))
+		return;
 
 	/* walk and update entry */
 	while (s) {
@@ -258,14 +180,14 @@ static void find(const char *ip, float weight, int instant_block)
 	}
 
 	/* append */
-	n = calloc(1, sizeof(struct tallow_struct));
+	n = calloc(1, sizeof(struct block_struct));
 	if (!n) {
 		fprintf(stderr, "Out of memory.\n");
 		exit(EXIT_FAILURE);
 	}
 
-	if (!head)
-		head = n;
+	if (!blocks)
+		blocks = n;
 	else
 		s->next = n;
 
@@ -288,7 +210,7 @@ static void find(const char *ip, float weight, int instant_block)
 static void sigusr1(int u __attribute__ ((unused)))
 {
 	fprintf(stderr, "Dumping score list on request:\n");
-	struct tallow_struct *s = head;
+	struct block_struct *s = blocks;
 	while (s) {
 		fprintf(stderr, "%ld %s %1.3f\n", s->time.tv_sec, s->ip, s->score);
 		s = s->next;
@@ -296,44 +218,6 @@ static void sigusr1(int u __attribute__ ((unused)))
 }
 #endif
 
-static void prune(void)
-{
-	struct tallow_struct *s = head;
-	struct tallow_struct *p;
-	struct timeval tv;
-
-	(void) gettimeofday(&tv, NULL);
-	p = NULL;
-
-	while (s) {
-		/*
-		 * Expire all records, but if they are blocked, make sure to
-		 * expire them *before* the ipset rule expires, otherwise
-		 * you might get an IP to bypass checks.
-		 */
-		time_t age = tv.tv_sec - s->time.tv_sec;
-		if ((age > expires) ||
-		    ((s->blocked) && (age > expires / 2))) {
-			dbg("Expired record for %s\n", s->ip);
-			if (p) {
-				p->next = s->next;
-				free(s->ip);
-				free(s);
-				s = p->next;
-				continue;
-			} else {
-				head = s->next;
-				free(s->ip);
-				free(s);
-				s = head;
-				p = NULL;
-				continue;
-			}
-		}
-		p = s;
-		s = s->next;
-	}
-}
 
 int main(void)
 {
@@ -341,9 +225,13 @@ int main(void)
 	FILE *f;
 	int timeout = 60;
 
+	json_load_patterns();
+
 	strcpy(ipt_path, "/usr/sbin");
 
 #ifdef DEBUG
+	fprintf(stderr, "Debug output enabled. Send SIGUSR1 to dump internal state table\n");
+
 	struct sigaction s;
 
 	memset(&s, 0, sizeof(struct sigaction));
@@ -354,7 +242,7 @@ int main(void)
 	if (access("/proc/sys/net/ipv6", R_OK | X_OK) == 0)
 		has_ipv6 = 1;
 
-	f = fopen("/etc/tallow.conf", "r");
+	f = fopen(SYSCONFDIR "/tallow.conf", "r");
 	if (f) {
 		char buf[256];
 		char *key;
@@ -394,8 +282,11 @@ int main(void)
 	if (!has_ipv6)
 		fprintf(stdout, "ipv6 support disabled.\n");
 
-	if (!whitelist)
+	if (!whitelist) {
 		whitelist_add("127.0.0.1");
+		whitelist_add("192.168.");
+		whitelist_add("10.");
+	}
 
 	r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY);
 	if (r < 0) {
@@ -403,9 +294,14 @@ int main(void)
 		exit(EXIT_FAILURE);
 	}
 
+	/* add all filters */
+	struct filter_struct *flt = filters;
+	while (flt) {
+		sd_journal_add_match(j, flt->filter, 0);
+		flt = flt->next;
+	}
 
-	/* ffwd journal */
-	sd_journal_add_match(j, FILTER_STRING, 0);
+	/* go to the tail and wait */
 	r = sd_journal_seek_tail(j);
 	sd_journal_wait(j, (uint64_t) 0);
 	dbg("sd_journal_seek_tail() returned %d\n", r);
@@ -413,18 +309,7 @@ int main(void)
 		r++;
 	dbg("Forwarded through %d items in the journal to reach the end\n", r);
 
-	fprintf(stderr, "Started\n");
-
-	for (int i = 0; i < PATTERN_COUNT; i++) {
-		int err;
-		const char *pcre_err;
-		patterns[i].re = pcre_compile(patterns[i].pattern, 0, &pcre_err, &err, NULL);
-		if (!patterns[i].re) {
-			fprintf(stderr, "PCRE compilation failed. Pattern %d, offset %d: %s\n",
-				i, err, pcre_err);
-			exit(EXIT_FAILURE);
-		}
-	}
+	fprintf(stderr, PACKAGE_STRING " Started\n");
 
 	for (;;) {
 		const void *d;
@@ -434,6 +319,9 @@ int main(void)
 		if (r == SD_JOURNAL_INVALIDATE) {
 			fprintf(stderr, "Journal was rotated, resetting\n");
 			sd_journal_seek_tail(j);
+		} else if (r == SD_JOURNAL_NOP) {
+			dbg("Timeout reached, waiting again\n");
+			continue;
 		}
 
 		while (sd_journal_next(j) != 0) {
@@ -441,31 +329,34 @@ int main(void)
 
 			if (sd_journal_get_data(j, "MESSAGE", &d, &l) < 0) {
 				fprintf(stderr, "Failed to read message field: %s\n", strerror(-r));
-				continue;
+				break;
 			}
 
 			m = strndup(d, l+1);
 			m[l] = '\0';
 
-			for (int i = 0; i < PATTERN_COUNT; i++) {
+			struct pattern_struct *pat = patterns;
+			while (pat) {
 				int off[MAX_OFFSETS];
-				int ret = pcre_exec(patterns[i].re, NULL, m, l, 0, 0, off, MAX_OFFSETS);
+				int ret = pcre_exec(pat->re, NULL, m, l, 0, 0, off, MAX_OFFSETS);
 				if (ret == 2) {
 					const char *s;
 					ret = pcre_get_substring(m, off, 2, 1, &s);
 					if (ret > 0) {
-						dbg("%s == %s\n", s, patterns[i].pattern);
-						find(s, patterns[i].weight, patterns[i].instant_block);
+						dbg("%s == %s\n", s, pat->pattern);
+						find(s, pat->weight, pat->instant_block);
 						pcre_free_substring(s);
 					}
 				}
+
+				pat = pat->next;
 			}
 
 			free(m);
 
 		}
 
-		prune();
+		prune(expires);
 	}
 
 	sd_journal_close(j);

tallow.conf.5

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "TALLOW" "5" "January 2018" "" ""
+.TH "TALLOW" "5" "October 2018" "" ""
 .
 .SH "NAME"
 \fBtallow\fR
@@ -25,13 +25,13 @@ This file is read on startup by the tallow(1) daemon, and can be used to provide
 \fBexpires\fR=\fB<int>\fR The number of seconds that IP addresses are blocked for\. Note that due to the implementation, IP addresses may be blocked for much longer than this period\. If IP addresses are seen, but not blocked within this period, they are also removed from the watch list\. Defaults to 3600s\.
 .
 .P
-\fBwhitelist\fR=\fB<ip address|pattern>\fR Specify an IP address or \fBpattern\fR that should never be blocked\. Multiple IP addresses can be included by repeating the \fBwhitelist\fR option several times\. By default, only 127\.0\.0\.1 is whitelisted\.
+\fBwhitelist\fR=\fB<ip address|pattern>\fR Specify an IP address or \fBpattern\fR that should never be blocked\. Multiple IP addresses can be included by repeating the \fBwhitelist\fR option several times\. By default, 127\.0\.0\.1, 192\.168\., and 10\. are whitelisted\. If you create a manual whitelist, you must include these entries if you want to continue them to be whitelisted as well, otherwise they will be omitted from the whitelist\.
 .
 .P
 If the last character of the listed ip adress is a \fB\.\fR or a \fB:\fR, then the matching is only performed on the leftmost characters of an IP address against the whitelist entry\. For instance, if you whitelist \fB10\.\fR then all IP addresses in the \fB10/8\fR subnet mask will match this whitelist entry and never be blocked\.
 .
 .P
-\fBipv6\fR=\fB<0|1>\fR Enable of disable ipv6 (ip6tables) support\. Ipv6 is disabled automatically on systems that do not appear to have ipv6 support and enabled when ipv6 is present\. Use this option to explicitly disable ipv6 support if your system does not have ipv6 or is missing ip6tables\. Even with ipv6 disabled, tallow will track and log ipv6 addresses\.
+\fBipv6\fR=\fB<0|1>\fR Enable or disable ipv6 (ip6tables) support\. Ipv6 is disabled automatically on systems that do not appear to have ipv6 support and enabled when ipv6 is present\. Use this option to explicitly disable ipv6 support if your system does not have ipv6 or is missing ip6tables\. Even with ipv6 disabled, tallow will track and log ipv6 addresses\.
 .
 .P
 \fBnocreate\fR=\fB<0|1>\fR Disable the creation of iptables rules and ipset sets\. By default, tallow will create new iptables(1) and ip6tables(1) rules when needed automatically\. If set to \fB1\fR, \fBtallow(1)\fR will not create any new iptables rules or ipset sets to work\. You should create them manually before tallow starts up and remove them afterwards\. To create them manually, you can use the following commands:

tallow.conf.5.md

@@ -33,8 +33,10 @@ watch list. Defaults to 3600s.
 `whitelist`=`<ip address|pattern>`
 Specify an IP address or `pattern` that should never be
 blocked. Multiple IP addresses can be included by repeating the
-`whitelist` option several times. By default, only 127.0.0.1 is
-whitelisted.
+`whitelist` option several times. By default, 127.0.0.1, 192.168., and
+10. are whitelisted. If you create a manual whitelist, you must include
+these entries if you want to continue them to be whitelisted as
+well, otherwise they will be omitted from the whitelist.
 
 If the last character of the listed ip adress is a `.` or a `:`, then
 the matching is only performed on the leftmost characters of an IP
@@ -43,7 +45,7 @@ address against the whitelist entry. For instance, if you whitelist
 whitelist entry and never be blocked.
 
 `ipv6`=`<0|1>`
-Enable of disable ipv6 (ip6tables) support. Ipv6 is disabled
+Enable or disable ipv6 (ip6tables) support. Ipv6 is disabled
 automatically on systems that do not appear to have ipv6 support
 and enabled when ipv6 is present. Use this option to explicitly
 disable ipv6 support if your system does not have ipv6 or is