v21

Add a tallow.patterns man page which explains the json configuration
files that contain regex patterns and banning thresholds. This
functionality was added by 9174590b04.

.github/workflows/ccpp.yml

@@ -0,0 +1,23 @@
+name: C/C++ CI
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: install dependencies
+      run: sudo apt-get install check libjson-c-dev libpcre2-dev libsystemd-dev
+    - name: autogen
+      run: sh autogen.sh
+    - name: configure
+      run: ./configure
+    - name: make
+      run: make
+    - name: make check
+      run: make check
+    - name: make distcheck
+      run: make distcheck

Makefile.am

@@ -25,9 +25,10 @@ EXTRA_DIST = \
 	data/tallow.service.in \
 	data/sshd.json \
 	man/tallow.conf.5.md \
+	man/tallow.patterns.5.md \
 	man/tallow.1.md
 
-dist_man_MANS = man/tallow.1 man/tallow.conf.5
+dist_man_MANS = man/tallow.1 man/tallow.conf.5 man/tallow.patterns.5
 
 dist_doc_DATA = tallow.conf
 
@@ -40,3 +41,4 @@ man/%.5: man/%.5.md
 
 man/%.1: man/%.1.md
 	ronn -r $< --pipe > $@
+

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.64])
-AC_INIT([tallow], [17], [auke-jan.h.kok@intel.com])
+AC_INIT([tallow], [21], [auke-jan.h.kok@intel.com])
 AM_INIT_AUTOMAKE([foreign -Wall -Werror -Wno-portability silent-rules subdir-objects color-tests
 	no-dist-gzip dist-xz])
 AC_CONFIG_FILES([Makefile])

man/tallow.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "TALLOW" "1" "October 2018" "" ""
+.TH "TALLOW" "1" "February 2020" "" ""
 .
 .SH "NAME"
 \fBtallow\fR
@@ -31,7 +31,7 @@ The \fBtallow\fR daemon itself has no runtime configuration\. All configuration
 The \fBUSR1\fR signal causes \fBtallow\fR to print out it\'s internal tracking table of IP addresses\. This requires that tallow is compiled with the \fB\-DDEBUG=1\fR symbol passed to the compiler\.
 .
 .SH "SEE ALSO"
-systemd\-journald(1), iptables(1), ipset(1), tallow\.conf(5)
+systemd\-journald(1), iptables(1), ipset(1), tallow\.conf(5), tallow\.patterns(5)
 .
 .SH "BUGS"
 \fBtallow\fR is \fBNOT A SECURITY SOLUTION\fR, nor does it protect against random password logins\. A attacker may still be able to logon to your systems if you allow password logins\.

man/tallow.1.md

@@ -48,7 +48,7 @@ the `-DDEBUG=1` symbol passed to the compiler.
 
 ## SEE ALSO
 
-systemd-journald(1), iptables(1), ipset(1), tallow.conf(5)
+systemd-journald(1), iptables(1), ipset(1), tallow.conf(5), tallow.patterns(5)
 
 ## BUGS
 

man/tallow.conf.5

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "TALLOW" "5" "October 2019" "" ""
+.TH "TALLOW" "5" "February 2020" "" ""
 .
 .SH "NAME"
 \fBtallow\fR
@@ -19,7 +19,10 @@ tallow\.conf \- Tallow daemon configuration file
 This file is read on startup by the tallow(1) daemon, and can be used to provide options to the tallow daemon\. If not present, tallow will operate with built\-in defaults\.
 .
 .SH "OPTIONS"
-\fBipt_path\fR=\fB<string>\fR Specifies the location of the ipset(1) program and iptables(1), ip6tables(1), or firewall\-cmd(1) programs\. By default, tallow will look in "/usr/sbin" for them\.
+\fBfwcmd_path\fR=\fB<string>\fR Specifies the location of the ipset(1) firewall\-cmd(1) programs\. By default, tallow will look in "/usr/sbin" for them\.
+.
+.P
+\fBipt_path\fR=\fB<string>\fR Specifies the location of the ipset(1) program and iptables(1) or ip6tables(1) programs\. By default, tallow will look in "/usr/sbin" for them\.
 .
 .P
 \fBexpires\fR=\fB<int>\fR The number of seconds that IP addresses are blocked for\. Note that due to the implementation, IP addresses may be blocked for much longer than this period\. If IP addresses are seen, but not blocked within this period, they are also removed from the watch list\. Defaults to 3600s\.
@@ -71,7 +74,7 @@ Use the following commands if you\'re using firewalld(1):
 .IP "" 0
 .
 .SH "SEE ALSO"
-tallow(1)
+tallow(1), tallow\.patterns(5)
 .
 .SH "AUTHOR"
 Auke Kok \fIauke\-jan\.h\.kok@intel\.com\fR

man/tallow.conf.5.md

@@ -19,10 +19,14 @@ tallow will operate with built-in defaults.
 
 ## OPTIONS
 
+`fwcmd_path`=`<string>`
+Specifies the location of the ipset(1) firewall-cmd(1) programs. By
+default, tallow will look in "/usr/sbin" for them.
+
 `ipt_path`=`<string>`
-Specifies the location of the ipset(1) program and iptables(1), 
-ip6tables(1), or firewall-cmd(1) programs. By default, tallow will 
-look in "/usr/sbin" for them.
+Specifies the location of the ipset(1) program and iptables(1) or
+ip6tables(1) programs. By default, tallow will look in "/usr/sbin"
+for them.
 
 `expires`=`<int>`
 The number of seconds that IP addresses are blocked for. Note that
@@ -58,7 +62,7 @@ default, tallow will create new firewall-cmd(1) or iptables(1) and ip6tables(1)
 rules when needed automatically. If set to `1`, `tallow(1)` will not create any
 new firewall DROP rules or ipset sets that are needed work. You should create
 them manually before tallow starts up and remove them afterwards using the sets
-of commands below. 
+of commands below.
 
 Use the following commands if you're using iptables(1):
 
@@ -83,7 +87,7 @@ Use the following commands if you're using firewalld(1):
 
 ## SEE ALSO
 
-tallow(1)
+tallow(1), tallow.patterns(5)
 
 ## AUTHOR
 

man/tallow.patterns.5

@@ -0,0 +1,128 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "TALLOW" "5" "February 2020" "" ""
+.
+.SH "NAME"
+\fBtallow\fR
+.
+.SH "tallow\.patterns"
+Tallow pattern matching configuration files\.
+.
+.SH "SYNOPSIS"
+tallow(1) uses regular expressions to match journal entries and extract an IP address from them\. JSON files are used to configure the patterns and banning thresholds used by tallow(1)\.
+.
+.P
+\fB/etc/tallow/*\.json\fR \fB/usr/share/tallow/*\.json\fR
+.
+.SH "DESCRIPTION"
+tallow(1) uses regular expressions to match journal entries and extract an IP address from them\. JSON files are used to configure the patterns and banning thresholds used by tallow(1)\. This adds the ability to extend the patterns tallow(1) will recognize\. Many JSON files can exist for logical grouping\. The tallow(1) daemon will read all JSON files in the configuration directories at startup\.
+.
+.P
+tallow(1) operates with default pattern definitions in\fB/usr/share/tallow/*\.json\fR\. Users can add more patterns with their own JSON files under \fB/etc/tallow\fR\. The default JSON files can be overridden by creating the same file under \fB/etc/tallow\fR\.
+.
+.SH "FILE FORMAT"
+Pattern configuration files use the JavaScript Object Notation (JSON) format\.
+.
+.P
+The JSON must be two levels deep and all properties are required\. The root object is an array containing objects with a \fBfilter\fR key and an \fBitems\fR key\.
+.
+.IP "\(bu" 4
+\fBfilter\fR is a string that defines a field for filtering the journal file\. This helps make sure patterns are only matched to a subset of journal entries\. See systemd\.journal\-fields(7) for valid journal fields\.
+.
+.IP "\(bu" 4
+\fBitems\fR is an array of objects that contains three elements: \fBban\fR, \fBscore\fR, and \fBpattern\fR\.
+.
+.IP "\(bu" 4
+\fBban\fR is an integer that defines the number of seconds to ban originating IP for\. If this value is > 0, the IP address get banned immediately when a journal entry matches \fBpattern\fR\.
+.
+.IP "\(bu" 4
+\fBscore\fR is a double that defines a value to add to the accumulated "score" of an originating IP address each time a journal entry matches the \fBpattern\fR\. If the combined score is > 1\.0, tallow bans the originating IP for the default time of 1 hour\. The \fBban\fR element value above is not used for bans made due to \fBscore\fR\.
+.
+.IP "\(bu" 4
+\fBpattern\fR is a string that defines a Perl Compatible Regular Expressions (PCRE) to match against the filtered journal entries\. The PCRE should extract exactly one substring: the originating IP address for tallow(1)\. See systemd\.journal\-fields(7) for valid journal fields\.
+.
+.IP "" 0
+
+.
+.IP "" 0
+.
+.SH "EXAMPLES"
+.
+.IP "1." 4
+The JSON below is a snippet from one of the default pattern configuration files for blocking certain failed \fBsshd\fR connections\.
+.
+.IP
+The first pattern will ban an IP address after it fails to login 6 times causing it to reach a total score > 1\.0\.
+.
+.IP
+The second pattern will ban an IP address for 10 seconds every time a login is attempted with an invalid user\. Additionally, it will ban the IP address for 1 hour if it attempts to login with an invalid user 6 times causing it to reach a total score > 1\.0\.
+.
+.IP
+See the \fB/usr/share/tallow/sshd\.json\fR file for more \fBsshd\fR examples\.
+.
+.IP "" 4
+.
+.nf
+
+[
+  {
+    "filter": "SYSLOG_IDENTIFIER=sshd",
+    "items": [
+      {
+        "ban": 0,
+        "score": 0\.2,
+        "pattern": "MESSAGE=Failed \.* for \.* from ([0\-9a\-z:\.]+) port \e\ed+ ssh2"
+      },
+      {
+        "ban": 10,
+        "score": 0\.2,
+        "pattern": "MESSAGE=Invalid user \.* from ([0\-9a\-z:\.]+) port \e\ed+"
+      }
+    ]
+  }
+]
+.
+.fi
+.
+.IP "" 0
+
+.
+.IP "2." 4
+The JSON below defines a pattern for blocking connections based on error logs from \fBnginx\-mainline\fR if placed in a \fB/etc/tallow/nginx\-mainline\.json\fR file\.
+.
+.IP
+The pattern will ban an IP address for 15 seconds every time it attempts to access a script that does not exist\. Additionally, it will ban the IP address for 1 hour if it attempts to access invalid scripts 4 times causing it to reach a total score > 1\.0\.
+.
+.IP "" 4
+.
+.nf
+
+[
+  {
+    "filter": "SYSLOG_IDENTIFIER=nginx\-mainline",
+    "items": [
+      {
+        "ban": 15,
+        "score": 0\.3,
+        "pattern": "\.Primary script unknown\. while reading response header from upstream, client: ([0\-9a\-z:\.]+),"
+      }
+    ]
+  }
+]
+.
+.fi
+.
+.IP "" 0
+
+.
+.IP "" 0
+.
+.SH "SEE ALSO"
+tallow(1), tallow\.conf(5)
+.
+.SH "BUGS"
+\fBtallow\fR is \fBNOT A SECURITY SOLUTION\fR, nor does it protect against random password logins\. An attacker may still be able to logon to your systems if you allow password logins\.
+.
+.SH "AUTHOR"
+Auke Kok \fIauke\-jan\.h\.kok@intel\.com\fR

man/tallow.patterns.5.md

@@ -0,0 +1,134 @@
+## tallow.patterns
+
+Tallow pattern matching configuration files.
+
+
+## SYNOPSIS
+
+tallow(1) uses regular expressions to match journal entries and extract an IP
+address from them. JSON files are used to configure the patterns and banning
+thresholds used by tallow(1).
+
+`/etc/tallow/*.json`
+`/usr/share/tallow/*.json`
+
+
+## DESCRIPTION
+
+tallow(1) uses regular expressions to match journal entries and extract an IP
+address from them. JSON files are used to configure the patterns and banning
+thresholds used by tallow(1). This adds the ability to extend the patterns
+tallow(1) will recognize. Many JSON files can exist for logical grouping. The
+tallow(1) daemon will read all JSON files in the configuration directories at
+startup.
+
+tallow(1) operates with default pattern definitions
+in`/usr/share/tallow/*.json`. Users can add more patterns with their own JSON
+files under `/etc/tallow`. The default JSON files can be overridden by creating
+the same file under `/etc/tallow`.
+
+
+## FILE FORMAT 
+
+Pattern configuration files use the JavaScript Object Notation (JSON) format.
+
+The JSON must be two levels deep and all properties are required. The root
+object is an array containing objects with a `filter` key and an `items` key. 
+
+* `filter` is a string that defines a field for filtering the journal file.
+  This helps make sure patterns are only matched to a subset of journal
+  entries. See systemd.journal-fields(7) for valid journal fields.
+
+* `items` is an array of objects that contains three elements: `ban`, `score`,
+  and `pattern`.
+
+   * `ban` is an integer that defines the number of seconds to ban originating
+     IP for. If this value is > 0, the IP address get banned immediately when a
+     journal entry matches `pattern`.
+
+   * `score` is a double that defines a value to add to the accumulated "score"
+     of an originating IP address each time a journal entry matches
+     the `pattern`. If the combined score is > 1.0, tallow bans the originating
+     IP for the default time of 1 hour. The `ban` element value above is not
+     used for bans made due to `score`. 
+
+   * `pattern` is a string that defines a Perl Compatible Regular Expressions
+     (PCRE) to match against the filtered journal entries. The PCRE should
+     extract exactly one substring: the originating IP address for tallow(1).
+     See systemd.journal-fields(7) for valid journal fields.
+
+
+
+## EXAMPLES
+
+1. The JSON below is a snippet from one of the default pattern configuration
+   files for blocking certain failed `sshd` connections. 
+
+   The first pattern will ban an IP address after it fails to login 6 times
+   causing it to reach a total score > 1.0.
+
+   The second pattern will ban an IP address for 10 seconds every time a login is
+   attempted with an invalid user. Additionally, it will ban the IP address for
+   1 hour if it attempts to login with an invalid user 6 times causing it to
+   reach a total score > 1.0.
+
+   See the `/usr/share/tallow/sshd.json` file for more `sshd` examples.
+
+   ```
+   [
+     {
+       "filter": "SYSLOG_IDENTIFIER=sshd",
+       "items": [
+         {
+           "ban": 0,
+           "score": 0.2,
+           "pattern": "MESSAGE=Failed .* for .* from ([0-9a-z:.]+) port \\d+ ssh2"
+         },
+         {
+           "ban": 10,
+           "score": 0.2,
+           "pattern": "MESSAGE=Invalid user .* from ([0-9a-z:.]+) port \\d+"
+         }
+       ]
+     }
+   ]
+   ```
+
+
+
+2. The JSON below defines a pattern for blocking connections based on error logs
+   from `nginx-mainline` if placed in a `/etc/tallow/nginx-mainline.json` file.
+
+   The pattern will ban an IP address for 15 seconds every time it attempts to
+   access a script that does not exist. Additionally, it will ban the IP
+   address for 1 hour if it attempts to access invalid scripts 4 times causing
+   it to reach a total score > 1.0.
+
+   ```
+   [
+     {
+       "filter": "SYSLOG_IDENTIFIER=nginx-mainline",
+       "items": [
+         {
+           "ban": 15,
+           "score": 0.3,
+           "pattern": ".Primary script unknown. while reading response header from upstream, client: ([0-9a-z:.]+),"
+         }
+       ]
+     }
+   ]
+   ```
+
+## SEE ALSO
+
+tallow(1), tallow.conf(5)
+
+## BUGS
+
+`tallow` is `NOT A SECURITY SOLUTION`, nor does it protect against random
+password logins. An attacker may still be able to logon to your systems if you
+allow password logins.
+
+## AUTHOR
+
+Auke Kok <auke-jan.h.kok@intel.com>

src/tallow.c

@@ -35,6 +35,7 @@
 #define MAX_OFFSETS 30
 
 static char ipt_path[PATH_MAX];
+static char fwcmd_path[PATH_MAX];
 static int expires = 3600;
 static int has_ipv6 = 0;
 static bool nocreate = false;
@@ -71,20 +72,20 @@ static void ext_ignore(char *fmt, ...)
 static void reset_rules(void)
 {
 	/* reset all rules in case the running fw changes */
-	ext_ignore("%s/firewall-cmd --permanent --direct --quiet --remove-rule ipv4 filter INPUT 1 -m set --match-set tallow src -j DROP", ipt_path);
-	ext_ignore("%s/firewall-cmd --quiet --permanent --delete-ipset=tallow", ipt_path);
+	ext_ignore("%s/firewall-cmd --permanent --direct --remove-rule ipv4 filter INPUT 1 -m set --match-set tallow src -j DROP 2> /dev/null", fwcmd_path);
+	ext_ignore("%s/firewall-cmd --permanent --delete-ipset=tallow 2> /dev/null", fwcmd_path);
 
 	/* delete iptables ref to set before the ipset! */
 	ext_ignore("%s/iptables -t filter -D INPUT -m set --match-set tallow src -j DROP 2> /dev/null", ipt_path);
 	ext_ignore("%s/ipset destroy tallow 2> /dev/null", ipt_path);
 
 	if (has_ipv6) {
-		ext_ignore("%s/firewall-cmd --permanent --direct --quiet --remove-rule ipv6 filter INPUT 1 -m set --match-set tallow6 src -j DROP", ipt_path);
-		ext_ignore("%s/firewall-cmd --permanent --delete-ipset=tallow6 --quiet", ipt_path);	
-		
+		ext_ignore("%s/firewall-cmd --permanent --direct --remove-rule ipv6 filter INPUT 1 -m set --match-set tallow6 src -j DROP 2> /dev/null", fwcmd_path);
+		ext_ignore("%s/firewall-cmd --permanent --delete-ipset=tallow6 2> /dev/null", fwcmd_path);
+
 		/* delete iptables ref to set before the ipset! */
 		ext_ignore("%s/ip6tables -t filter -D INPUT -m set --match-set tallow6 src -j DROP 2> /dev/null", ipt_path);
-		ext_ignore("%s/ipset destroy tallow6 2> /dev/null", ipt_path);		
+		ext_ignore("%s/ipset destroy tallow6 2> /dev/null", ipt_path);
 	}
 }
 
@@ -100,48 +101,44 @@ static void setup(void)
 
 	/* firewalld */
 	char *fwd_path;
-	if (asprintf(&fwd_path, "%s/firewall-cmd", ipt_path) < 0)
-	{
-		fprintf(stderr, "Unable to allocate buffer for path to firewall-cmd.\n");
+	if (asprintf(&fwd_path, "%s/firewall-cmd", fwcmd_path) < 0) {
 		exit(EXIT_FAILURE);
 	}
 
-	if ((access(fwd_path, X_OK) == 0) && ext("%s/firewall-cmd --state --quiet", ipt_path) == 0) {
+	if ((access(fwd_path, X_OK) == 0) && ext("%s/firewall-cmd --state --quiet", fwcmd_path) == 0) {
 		fprintf(stdout, "firewalld is running and will be used by tallow.\n");
 
 		reset_rules();
 
 		/* create ipv4 rule and ipset */
-		if (ext("%s/firewall-cmd --permanent --quiet --new-ipset=tallow --type=hash:ip --family=inet --option=timeout=%d", ipt_path, expires)) {
+		if (ext("%s/firewall-cmd --permanent --quiet --new-ipset=tallow --type=hash:ip --family=inet --option=timeout=%d", fwcmd_path, expires)) {
 			fprintf(stderr, "Unable to create ipv4 ipset with firewall-cmd.\n");
 			exit(EXIT_FAILURE);
 		}
-		if (ext("%s/firewall-cmd --permanent --direct --quiet --add-rule ipv4 filter INPUT 1 -m set --match-set tallow src -j DROP", ipt_path)) {
+		if (ext("%s/firewall-cmd --permanent --direct --quiet --add-rule ipv4 filter INPUT 1 -m set --match-set tallow src -j DROP", fwcmd_path)) {
 			fprintf(stderr, "Unable to create ipv4 firewalld rule.\n");
 			exit(EXIT_FAILURE);
 		}
 
 		/* create ipv6 rule and ipset */
 		if (has_ipv6) {
-			if (ext("%s/firewall-cmd --permanent --quiet --new-ipset=tallow6 --type=hash:ip --family=inet6 --option=timeout=%d", ipt_path, expires)) {
+			if (ext("%s/firewall-cmd --permanent --quiet --new-ipset=tallow6 --type=hash:ip --family=inet6 --option=timeout=%d", fwcmd_path, expires)) {
 				fprintf(stderr, "Unable to create ipv6 ipset with firewall-cmd.\n");
 				exit(EXIT_FAILURE);
 			}
-			if (ext("%s/firewall-cmd --permanent --direct --quiet --add-rule ipv6 filter INPUT 1 -m set --match-set tallow6 src -j DROP ", ipt_path)) {
+			if (ext("%s/firewall-cmd --permanent --direct --quiet --add-rule ipv6 filter INPUT 1 -m set --match-set tallow6 src -j DROP ", fwcmd_path)) {
 				fprintf(stderr, "Unable to create ipv6 firewalld rule.\n");
 				exit(EXIT_FAILURE);
 			}
 		}
 
 		/* reload firewalld for ipsets to load */
-		if (ext("%s/firewall-cmd --reload --quiet", ipt_path, expires)) {
+		if (ext("%s/firewall-cmd --reload --quiet", fwcmd_path, expires)) {
 			fprintf(stderr, "Unable to reload firewalld rules.\n");
 			exit(EXIT_FAILURE);
 		}
-	}
-	/* iptables */
-	else {
-
+	} else {
+		/* iptables */
 		reset_rules();
 
 		/* create ipv4 rule and ipset */
@@ -291,11 +288,12 @@ int main(void)
 	int r;
 	FILE *f;
 	int timeout = 60;
-	long long int last_timestamp = 0;
+	long long unsigned int last_timestamp = 0;
 
 	json_load_patterns();
 
 	strcpy(ipt_path, "/usr/sbin");
+	strcpy(fwcmd_path, "/usr/sbin");
 
 #ifdef DEBUG
 	fprintf(stderr, "Debug output enabled. Send SIGUSR1 to dump internal state table\n");
@@ -335,6 +333,8 @@ int main(void)
 			// todo: filter leading/trailing whitespace
 			if (!strcmp(key, "ipt_path"))
 				strncpy(ipt_path, val, PATH_MAX - 1);
+			if (!strcmp(key, "fwcmd_path"))
+				strncpy(fwcmd_path, val, PATH_MAX - 1);
 			if (!strcmp(key, "expires"))
 				expires = atoi(val);
 			if (!strcmp(key, "whitelist"))
@@ -400,11 +400,13 @@ int main(void)
 			 * this happens when the journal rotates - we get replayed events
 			 */
 			if (sd_journal_get_data(j, "_SOURCE_REALTIME_TIMESTAMP", &dt, &dl) == 0) {
-				long long int lt = atoi(dt + strlen("_SOURCE_REALTIME_TIMESTAMP="));
+				long long unsigned int lt = atoll(dt + strlen("_SOURCE_REALTIME_TIMESTAMP="));
 				if (lt > last_timestamp)
 					last_timestamp = lt;
-				else if (lt < last_timestamp)
+				else if (lt < last_timestamp) {
+					dbg("Discarding old entry: %llu - %llu\n", lt, last_timestamp);
 					continue;
+				}
 			}
 
 			if (sd_journal_get_data(j, "MESSAGE", &d, &l) < 0) {

tallow.conf

@@ -1,6 +1,7 @@
 
 # tallow.conf - see `man tallow.conf` for more information
 
+#fwcmd_path=/usr/sbin
 #ipt_path=/usr/sbin
 #expires=3600
 #whitelist=127.0.0.1