Implement rolling 24h as standard behavior of FTL. Can be set to stat…

…ic views of up to 24h (analyzing pihole.log) and up to 48h (analyzing pihole.log.1 in addition)

FTL.h

@@ -45,6 +45,18 @@
 
 #define SOCKETBUFFERLEN 1024
 
+// Maximum time from now until we will parse logs that are in the past [seconds]
+// Default: 86400 (24 hours)
+#define MAXLOGAGE 86400
+
+// How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]
+// Default: 3600 (once per hour)
+#define GCinterval 3600
+
+// Delay applied to the garbage collecting [seconds]
+// Default -60 (one minute before a full hour)
+#define GCdelay (-60)
+
 // Static structs
 typedef struct {
 	const char* conf;
@@ -102,6 +114,8 @@ typedef struct {
 	// 0 = unknown, 1 = gravity.list (blocked), 2 = reply from upstream, 3 = cache, 4 = wildcard blocked
 	int domainID;
 	int clientID;
+	int forwardID;
+	bool valid;
 } queriesDataStruct;
 
 typedef struct {
@@ -166,6 +180,7 @@ bool initialscan;
 bool debug;
 bool debugthreads;
 bool debugclients;
+bool debugGC;
 bool threadlock;
 
 char ** wildcarddomains;

Makefile

@@ -9,7 +9,7 @@
 # Please see LICENSE file for your rights under this license.
 
 DEPS = FTL.h routines.h version.h
-OBJ = main.o structs.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o config.o
+OBJ = main.o structs.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o gc.o config.o
 
 # Get git commit version and date
 GIT_BRANCH := $(shell git branch | sed -n 's/^\* //p')

args.c

@@ -14,6 +14,7 @@
 bool debug = false;
 bool debugthreads = false;
 bool debugclients = false;
+bool debugGC = false;
 void parse_args(int argc, char* argv[])
 {
 	int i;
@@ -33,6 +34,12 @@ void parse_args(int argc, char* argv[])
 			debugclients = true;
 		}
 
+		if(strcmp(argv[i], "debugGC") == 0)
+		{
+			debug = true;
+			debugGC = true;
+		}
+
 		if(strcmp(argv[i], "test") == 0)
 			killed = 1;
 

gc.c

@@ -0,0 +1,94 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Garbage collection routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+
+void *GC_thread(void *val)
+{
+	// Set thread name
+	prctl(PR_SET_NAME,"GC",0,0,0);
+
+	// Lock FTL's data structure, since it is likely that it will be changed here
+	// Requests should not be processed/answered when data is about to change
+	enable_lock("GC_thread");
+
+	// Get minimum time stamp to keep
+	int mintime = time(NULL) - GCdelay - MAXLOGAGE;
+	if(debugGC)
+	{
+		time_t timestamp = mintime;
+		logg_str("GC all queries older than: ", strtok(ctime(&timestamp),"\n"));
+	}
+
+	// Process all queries
+	int i;
+	for(i=0; i < counters.queries; i++)
+	{
+		if(queries[i].timestamp < mintime && queries[i].valid)
+		{
+			// Adjust total counters and total over time data
+			// We cannot edit counters.queries directly as it is used
+			// as max ID for the queries[] struct
+			counters.invalidqueries++;
+			overTime[queries[i].timeidx].total--;
+
+			// Adjust client and domain counters
+			clients[queries[i].clientID].count--;
+			domains[queries[i].domainID].count--;
+			forwarded[queries[i].forwardID].count--;
+
+			// Change other counters according to status of this query
+			switch(queries[i].status)
+			{
+				case 0: counters.unknown--; break;
+				case 1: counters.blocked--; overTime[queries[i].timeidx].blocked--; domains[queries[i].domainID].blockedcount--; break;
+				case 2: break;
+				case 3: counters.cached--; break;
+				case 4: counters.wildcardblocked--; overTime[queries[i].timeidx].blocked--; break;
+				default: /* That cannot happen */ break;
+			}
+
+			switch(queries[i].type)
+			{
+				case 1: counters.IPv4--; break;
+				case 2: counters.IPv6--; break;
+				default: logg("ERROR in GC"); break;
+			}
+
+			// Remove forwarded data from overTime and total forwarded count
+			int j;
+			for(j = 0; j < overTime[queries[i].timeidx].forwardnum; j++)
+			{
+				forwarded[j].count -= overTime[queries[i].timeidx].forwarddata[j];
+				overTime[queries[i].timeidx].forwarddata[j] = 0;
+			}
+
+			// Mark this query as garbage collected
+			queries[i].valid = false;
+
+			if(debugGC)
+			{
+				time_t timestamp = queries[i].timestamp;
+				logg_str("GC query with time: ", strtok(ctime(&timestamp),"\n"));
+			}
+		}
+	}
+
+	if(debugGC)
+	{
+		logg_int("GC queries: ", counters.invalidqueries);
+	}
+
+	// Release thread lock
+	disable_lock("GC_thread");
+
+
+	return NULL;
+}

main.c

@@ -64,9 +64,36 @@ int main (int argc, char* argv[]) {
 		killed = 1;
 	}
 
+	if(config.rolling_24h)
+	{
+		pthread_t GCthread;
+		if(pthread_create( &GCthread, &attr, GC_thread, NULL ) != 0)
+		{
+			logg("Unable to start initial GC thread. Exiting...");
+			killed = 1;
+		}
+	}
+
 	while(!killed)
 	{
 		sleepms(100);
+
+		// Garbadge collect in regular interval, but don't do it if the threadlock is set
+		if(config.rolling_24h && ((time(NULL) - GCdelay)%GCinterval) == 0 && !threadlock)
+		{
+			if(debug)
+				logg_int("Running GC on data structure due to set interval of [s]: ", GCinterval);
+
+			pthread_t GCthread;
+			if(pthread_create( &GCthread, &attr, GC_thread, NULL ) != 0)
+			{
+				logg("Unable to open GC thread. Exiting...");
+				killed = 1;
+			}
+
+			while(((time(NULL) - GCdelay)%GCinterval) == 0)
+				sleepms(100);
+		}
 	}
 
 

parser.c

@@ -140,6 +140,33 @@ void process_pihole_log(int file)
 			// Ensure we have enough space in the queries struct
 			memory_check(QUERIES);
 
+			// Get timestamp
+			int querytimestamp, overTimetimestamp;
+			extracttimestamp(readbuffer, &querytimestamp, &overTimetimestamp);
+
+			int timeidx;
+			bool found = false;
+			for(i=0; i < counters.overTime; i++)
+			{
+				if(overTime[i].timestamp == overTimetimestamp)
+				{
+					found = true;
+					timeidx = i;
+					break;
+				}
+			}
+			if(!found)
+			{
+				memory_check(OVERTIME);
+				timeidx = counters.overTime;
+				overTime[timeidx].timestamp = overTimetimestamp;
+				overTime[timeidx].total = 0;
+				overTime[timeidx].blocked = 0;
+				overTime[timeidx].forwardnum = 0;
+				overTime[timeidx].forwarddata = NULL;
+				counters.overTime++;
+			}
+
 			// Get domain
 			// domainstart = pointer to | in "query[AAAA] |host.name from ww.xx.yy.zz\n"
 			const char *domainstart = strstr(readbuffer, "] ");
@@ -179,33 +206,6 @@ void process_pihole_log(int file)
 			char *client = calloc(clientlen+1,sizeof(char));
 			strncpy(client,domainend+6,clientlen);
 
-			// Get timestamp
-			int querytimestamp, overTimetimestamp;
-			extracttimestamp(readbuffer, &querytimestamp, &overTimetimestamp);
-
-			int timeidx;
-			bool found = false;
-			for(i=0; i < counters.overTime; i++)
-			{
-				if(overTime[i].timestamp == overTimetimestamp)
-				{
-					found = true;
-					timeidx = i;
-					break;
-				}
-			}
-			if(!found)
-			{
-				memory_check(OVERTIME);
-				timeidx = counters.overTime;
-				overTime[timeidx].timestamp = overTimetimestamp;
-				overTime[timeidx].total = 0;
-				overTime[timeidx].blocked = 0;
-				overTime[timeidx].forwardnum = 0;
-				overTime[timeidx].forwarddata = NULL;
-				counters.overTime++;
-			}
-
 			// Get type
 			unsigned char type = 0;
 			if(strstr(readbuffer,"query[A]") != NULL)
@@ -369,6 +369,7 @@ void process_pihole_log(int file)
 			queries[counters.queries].domainID = domainID;
 			queries[counters.queries].clientID = clientID;
 			queries[counters.queries].timeidx = timeidx;
+			queries[counters.queries].valid = true;
 
 			// Increase DNS queries counter
 			counters.queries++;
@@ -394,7 +395,6 @@ void process_pihole_log(int file)
 		}
 		else if(strstr(readbuffer,": forwarded") != NULL)
 		{
-
 			// Get forward destination
 			// forwardstart = pointer to | in "forwarded domain.name| to www.xxx.yyy.zzz\n"
 			const char *forwardstart = strstr(readbuffer, " to ");
@@ -458,7 +458,6 @@ void process_pihole_log(int file)
 					logg_str("Added new forward server: ", forwarded[forwardID].ip);
 			}
 
-			// Determine time index for this forward request
 			// Get timestamp
 			int querytimestamp, overTimetimestamp, timeidx;
 			extracttimestamp(readbuffer, &querytimestamp, &overTimetimestamp);

request.c

@@ -354,6 +354,9 @@ void process_request(char *client_message, int *sock)
 		int i;
 		for(i=ibeg; i < counters.queries; i++)
 		{
+			// Check if this query has been removed due to garbage collection
+			if(!queries[i].valid) continue;
+
 			char type[5];
 			if(queries[i].type == 1)
 			{
@@ -424,6 +427,7 @@ void process_request(char *client_message, int *sock)
 			// User wants a different number of requests
 			if(num >= counters.queries)
 				num = 0;
+
 			if(debugclients)
 				logg_int("Showing several blocked domains ",num);
 		}
@@ -432,12 +436,16 @@ void process_request(char *client_message, int *sock)
 		int found = 0;
 		for(i = counters.queries - 1; i > 0 ; i--)
 		{
+			// Check if this query has been removed due to garbage collection
+			if(!queries[i].valid) continue;
+
 			if(queries[i].status == 1 || queries[i].status == 4)
 			{
 				found++;
 				sprintf(server_message,"%s\n",domains[queries[i].domainID].domain);
 				swrite(server_message, *sock);
 			}
+
 			if(found >= num)
 			{
 				break;

routines.h

@@ -72,3 +72,5 @@ void enable_lock(const char *message);
 void disable_lock(const char *message);
 
 void read_FTLconf(void);
+
+void *GC_thread(void *val);