Merge branch 'Talkabout-redis-expire-records'

cachedb/cachedb.c

@@ -160,7 +160,7 @@ testframe_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 
 static void
 testframe_store(struct module_env* env, struct cachedb_env* cachedb_env,
-	char* key, uint8_t* data, size_t data_len)
+	char* key, uint8_t* data, size_t data_len, time_t ATTR_UNUSED(ttl))
 {
 	struct testframe_moddata* d = (struct testframe_moddata*)
 		cachedb_env->backend_data;
@@ -606,7 +606,8 @@ cachedb_extcache_store(struct module_qstate* qstate, struct cachedb_env* ie)
 	/* call backend */
 	(*ie->backend->store)(qstate->env, ie, key,
 		sldns_buffer_begin(qstate->env->scratch_buffer),
-		sldns_buffer_limit(qstate->env->scratch_buffer));
+		sldns_buffer_limit(qstate->env->scratch_buffer),
+		qstate->return_msg->rep->ttl);
 }
 
 /**

cachedb/cachedb.h

@@ -84,7 +84,7 @@ struct cachedb_backend {
 	
 	/** Store (env, cachedb_env, key, data, data_len) */
 	void (*store)(struct module_env*, struct cachedb_env*, char*,
-		uint8_t*, size_t);
+		uint8_t*, size_t, time_t);
 };
 
 #define CACHEDB_HASHSIZE 256 /* bit hash */

cachedb/redis.c

@@ -59,6 +59,9 @@ struct redis_moddata {
 	struct timeval timeout;	 /* timeout for connection setup and commands */
 };
 
+static redisReply* redis_command(struct module_env*, struct cachedb_env*,
+	const char*, const uint8_t*, size_t);
+
 static redisContext*
 redis_connect(const struct redis_moddata* moddata)
 {
@@ -114,6 +117,33 @@ redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 	for(i = 0; i < moddata->numctxs; i++)
 		moddata->ctxs[i] = redis_connect(moddata);
 	cachedb_env->backend_data = moddata;
+	if(env->cfg->redis_expire_records) {
+		redisReply* rep = NULL;
+		int redis_reply_type = 0;
+		/** check if setex command is supported */
+		rep = redis_command(env, cachedb_env,
+			"SETEX __UNBOUND_REDIS_CHECK__ 1 none", NULL, 0);
+		if(!rep) {
+			/** init failed, no response from redis server*/
+			log_err("redis_init: failed to init redis, the "
+				"redis-expire-records option requires the SETEX command "
+				"(redis >= 2.0.0)");
+			return 0;
+		}
+		redis_reply_type = rep->type;
+		freeReplyObject(rep);
+		switch(redis_reply_type) {
+		case REDIS_REPLY_STATUS:
+			break;
+		default:
+			/** init failed, setex command not supported */
+			log_err("redis_init: failed to init redis, the "
+				"redis-expire-records option requires the SETEX command "
+				"(redis >= 2.0.0)");
+			return 0;
+		}
+	}
+
 	return 1;
 }
 
@@ -219,7 +249,7 @@ redis_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 	rep = redis_command(env, cachedb_env, cmdbuf, NULL, 0);
 	if(!rep)
 		return 0;
-	switch (rep->type) {
+	switch(rep->type) {
 	case REDIS_REPLY_NIL:
 		verbose(VERB_ALGO, "redis_lookup: no data cached");
 		break;
@@ -249,16 +279,33 @@ redis_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 
 static void
 redis_store(struct module_env* env, struct cachedb_env* cachedb_env,
-	char* key, uint8_t* data, size_t data_len)
+	char* key, uint8_t* data, size_t data_len, time_t ttl)
 {
 	redisReply* rep;
-	char cmdbuf[4+(CACHEDB_HASHSIZE/8)*2+3+1]; /* "SET " + key + " %b" */
 	int n;
+	int set_ttl = (env->cfg->redis_expire_records &&
+		(!env->cfg->serve_expired || env->cfg->serve_expired_ttl > 0));
+	/* Supported commands:
+	 * - "SET " + key + " %b"
+	 * - "SETEX " + key + " " + ttl + " %b"
+	 */
+	char cmdbuf[6+(CACHEDB_HASHSIZE/8)*2+11+3+1];
+
+	if (!set_ttl) {
+		verbose(VERB_ALGO, "redis_store %s (%d bytes)", key, (int)data_len);
+		/* build command to set to a binary safe string */
+		n = snprintf(cmdbuf, sizeof(cmdbuf), "SET %s %%b", key);
+	} else {
+		/* add expired ttl time to redis ttl to avoid premature eviction of key */
+		ttl += env->cfg->serve_expired_ttl;
+		verbose(VERB_ALGO, "redis_store %s (%d bytes) with ttl %u",
+			key, (int)data_len, (uint32_t)ttl);
+		/* build command to set to a binary safe string */
+		n = snprintf(cmdbuf, sizeof(cmdbuf), "SETEX %s %u %%b", key,
+			(uint32_t)ttl);
+	}
 
-	verbose(VERB_ALGO, "redis_store %s (%d bytes)", key, (int)data_len);
 
-	/* build command to set to a binary safe string */
-	n = snprintf(cmdbuf, sizeof(cmdbuf), "SET %s %%b", key);
 	if(n < 0 || n >= (int)sizeof(cmdbuf)) {
 		log_err("redis_store: unexpected failure to build command");
 		return;

doc/Changelog

@@ -1,3 +1,6 @@
+1 April 2020: George
+	- Merge PR #206: Redis TTL, by Talkabout.
+
 30 March 2020: Wouter
 	- Merge PR #207: Clarify if-automatic listens on 0.0.0.0 and ::
 	- Merge PR #208: Fix uncached CLIENT_RESPONSE'es on stateful

doc/unbound.conf.5.in

@@ -2130,6 +2130,14 @@ If this timeout expires Unbound closes the connection, treats it as
 if the Redis server does not have the requested data, and will try to
 re-establish a new connection later.
 This option defaults to 100 milliseconds.
+.TP
+.B redis-expire-records: \fI<yes or no>
+If Redis record expiration is enabled.  If yes, unbound sets timeout for Redis
+records so that Redis can evict keys that have expired automatically.  If
+unbound is configured with \fBserve-expired\fR and \fBserve-expired-ttl\fR is 0,
+this option is internally reverted to "no".  Redis SETEX support is required
+for this option (Redis >= 2.0.0).
+This option defaults to no.
 .SS DNSTAP Logging Options
 DNSTAP support, when compiled in, is enabled in the \fBdnstap:\fR section.
 This starts an extra thread (when compiled with threading) that writes

util/config_file.c

@@ -337,6 +337,7 @@ config_create(void)
 	if(!(cfg->redis_server_host = strdup("127.0.0.1"))) goto error_exit;
 	cfg->redis_timeout = 100;
 	cfg->redis_server_port = 6379;
+	cfg->redis_expire_records = 0;
 #endif  /* USE_REDIS */
 #endif  /* USE_CACHEDB */
 #ifdef USE_IPSET
@@ -1135,6 +1136,7 @@ config_get_option(struct config_file* cfg, const char* opt,
 	else O_STR(opt, "redis-server-host", redis_server_host)
 	else O_DEC(opt, "redis-server-port", redis_server_port)
 	else O_DEC(opt, "redis-timeout", redis_timeout)
+	else O_YNO(opt, "redis-expire-records", redis_expire_records)
 #endif  /* USE_REDIS */
 #endif  /* USE_CACHEDB */
 #ifdef USE_IPSET

util/config_file.h

@@ -598,6 +598,8 @@ struct config_file {
 	int redis_server_port;
 	/** timeout (in ms) for communication with the redis server */
 	int redis_timeout;
+	/** set timeout on redis records based on DNS response ttl */
+	int redis_expire_records;
 #endif
 #endif
 

util/configlexer.lex

@@ -500,6 +500,7 @@ secret-seed{COLON}		{ YDVAR(1, VAR_CACHEDB_SECRETSEED) }
 redis-server-host{COLON}	{ YDVAR(1, VAR_CACHEDB_REDISHOST) }
 redis-server-port{COLON}	{ YDVAR(1, VAR_CACHEDB_REDISPORT) }
 redis-timeout{COLON}		{ YDVAR(1, VAR_CACHEDB_REDISTIMEOUT) }
+redis-expire-records{COLON}	{ YDVAR(1, VAR_CACHEDB_REDISEXPIRERECORDS) }
 ipset{COLON}			{ YDVAR(0, VAR_IPSET) }
 name-v4{COLON}			{ YDVAR(1, VAR_IPSET_NAME_V4) }
 name-v6{COLON}			{ YDVAR(1, VAR_IPSET_NAME_V6) }

util/configparser.h

@@ -297,41 +297,42 @@ extern int yydebug;
     VAR_CACHEDB_REDISHOST = 503,
     VAR_CACHEDB_REDISPORT = 504,
     VAR_CACHEDB_REDISTIMEOUT = 505,
-    VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM = 506,
-    VAR_FOR_UPSTREAM = 507,
-    VAR_AUTH_ZONE = 508,
-    VAR_ZONEFILE = 509,
-    VAR_MASTER = 510,
-    VAR_URL = 511,
-    VAR_FOR_DOWNSTREAM = 512,
-    VAR_FALLBACK_ENABLED = 513,
-    VAR_TLS_ADDITIONAL_PORT = 514,
-    VAR_LOW_RTT = 515,
-    VAR_LOW_RTT_PERMIL = 516,
-    VAR_FAST_SERVER_PERMIL = 517,
-    VAR_FAST_SERVER_NUM = 518,
-    VAR_ALLOW_NOTIFY = 519,
-    VAR_TLS_WIN_CERT = 520,
-    VAR_TCP_CONNECTION_LIMIT = 521,
-    VAR_FORWARD_NO_CACHE = 522,
-    VAR_STUB_NO_CACHE = 523,
-    VAR_LOG_SERVFAIL = 524,
-    VAR_DENY_ANY = 525,
-    VAR_UNKNOWN_SERVER_TIME_LIMIT = 526,
-    VAR_LOG_TAG_QUERYREPLY = 527,
-    VAR_STREAM_WAIT_SIZE = 528,
-    VAR_TLS_CIPHERS = 529,
-    VAR_TLS_CIPHERSUITES = 530,
-    VAR_IPSET = 531,
-    VAR_IPSET_NAME_V4 = 532,
-    VAR_IPSET_NAME_V6 = 533,
-    VAR_TLS_SESSION_TICKET_KEYS = 534,
-    VAR_RPZ = 535,
-    VAR_TAGS = 536,
-    VAR_RPZ_ACTION_OVERRIDE = 537,
-    VAR_RPZ_CNAME_OVERRIDE = 538,
-    VAR_RPZ_LOG = 539,
-    VAR_RPZ_LOG_NAME = 540
+    VAR_CACHEDB_REDISEXPIRERECORDS = 506,
+    VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM = 507,
+    VAR_FOR_UPSTREAM = 508,
+    VAR_AUTH_ZONE = 509,
+    VAR_ZONEFILE = 510,
+    VAR_MASTER = 511,
+    VAR_URL = 512,
+    VAR_FOR_DOWNSTREAM = 513,
+    VAR_FALLBACK_ENABLED = 514,
+    VAR_TLS_ADDITIONAL_PORT = 515,
+    VAR_LOW_RTT = 516,
+    VAR_LOW_RTT_PERMIL = 517,
+    VAR_FAST_SERVER_PERMIL = 518,
+    VAR_FAST_SERVER_NUM = 519,
+    VAR_ALLOW_NOTIFY = 520,
+    VAR_TLS_WIN_CERT = 521,
+    VAR_TCP_CONNECTION_LIMIT = 522,
+    VAR_FORWARD_NO_CACHE = 523,
+    VAR_STUB_NO_CACHE = 524,
+    VAR_LOG_SERVFAIL = 525,
+    VAR_DENY_ANY = 526,
+    VAR_UNKNOWN_SERVER_TIME_LIMIT = 527,
+    VAR_LOG_TAG_QUERYREPLY = 528,
+    VAR_STREAM_WAIT_SIZE = 529,
+    VAR_TLS_CIPHERS = 530,
+    VAR_TLS_CIPHERSUITES = 531,
+    VAR_IPSET = 532,
+    VAR_IPSET_NAME_V4 = 533,
+    VAR_IPSET_NAME_V6 = 534,
+    VAR_TLS_SESSION_TICKET_KEYS = 535,
+    VAR_RPZ = 536,
+    VAR_TAGS = 537,
+    VAR_RPZ_ACTION_OVERRIDE = 538,
+    VAR_RPZ_CNAME_OVERRIDE = 539,
+    VAR_RPZ_LOG = 540,
+    VAR_RPZ_LOG_NAME = 541
   };
 #endif
 /* Tokens.  */
@@ -583,41 +584,42 @@ extern int yydebug;
 #define VAR_CACHEDB_REDISHOST 503
 #define VAR_CACHEDB_REDISPORT 504
 #define VAR_CACHEDB_REDISTIMEOUT 505
-#define VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM 506
-#define VAR_FOR_UPSTREAM 507
-#define VAR_AUTH_ZONE 508
-#define VAR_ZONEFILE 509
-#define VAR_MASTER 510
-#define VAR_URL 511
-#define VAR_FOR_DOWNSTREAM 512
-#define VAR_FALLBACK_ENABLED 513
-#define VAR_TLS_ADDITIONAL_PORT 514
-#define VAR_LOW_RTT 515
-#define VAR_LOW_RTT_PERMIL 516
-#define VAR_FAST_SERVER_PERMIL 517
-#define VAR_FAST_SERVER_NUM 518
-#define VAR_ALLOW_NOTIFY 519
-#define VAR_TLS_WIN_CERT 520
-#define VAR_TCP_CONNECTION_LIMIT 521
-#define VAR_FORWARD_NO_CACHE 522
-#define VAR_STUB_NO_CACHE 523
-#define VAR_LOG_SERVFAIL 524
-#define VAR_DENY_ANY 525
-#define VAR_UNKNOWN_SERVER_TIME_LIMIT 526
-#define VAR_LOG_TAG_QUERYREPLY 527
-#define VAR_STREAM_WAIT_SIZE 528
-#define VAR_TLS_CIPHERS 529
-#define VAR_TLS_CIPHERSUITES 530
-#define VAR_IPSET 531
-#define VAR_IPSET_NAME_V4 532
-#define VAR_IPSET_NAME_V6 533
-#define VAR_TLS_SESSION_TICKET_KEYS 534
-#define VAR_RPZ 535
-#define VAR_TAGS 536
-#define VAR_RPZ_ACTION_OVERRIDE 537
-#define VAR_RPZ_CNAME_OVERRIDE 538
-#define VAR_RPZ_LOG 539
-#define VAR_RPZ_LOG_NAME 540
+#define VAR_CACHEDB_REDISEXPIRERECORDS 506
+#define VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM 507
+#define VAR_FOR_UPSTREAM 508
+#define VAR_AUTH_ZONE 509
+#define VAR_ZONEFILE 510
+#define VAR_MASTER 511
+#define VAR_URL 512
+#define VAR_FOR_DOWNSTREAM 513
+#define VAR_FALLBACK_ENABLED 514
+#define VAR_TLS_ADDITIONAL_PORT 515
+#define VAR_LOW_RTT 516
+#define VAR_LOW_RTT_PERMIL 517
+#define VAR_FAST_SERVER_PERMIL 518
+#define VAR_FAST_SERVER_NUM 519
+#define VAR_ALLOW_NOTIFY 520
+#define VAR_TLS_WIN_CERT 521
+#define VAR_TCP_CONNECTION_LIMIT 522
+#define VAR_FORWARD_NO_CACHE 523
+#define VAR_STUB_NO_CACHE 524
+#define VAR_LOG_SERVFAIL 525
+#define VAR_DENY_ANY 526
+#define VAR_UNKNOWN_SERVER_TIME_LIMIT 527
+#define VAR_LOG_TAG_QUERYREPLY 528
+#define VAR_STREAM_WAIT_SIZE 529
+#define VAR_TLS_CIPHERS 530
+#define VAR_TLS_CIPHERSUITES 531
+#define VAR_IPSET 532
+#define VAR_IPSET_NAME_V4 533
+#define VAR_IPSET_NAME_V6 534
+#define VAR_TLS_SESSION_TICKET_KEYS 535
+#define VAR_RPZ 536
+#define VAR_TAGS 537
+#define VAR_RPZ_ACTION_OVERRIDE 538
+#define VAR_RPZ_CNAME_OVERRIDE 539
+#define VAR_RPZ_LOG 540
+#define VAR_RPZ_LOG_NAME 541
 
 /* Value type.  */
 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
@@ -627,7 +629,7 @@ union YYSTYPE
 
 	char*	str;
 
-#line 631 "util/configparser.h"
+#line 633 "util/configparser.h"
 
 };
 typedef union YYSTYPE YYSTYPE;

util/configparser.y

@@ -162,6 +162,7 @@ extern struct config_parser_state* cfg_parser;
 %token VAR_IPSECMOD_MAX_TTL VAR_IPSECMOD_WHITELIST VAR_IPSECMOD_STRICT
 %token VAR_CACHEDB VAR_CACHEDB_BACKEND VAR_CACHEDB_SECRETSEED
 %token VAR_CACHEDB_REDISHOST VAR_CACHEDB_REDISPORT VAR_CACHEDB_REDISTIMEOUT
+%token VAR_CACHEDB_REDISEXPIRERECORDS
 %token VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM VAR_FOR_UPSTREAM
 %token VAR_AUTH_ZONE VAR_ZONEFILE VAR_MASTER VAR_URL VAR_FOR_DOWNSTREAM
 %token VAR_FALLBACK_ENABLED VAR_TLS_ADDITIONAL_PORT VAR_LOW_RTT VAR_LOW_RTT_PERMIL
@@ -3077,7 +3078,8 @@ cachedbstart: VAR_CACHEDB
 contents_cachedb: contents_cachedb content_cachedb
 	| ;
 content_cachedb: cachedb_backend_name | cachedb_secret_seed |
-	redis_server_host | redis_server_port | redis_timeout
+	redis_server_host | redis_server_port | redis_timeout |
+	redis_expire_records
 	;
 cachedb_backend_name: VAR_CACHEDB_BACKEND STRING_ARG
 	{
@@ -3143,6 +3145,19 @@ redis_timeout: VAR_CACHEDB_REDISTIMEOUT STRING_ARG
 		free($2);
 	}
 	;
+redis_expire_records: VAR_CACHEDB_REDISEXPIRERECORDS STRING_ARG
+	{
+	#if defined(USE_CACHEDB) && defined(USE_REDIS)
+		OUTYY(("P(redis_expire_records:%s)\n", $2));
+		if(strcmp($2, "yes") != 0 && strcmp($2, "no") != 0)
+			yyerror("expected yes or no.");
+		else cfg_parser->cfg->redis_expire_records = (strcmp($2, "yes")==0);
+	#else
+		OUTYY(("P(Compiled without cachedb or redis, ignoring)\n"));
+	#endif
+		free($2);
+	}
+	;
 server_tcp_connection_limit: VAR_TCP_CONNECTION_LIMIT STRING_ARG STRING_ARG
 	{
 		OUTYY(("P(server_tcp_connection_limit:%s %s)\n", $2, $3));