Merge branch 'master' into features/padding

cachedb/cachedb.c

@@ -160,7 +160,7 @@ testframe_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 
 static void
 testframe_store(struct module_env* env, struct cachedb_env* cachedb_env,
-	char* key, uint8_t* data, size_t data_len)
+	char* key, uint8_t* data, size_t data_len, time_t ATTR_UNUSED(ttl))
 {
 	struct testframe_moddata* d = (struct testframe_moddata*)
 		cachedb_env->backend_data;
@@ -606,7 +606,8 @@ cachedb_extcache_store(struct module_qstate* qstate, struct cachedb_env* ie)
 	/* call backend */
 	(*ie->backend->store)(qstate->env, ie, key,
 		sldns_buffer_begin(qstate->env->scratch_buffer),
-		sldns_buffer_limit(qstate->env->scratch_buffer));
+		sldns_buffer_limit(qstate->env->scratch_buffer),
+		qstate->return_msg->rep->ttl);
 }
 
 /**

cachedb/cachedb.h

@@ -84,7 +84,7 @@ struct cachedb_backend {
 	
 	/** Store (env, cachedb_env, key, data, data_len) */
 	void (*store)(struct module_env*, struct cachedb_env*, char*,
-		uint8_t*, size_t);
+		uint8_t*, size_t, time_t);
 };
 
 #define CACHEDB_HASHSIZE 256 /* bit hash */

cachedb/redis.c

@@ -59,6 +59,9 @@ struct redis_moddata {
 	struct timeval timeout;	 /* timeout for connection setup and commands */
 };
 
+static redisReply* redis_command(struct module_env*, struct cachedb_env*,
+	const char*, const uint8_t*, size_t);
+
 static redisContext*
 redis_connect(const struct redis_moddata* moddata)
 {
@@ -114,6 +117,33 @@ redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 	for(i = 0; i < moddata->numctxs; i++)
 		moddata->ctxs[i] = redis_connect(moddata);
 	cachedb_env->backend_data = moddata;
+	if(env->cfg->redis_expire_records) {
+		redisReply* rep = NULL;
+		int redis_reply_type = 0;
+		/** check if setex command is supported */
+		rep = redis_command(env, cachedb_env,
+			"SETEX __UNBOUND_REDIS_CHECK__ 1 none", NULL, 0);
+		if(!rep) {
+			/** init failed, no response from redis server*/
+			log_err("redis_init: failed to init redis, the "
+				"redis-expire-records option requires the SETEX command "
+				"(redis >= 2.0.0)");
+			return 0;
+		}
+		redis_reply_type = rep->type;
+		freeReplyObject(rep);
+		switch(redis_reply_type) {
+		case REDIS_REPLY_STATUS:
+			break;
+		default:
+			/** init failed, setex command not supported */
+			log_err("redis_init: failed to init redis, the "
+				"redis-expire-records option requires the SETEX command "
+				"(redis >= 2.0.0)");
+			return 0;
+		}
+	}
+
 	return 1;
 }
 
@@ -219,7 +249,7 @@ redis_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 	rep = redis_command(env, cachedb_env, cmdbuf, NULL, 0);
 	if(!rep)
 		return 0;
-	switch (rep->type) {
+	switch(rep->type) {
 	case REDIS_REPLY_NIL:
 		verbose(VERB_ALGO, "redis_lookup: no data cached");
 		break;
@@ -249,16 +279,33 @@ redis_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 
 static void
 redis_store(struct module_env* env, struct cachedb_env* cachedb_env,
-	char* key, uint8_t* data, size_t data_len)
+	char* key, uint8_t* data, size_t data_len, time_t ttl)
 {
 	redisReply* rep;
-	char cmdbuf[4+(CACHEDB_HASHSIZE/8)*2+3+1]; /* "SET " + key + " %b" */
 	int n;
+	int set_ttl = (env->cfg->redis_expire_records &&
+		(!env->cfg->serve_expired || env->cfg->serve_expired_ttl > 0));
+	/* Supported commands:
+	 * - "SET " + key + " %b"
+	 * - "SETEX " + key + " " + ttl + " %b"
+	 */
+	char cmdbuf[6+(CACHEDB_HASHSIZE/8)*2+11+3+1];
+
+	if (!set_ttl) {
+		verbose(VERB_ALGO, "redis_store %s (%d bytes)", key, (int)data_len);
+		/* build command to set to a binary safe string */
+		n = snprintf(cmdbuf, sizeof(cmdbuf), "SET %s %%b", key);
+	} else {
+		/* add expired ttl time to redis ttl to avoid premature eviction of key */
+		ttl += env->cfg->serve_expired_ttl;
+		verbose(VERB_ALGO, "redis_store %s (%d bytes) with ttl %u",
+			key, (int)data_len, (uint32_t)ttl);
+		/* build command to set to a binary safe string */
+		n = snprintf(cmdbuf, sizeof(cmdbuf), "SETEX %s %u %%b", key,
+			(uint32_t)ttl);
+	}
 
-	verbose(VERB_ALGO, "redis_store %s (%d bytes)", key, (int)data_len);
 
-	/* build command to set to a binary safe string */
-	n = snprintf(cmdbuf, sizeof(cmdbuf), "SET %s %%b", key);
 	if(n < 0 || n >= (int)sizeof(cmdbuf)) {
 		log_err("redis_store: unexpected failure to build command");
 		return;

doc/Changelog

@@ -1,3 +1,6 @@
+1 April 2020: George
+	- Merge PR #206: Redis TTL, by Talkabout.
+
 30 March 2020: Wouter
 	- Merge PR #207: Clarify if-automatic listens on 0.0.0.0 and ::
 	- Merge PR #208: Fix uncached CLIENT_RESPONSE'es on stateful

doc/unbound.conf.5.in

@@ -2150,6 +2150,14 @@ If this timeout expires Unbound closes the connection, treats it as
 if the Redis server does not have the requested data, and will try to
 re-establish a new connection later.
 This option defaults to 100 milliseconds.
+.TP
+.B redis-expire-records: \fI<yes or no>
+If Redis record expiration is enabled.  If yes, unbound sets timeout for Redis
+records so that Redis can evict keys that have expired automatically.  If
+unbound is configured with \fBserve-expired\fR and \fBserve-expired-ttl\fR is 0,
+this option is internally reverted to "no".  Redis SETEX support is required
+for this option (Redis >= 2.0.0).
+This option defaults to no.
 .SS DNSTAP Logging Options
 DNSTAP support, when compiled in, is enabled in the \fBdnstap:\fR section.
 This starts an extra thread (when compiled with threading) that writes

util/config_file.c

@@ -341,6 +341,7 @@ config_create(void)
 	if(!(cfg->redis_server_host = strdup("127.0.0.1"))) goto error_exit;
 	cfg->redis_timeout = 100;
 	cfg->redis_server_port = 6379;
+	cfg->redis_expire_records = 0;
 #endif  /* USE_REDIS */
 #endif  /* USE_CACHEDB */
 #ifdef USE_IPSET
@@ -1147,6 +1148,7 @@ config_get_option(struct config_file* cfg, const char* opt,
 	else O_STR(opt, "redis-server-host", redis_server_host)
 	else O_DEC(opt, "redis-server-port", redis_server_port)
 	else O_DEC(opt, "redis-timeout", redis_timeout)
+	else O_YNO(opt, "redis-expire-records", redis_expire_records)
 #endif  /* USE_REDIS */
 #endif  /* USE_CACHEDB */
 #ifdef USE_IPSET

util/config_file.h

@@ -609,6 +609,8 @@ struct config_file {
 	int redis_server_port;
 	/** timeout (in ms) for communication with the redis server */
 	int redis_timeout;
+	/** set timeout on redis records based on DNS response ttl */
+	int redis_expire_records;
 #endif
 #endif
 

util/configlexer.lex

@@ -504,6 +504,7 @@ secret-seed{COLON}		{ YDVAR(1, VAR_CACHEDB_SECRETSEED) }
 redis-server-host{COLON}	{ YDVAR(1, VAR_CACHEDB_REDISHOST) }
 redis-server-port{COLON}	{ YDVAR(1, VAR_CACHEDB_REDISPORT) }
 redis-timeout{COLON}		{ YDVAR(1, VAR_CACHEDB_REDISTIMEOUT) }
+redis-expire-records{COLON}	{ YDVAR(1, VAR_CACHEDB_REDISEXPIRERECORDS) }
 ipset{COLON}			{ YDVAR(0, VAR_IPSET) }
 name-v4{COLON}			{ YDVAR(1, VAR_IPSET_NAME_V4) }
 name-v6{COLON}			{ YDVAR(1, VAR_IPSET_NAME_V6) }

util/configparser.h

@@ -250,41 +250,42 @@
 #define VAR_CACHEDB_REDISHOST 506
 #define VAR_CACHEDB_REDISPORT 507
 #define VAR_CACHEDB_REDISTIMEOUT 508
-#define VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM 509
-#define VAR_FOR_UPSTREAM 510
-#define VAR_AUTH_ZONE 511
-#define VAR_ZONEFILE 512
-#define VAR_MASTER 513
-#define VAR_URL 514
-#define VAR_FOR_DOWNSTREAM 515
-#define VAR_FALLBACK_ENABLED 516
-#define VAR_TLS_ADDITIONAL_PORT 517
-#define VAR_LOW_RTT 518
-#define VAR_LOW_RTT_PERMIL 519
-#define VAR_FAST_SERVER_PERMIL 520
-#define VAR_FAST_SERVER_NUM 521
-#define VAR_ALLOW_NOTIFY 522
-#define VAR_TLS_WIN_CERT 523
-#define VAR_TCP_CONNECTION_LIMIT 524
-#define VAR_FORWARD_NO_CACHE 525
-#define VAR_STUB_NO_CACHE 526
-#define VAR_LOG_SERVFAIL 527
-#define VAR_DENY_ANY 528
-#define VAR_UNKNOWN_SERVER_TIME_LIMIT 529
-#define VAR_LOG_TAG_QUERYREPLY 530
-#define VAR_STREAM_WAIT_SIZE 531
-#define VAR_TLS_CIPHERS 532
-#define VAR_TLS_CIPHERSUITES 533
-#define VAR_IPSET 534
-#define VAR_IPSET_NAME_V4 535
-#define VAR_IPSET_NAME_V6 536
-#define VAR_TLS_SESSION_TICKET_KEYS 537
-#define VAR_RPZ 538
-#define VAR_TAGS 539
-#define VAR_RPZ_ACTION_OVERRIDE 540
-#define VAR_RPZ_CNAME_OVERRIDE 541
-#define VAR_RPZ_LOG 542
-#define VAR_RPZ_LOG_NAME 543
+#define VAR_CACHEDB_REDISEXPIRERECORDS 509
+#define VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM 510
+#define VAR_FOR_UPSTREAM 511
+#define VAR_AUTH_ZONE 512
+#define VAR_ZONEFILE 513
+#define VAR_MASTER 514
+#define VAR_URL 515
+#define VAR_FOR_DOWNSTREAM 516
+#define VAR_FALLBACK_ENABLED 517
+#define VAR_TLS_ADDITIONAL_PORT 518
+#define VAR_LOW_RTT 519
+#define VAR_LOW_RTT_PERMIL 520
+#define VAR_FAST_SERVER_PERMIL 521
+#define VAR_FAST_SERVER_NUM 522
+#define VAR_ALLOW_NOTIFY 523
+#define VAR_TLS_WIN_CERT 524
+#define VAR_TCP_CONNECTION_LIMIT 525
+#define VAR_FORWARD_NO_CACHE 526
+#define VAR_STUB_NO_CACHE 527
+#define VAR_LOG_SERVFAIL 528
+#define VAR_DENY_ANY 529
+#define VAR_UNKNOWN_SERVER_TIME_LIMIT 530
+#define VAR_LOG_TAG_QUERYREPLY 531
+#define VAR_STREAM_WAIT_SIZE 532
+#define VAR_TLS_CIPHERS 533
+#define VAR_TLS_CIPHERSUITES 534
+#define VAR_IPSET 535
+#define VAR_IPSET_NAME_V4 536
+#define VAR_IPSET_NAME_V6 537
+#define VAR_TLS_SESSION_TICKET_KEYS 538
+#define VAR_RPZ 539
+#define VAR_TAGS 540
+#define VAR_RPZ_ACTION_OVERRIDE 541
+#define VAR_RPZ_CNAME_OVERRIDE 542
+#define VAR_RPZ_LOG 543
+#define VAR_RPZ_LOG_NAME 544
 #ifdef YYSTYPE
 #undef  YYSTYPE_IS_DECLARED
 #define YYSTYPE_IS_DECLARED 1

util/configparser.y

@@ -164,6 +164,7 @@ extern struct config_parser_state* cfg_parser;
 %token VAR_IPSECMOD_MAX_TTL VAR_IPSECMOD_WHITELIST VAR_IPSECMOD_STRICT
 %token VAR_CACHEDB VAR_CACHEDB_BACKEND VAR_CACHEDB_SECRETSEED
 %token VAR_CACHEDB_REDISHOST VAR_CACHEDB_REDISPORT VAR_CACHEDB_REDISTIMEOUT
+%token VAR_CACHEDB_REDISEXPIRERECORDS
 %token VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM VAR_FOR_UPSTREAM
 %token VAR_AUTH_ZONE VAR_ZONEFILE VAR_MASTER VAR_URL VAR_FOR_DOWNSTREAM
 %token VAR_FALLBACK_ENABLED VAR_TLS_ADDITIONAL_PORT VAR_LOW_RTT VAR_LOW_RTT_PERMIL
@@ -3117,7 +3118,8 @@ cachedbstart: VAR_CACHEDB
 contents_cachedb: contents_cachedb content_cachedb
 	| ;
 content_cachedb: cachedb_backend_name | cachedb_secret_seed |
-	redis_server_host | redis_server_port | redis_timeout
+	redis_server_host | redis_server_port | redis_timeout |
+	redis_expire_records
 	;
 cachedb_backend_name: VAR_CACHEDB_BACKEND STRING_ARG
 	{
@@ -3183,6 +3185,19 @@ redis_timeout: VAR_CACHEDB_REDISTIMEOUT STRING_ARG
 		free($2);
 	}
 	;
+redis_expire_records: VAR_CACHEDB_REDISEXPIRERECORDS STRING_ARG
+	{
+	#if defined(USE_CACHEDB) && defined(USE_REDIS)
+		OUTYY(("P(redis_expire_records:%s)\n", $2));
+		if(strcmp($2, "yes") != 0 && strcmp($2, "no") != 0)
+			yyerror("expected yes or no.");
+		else cfg_parser->cfg->redis_expire_records = (strcmp($2, "yes")==0);
+	#else
+		OUTYY(("P(Compiled without cachedb or redis, ignoring)\n"));
+	#endif
+		free($2);
+	}
+	;
 server_tcp_connection_limit: VAR_TCP_CONNECTION_LIMIT STRING_ARG STRING_ARG
 	{
 		OUTYY(("P(server_tcp_connection_limit:%s %s)\n", $2, $3));