Set hard thread affinity for each zone

After switching to per-thread resources in the zonemgr, the performance
was decreased because the memory context, zonetask and loadtask was
picked from the pool at random.

Pin the zone to single threadid (.tid) and align the memory context,
zonetask and loadtask to be the same, this sets the hard affinity of the
zone to the netmgr thread.

bin/check/check-tool.c

@@ -593,7 +593,7 @@ load_zone(isc_mem_t *mctx, const char *zonename, const char *filename,
 			zonename, filename, classname);
 	}
 
-	CHECK(dns_zone_create(&zone, mctx));
+	CHECK(dns_zone_create(&zone, mctx, 0));
 
 	dns_zone_settype(zone, dns_zone_primary);
 

bin/named/server.c

@@ -431,7 +431,7 @@ configure_alternates(const cfg_obj_t *config, dns_view_t *view,
 
 static isc_result_t
 configure_zone(const cfg_obj_t *config, const cfg_obj_t *zconfig,
-	       const cfg_obj_t *vconfig, isc_mem_t *mctx, dns_view_t *view,
+	       const cfg_obj_t *vconfig, dns_view_t *view,
 	       dns_viewlist_t *viewlist, dns_kasplist_t *kasplist,
 	       cfg_aclconfctx_t *aclconf, bool added, bool old_rpz_ok,
 	       bool modify);
@@ -442,7 +442,7 @@ configure_zone_setviewcommit(isc_result_t result, const cfg_obj_t *zconfig,
 
 static isc_result_t
 configure_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
-		   isc_mem_t *mctx, cfg_aclconfctx_t *actx);
+		   cfg_aclconfctx_t *actx);
 
 static isc_result_t
 add_keydata_zone(dns_view_t *view, const char *directory, isc_mem_t *mctx);
@@ -1951,7 +1951,7 @@ dns64_reverse(dns_view_t *view, isc_mem_t *mctx, isc_netaddr_t *na,
 	isc_buffer_constinit(&b, reverse, strlen(reverse));
 	isc_buffer_add(&b, strlen(reverse));
 	CHECK(dns_name_fromtext(name, &b, dns_rootname, 0, NULL));
-	CHECK(dns_zone_create(&zone, mctx));
+	CHECK(dns_zone_create(&zone, mctx, 0));
 	CHECK(dns_zone_setorigin(zone, name));
 	dns_zone_setview(zone, view);
 	CHECK(dns_zonemgr_managezone(named_g_server->zonemgr, zone));
@@ -2773,10 +2773,10 @@ catz_addmodzone_taskaction(isc_task_t *task, isc_event_t *event0) {
 	result = isc_task_beginexclusive(task);
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 	dns_view_thaw(ev->view);
-	result = configure_zone(
-		cfg->config, zoneobj, cfg->vconfig, ev->cbd->server->mctx,
-		ev->view, &ev->cbd->server->viewlist,
-		&ev->cbd->server->kasplist, cfg->actx, true, false, ev->mod);
+	result = configure_zone(cfg->config, zoneobj, cfg->vconfig, ev->view,
+				&ev->cbd->server->viewlist,
+				&ev->cbd->server->kasplist, cfg->actx, true,
+				false, ev->mod);
 	dns_view_freeze(ev->view);
 	isc_task_endexclusive(task);
 
@@ -3620,7 +3620,7 @@ create_ipv4only_zone(dns_zone_t *pzone, dns_view_t *view,
 		/*
 		 * Create the actual zone.
 		 */
-		CHECK(dns_zone_create(&zone, mctx));
+		CHECK(dns_zone_create(&zone, mctx, 0));
 		CHECK(dns_zone_setorigin(zone, name));
 		CHECK(dns_zonemgr_managezone(named_g_server->zonemgr, zone));
 		dns_zone_setclass(zone, view->rdclass);
@@ -4139,9 +4139,8 @@ configure_view(dns_view_t *view, dns_viewlist_t *viewlist, cfg_obj_t *config,
 	     element = cfg_list_next(element))
 	{
 		const cfg_obj_t *zconfig = cfg_listelt_value(element);
-		CHECK(configure_zone(config, zconfig, vconfig, mctx, view,
-				     viewlist, kasplist, actx, false,
-				     old_rpz_ok, false));
+		CHECK(configure_zone(config, zconfig, vconfig, view, viewlist,
+				     kasplist, actx, false, old_rpz_ok, false));
 	}
 	zones_configured = true;
 
@@ -4177,7 +4176,7 @@ configure_view(dns_view_t *view, dns_viewlist_t *viewlist, cfg_obj_t *config,
 	 * from the newzone file for zones that were added during previous
 	 * runs.
 	 */
-	CHECK(configure_newzones(view, config, vconfig, mctx, actx));
+	CHECK(configure_newzones(view, config, vconfig, actx));
 
 	/*
 	 * Create Dynamically Loadable Zone driver.
@@ -6412,7 +6411,7 @@ create_view(const cfg_obj_t *vconfig, dns_viewlist_t *viewlist,
  */
 static isc_result_t
 configure_zone(const cfg_obj_t *config, const cfg_obj_t *zconfig,
-	       const cfg_obj_t *vconfig, isc_mem_t *mctx, dns_view_t *view,
+	       const cfg_obj_t *vconfig, dns_view_t *view,
 	       dns_viewlist_t *viewlist, dns_kasplist_t *kasplist,
 	       cfg_aclconfctx_t *aclconf, bool added, bool old_rpz_ok,
 	       bool modify) {
@@ -6796,7 +6795,8 @@ configure_zone(const cfg_obj_t *config, const cfg_obj_t *zconfig,
 	if (inline_signing) {
 		dns_zone_getraw(zone, &raw);
 		if (raw == NULL) {
-			CHECK(dns_zone_create(&raw, mctx));
+			CHECK(dns_zone_create(&raw, dns_zone_getmem(zone),
+					      dns_zone_gettid(zone)));
 			CHECK(dns_zone_setorigin(raw, origin));
 			dns_zone_setview(raw, view);
 			dns_zone_setstats(raw, named_g_server->zonestats);
@@ -7886,7 +7886,7 @@ configure_zone_setviewcommit(isc_result_t result, const cfg_obj_t *zconfig,
 
 static isc_result_t
 configure_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
-		   isc_mem_t *mctx, cfg_aclconfctx_t *actx) {
+		   cfg_aclconfctx_t *actx) {
 	isc_result_t result;
 	ns_cfgctx_t *nzctx;
 	const cfg_obj_t *zonelist;
@@ -7908,7 +7908,7 @@ configure_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 	     element = cfg_list_next(element))
 	{
 		const cfg_obj_t *zconfig = cfg_listelt_value(element);
-		CHECK(configure_zone(config, zconfig, vconfig, mctx, view,
+		CHECK(configure_zone(config, zconfig, vconfig, view,
 				     &named_g_server->viewlist,
 				     &named_g_server->kasplist, actx, true,
 				     false, false));
@@ -8005,7 +8005,7 @@ cleanup:
  */
 typedef isc_result_t (*newzone_cfg_cb_t)(const cfg_obj_t *zconfig,
 					 cfg_obj_t *config, cfg_obj_t *vconfig,
-					 isc_mem_t *mctx, dns_view_t *view,
+					 dns_view_t *view,
 					 cfg_aclconfctx_t *actx);
 
 /*%
@@ -8021,7 +8021,7 @@ typedef isc_result_t (*newzone_cfg_cb_t)(const cfg_obj_t *zconfig,
  */
 static isc_result_t
 for_all_newzone_cfgs(newzone_cfg_cb_t callback, cfg_obj_t *config,
-		     cfg_obj_t *vconfig, isc_mem_t *mctx, dns_view_t *view,
+		     cfg_obj_t *vconfig, dns_view_t *view,
 		     cfg_aclconfctx_t *actx, MDB_txn *txn, MDB_dbi dbi) {
 	const cfg_obj_t *zconfig, *zlist;
 	isc_result_t result = ISC_R_SUCCESS;
@@ -8064,7 +8064,7 @@ for_all_newzone_cfgs(newzone_cfg_cb_t callback, cfg_obj_t *config,
 		/*
 		 * Invoke callback.
 		 */
-		result = callback(zconfig, config, vconfig, mctx, view, actx);
+		result = callback(zconfig, config, vconfig, view, actx);
 		if (result != ISC_R_SUCCESS) {
 			break;
 		}
@@ -8091,10 +8091,10 @@ for_all_newzone_cfgs(newzone_cfg_cb_t callback, cfg_obj_t *config,
  */
 static isc_result_t
 configure_newzone(const cfg_obj_t *zconfig, cfg_obj_t *config,
-		  cfg_obj_t *vconfig, isc_mem_t *mctx, dns_view_t *view,
+		  cfg_obj_t *vconfig, dns_view_t *view,
 		  cfg_aclconfctx_t *actx) {
 	return (configure_zone(
-		config, zconfig, vconfig, mctx, view, &named_g_server->viewlist,
+		config, zconfig, vconfig, view, &named_g_server->viewlist,
 		&named_g_server->kasplist, actx, true, false, false));
 }
 
@@ -8103,11 +8103,10 @@ configure_newzone(const cfg_obj_t *zconfig, cfg_obj_t *config,
  */
 static isc_result_t
 configure_newzone_revert(const cfg_obj_t *zconfig, cfg_obj_t *config,
-			 cfg_obj_t *vconfig, isc_mem_t *mctx, dns_view_t *view,
+			 cfg_obj_t *vconfig, dns_view_t *view,
 			 cfg_aclconfctx_t *actx) {
 	UNUSED(config);
 	UNUSED(vconfig);
-	UNUSED(mctx);
 	UNUSED(actx);
 
 	configure_zone_setviewcommit(ISC_R_FAILURE, zconfig, view);
@@ -8117,7 +8116,7 @@ configure_newzone_revert(const cfg_obj_t *zconfig, cfg_obj_t *config,
 
 static isc_result_t
 configure_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
-		   isc_mem_t *mctx, cfg_aclconfctx_t *actx) {
+		   cfg_aclconfctx_t *actx) {
 	isc_result_t result;
 	MDB_txn *txn = NULL;
 	MDB_dbi dbi;
@@ -8140,8 +8139,8 @@ configure_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 		      "for view '%s'",
 		      view->new_zone_db, view->name);
 
-	result = for_all_newzone_cfgs(configure_newzone, config, vconfig, mctx,
-				      view, actx, txn, dbi);
+	result = for_all_newzone_cfgs(configure_newzone, config, vconfig, view,
+				      actx, txn, dbi);
 	if (result != ISC_R_SUCCESS) {
 		/*
 		 * An error was encountered while attempting to configure zones
@@ -8152,7 +8151,7 @@ configure_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 		 * terms of trying to make things right.
 		 */
 		(void)for_all_newzone_cfgs(configure_newzone_revert, config,
-					   vconfig, mctx, view, actx, txn, dbi);
+					   vconfig, view, actx, txn, dbi);
 	}
 
 	(void)nzd_close(&txn, false);
@@ -13733,10 +13732,9 @@ do_addzone(named_server_t *server, ns_cfgctx_t *cfg, dns_view_t *view,
 
 	/* Mark view unfrozen and configure zone */
 	dns_view_thaw(view);
-	result = configure_zone(cfg->config, zoneobj, cfg->vconfig,
-				server->mctx, view, &server->viewlist,
-				&server->kasplist, cfg->actx, true, false,
-				false);
+	result = configure_zone(cfg->config, zoneobj, cfg->vconfig, view,
+				&server->viewlist, &server->kasplist, cfg->actx,
+				true, false, false);
 	dns_view_freeze(view);
 
 	isc_task_endexclusive(server->task);
@@ -13921,10 +13919,9 @@ do_modzone(named_server_t *server, ns_cfgctx_t *cfg, dns_view_t *view,
 
 	/* Reconfigure the zone */
 	dns_view_thaw(view);
-	result = configure_zone(cfg->config, zoneobj, cfg->vconfig,
-				server->mctx, view, &server->viewlist,
-				&server->kasplist, cfg->actx, true, false,
-				true);
+	result = configure_zone(cfg->config, zoneobj, cfg->vconfig, view,
+				&server->viewlist, &server->kasplist, cfg->actx,
+				true, false, true);
 	dns_view_freeze(view);
 
 	exclusive = false;

bin/tests/system/dyndb/driver/zone.c

@@ -67,7 +67,7 @@ create_zone(sample_instance_t *const inst, dns_name_t *const name,
 
 	zone_argv[0] = inst->db_name;
 
-	result = dns_zone_create(&raw, inst->mctx);
+	result = dns_zone_create(&raw, inst->mctx, 0); /* FIXME */
 	if (result != ISC_R_SUCCESS) {
 		log_write(ISC_LOG_ERROR, "create_zone: dns_zone_create -> %s\n",
 			  isc_result_totext(result));

lib/dns/dlz.c

@@ -58,7 +58,9 @@
 #include <isc/commandline.h>
 #include <isc/magic.h>
 #include <isc/mem.h>
+#include <isc/netmgr.h>
 #include <isc/once.h>
+#include <isc/random.h>
 #include <isc/rwlock.h>
 #include <isc/string.h>
 #include <isc/util.h>
@@ -454,7 +456,7 @@ dns_dlz_writeablezone(dns_view_t *view, dns_dlzdb_t *dlzdb,
 	INSIST(dupzone == NULL);
 
 	/* Create it */
-	result = dns_zone_create(&zone, view->mctx);
+	result = dns_zone_create(&zone, view->mctx, 0);
 	if (result != ISC_R_SUCCESS) {
 		goto cleanup;
 	}

lib/dns/include/dns/zone.h

@@ -147,7 +147,7 @@ ISC_LANG_BEGINDECLS
  ***/
 
 isc_result_t
-dns_zone_create(dns_zone_t **zonep, isc_mem_t *mctx);
+dns_zone_create(dns_zone_t **zonep, isc_mem_t *mctx, unsigned int tid);
 /*%<
  *	Creates a new empty zone and attach '*zonep' to it.
  *
@@ -2751,3 +2751,23 @@ dns_zonetype_name(dns_zonetype_t type);
 /*%<
  * Return the name of the zone type 'type'.
  */
+
+isc_mem_t *
+dns_zone_getmem(dns_zone_t *zone);
+/**<
+ * \brief Return memory context associated with the zone.
+ *
+ * @param zone valid dns_zone_t object.
+ *
+ * @return memory context associated with the zone
+ */
+
+unsigned int
+dns_zone_gettid(dns_zone_t *zone);
+/**<
+ * \brief Return thread-id associated with the zone.
+ *
+ * \param valid dns_zone_t object
+ *
+ * \return thread id associated with the zone
+ */

lib/dns/zone.c

@@ -234,6 +234,8 @@ struct dns_zone {
 	isc_rwlock_t dblock;
 	dns_db_t *db; /* Locked by dblock */
 
+	unsigned int tid;
+
 	/* Locked */
 	dns_zonemgr_t *zmgr;
 	ISC_LINK(dns_zone_t) link; /* Used by zmgr. */
@@ -595,6 +597,7 @@ struct dns_zonemgr {
 	isc_taskmgr_t *taskmgr;
 	isc_timermgr_t *timermgr;
 	isc_nm_t *netmgr;
+	unsigned int workers;
 	atomic_uint_fast32_t nzonetasks;
 	isc_pool_t *zonetasks;
 	atomic_uint_fast32_t nloadtasks;
@@ -1103,48 +1106,50 @@ inc_stats(dns_zone_t *zone, isc_statscounter_t counter) {
  ***/
 
 isc_result_t
-dns_zone_create(dns_zone_t **zonep, isc_mem_t *mctx) {
+dns_zone_create(dns_zone_t **zonep, isc_mem_t *mctx, unsigned int tid) {
 	isc_result_t result;
 	isc_time_t now;
 	dns_zone_t *zone = NULL;
-	dns_zone_t z = { .masterformat = dns_masterformat_none,
-			 .journalsize = -1,
-			 .rdclass = dns_rdataclass_none,
-			 .type = dns_zone_none,
-			 .refresh = DNS_ZONE_DEFAULTREFRESH,
-			 .retry = DNS_ZONE_DEFAULTRETRY,
-			 .maxrefresh = DNS_ZONE_MAXREFRESH,
-			 .minrefresh = DNS_ZONE_MINREFRESH,
-			 .maxretry = DNS_ZONE_MAXRETRY,
-			 .minretry = DNS_ZONE_MINRETRY,
-			 .notifytype = dns_notifytype_yes,
-			 .zero_no_soa_ttl = true,
-			 .check_names = dns_severity_ignore,
-			 .idlein = DNS_DEFAULT_IDLEIN,
-			 .idleout = DNS_DEFAULT_IDLEOUT,
-			 .notifysrc4dscp = -1,
-			 .notifysrc6dscp = -1,
-			 .parentalsrc4dscp = -1,
-			 .parentalsrc6dscp = -1,
-			 .xfrsource4dscp = -1,
-			 .xfrsource6dscp = -1,
-			 .altxfrsource4dscp = -1,
-			 .altxfrsource6dscp = -1,
-			 .maxxfrin = MAX_XFER_TIME,
-			 .maxxfrout = MAX_XFER_TIME,
-			 .sigvalidityinterval = 30 * 24 * 3600,
-			 .sigresigninginterval = 7 * 24 * 3600,
-			 .statlevel = dns_zonestat_none,
-			 .notifydelay = 5,
-			 .signatures = 10,
-			 .nodes = 100,
-			 .privatetype = (dns_rdatatype_t)0xffffU,
-			 .rpz_num = DNS_RPZ_INVALID_NUM,
-			 .requestixfr = true,
-			 .ixfr_ratio = 100,
-			 .requestexpire = true,
-			 .updatemethod = dns_updatemethod_increment,
-			 .magic = ZONE_MAGIC };
+	dns_zone_t z = {
+		.masterformat = dns_masterformat_none,
+		.journalsize = -1,
+		.rdclass = dns_rdataclass_none,
+		.type = dns_zone_none,
+		.refresh = DNS_ZONE_DEFAULTREFRESH,
+		.retry = DNS_ZONE_DEFAULTRETRY,
+		.maxrefresh = DNS_ZONE_MAXREFRESH,
+		.minrefresh = DNS_ZONE_MINREFRESH,
+		.maxretry = DNS_ZONE_MAXRETRY,
+		.minretry = DNS_ZONE_MINRETRY,
+		.notifytype = dns_notifytype_yes,
+		.zero_no_soa_ttl = true,
+		.check_names = dns_severity_ignore,
+		.idlein = DNS_DEFAULT_IDLEIN,
+		.idleout = DNS_DEFAULT_IDLEOUT,
+		.notifysrc4dscp = -1,
+		.notifysrc6dscp = -1,
+		.parentalsrc4dscp = -1,
+		.parentalsrc6dscp = -1,
+		.xfrsource4dscp = -1,
+		.xfrsource6dscp = -1,
+		.altxfrsource4dscp = -1,
+		.altxfrsource6dscp = -1,
+		.maxxfrin = MAX_XFER_TIME,
+		.maxxfrout = MAX_XFER_TIME,
+		.sigvalidityinterval = 30 * 24 * 3600,
+		.sigresigninginterval = 7 * 24 * 3600,
+		.statlevel = dns_zonestat_none,
+		.notifydelay = 5,
+		.signatures = 10,
+		.nodes = 100,
+		.privatetype = (dns_rdatatype_t)0xffffU,
+		.rpz_num = DNS_RPZ_INVALID_NUM,
+		.requestixfr = true,
+		.ixfr_ratio = 100,
+		.requestexpire = true,
+		.updatemethod = dns_updatemethod_increment,
+		.tid = tid,
+	};
 
 	REQUIRE(zonep != NULL && *zonep == NULL);
 	REQUIRE(mctx != NULL);
@@ -1206,6 +1211,8 @@ dns_zone_create(dns_zone_t **zonep, isc_mem_t *mctx) {
 		goto free_refs;
 	}
 
+	zone->magic = ZONE_MAGIC;
+
 	/* Must be after magic is set. */
 	dns_zone_setdbtype(zone, dbargc_default, dbargv_default);
 
@@ -18938,7 +18945,7 @@ dns_zonemgr_createzone(dns_zonemgr_t *zmgr, dns_zone_t **zonep) {
 	isc_result_t result;
 	isc_mem_t *mctx = NULL;
 	dns_zone_t *zone = NULL;
-	void *item;
+	unsigned int tid;
 
 	REQUIRE(DNS_ZONEMGR_VALID(zmgr));
 	REQUIRE(zonep != NULL && *zonep == NULL);
@@ -18947,14 +18954,14 @@ dns_zonemgr_createzone(dns_zonemgr_t *zmgr, dns_zone_t **zonep) {
 		return (ISC_R_FAILURE);
 	}
 
-	item = isc_pool_get(zmgr->mctxpool);
-	if (item == NULL) {
+	tid = isc_random_uniform(zmgr->workers);
+
+	mctx = isc_pool_get(zmgr->mctxpool, tid);
+	if (mctx == NULL) {
 		return (ISC_R_FAILURE);
 	}
 
-	isc_mem_attach((isc_mem_t *)item, &mctx);
-	result = dns_zone_create(&zone, mctx);
-	isc_mem_detach(&mctx);
+	result = dns_zone_create(&zone, mctx, tid);
 
 	if (result == ISC_R_SUCCESS) {
 		*zonep = zone;
@@ -18978,8 +18985,9 @@ dns_zonemgr_managezone(dns_zonemgr_t *zmgr, dns_zone_t *zone) {
 	REQUIRE(zone->timer == NULL);
 	REQUIRE(zone->zmgr == NULL);
 
-	isc_task_attach(isc_pool_get(zmgr->zonetasks), &zone->task);
-	isc_task_attach(isc_pool_get(zmgr->loadtasks), &zone->loadtask);
+	isc_task_attach(isc_pool_get(zmgr->zonetasks, zone->tid), &zone->task);
+	isc_task_attach(isc_pool_get(zmgr->loadtasks, zone->tid),
+			&zone->loadtask);
 
 	/*
 	 * Set the task name.  The tag will arbitrarily point to one
@@ -19201,6 +19209,8 @@ static isc_result_t
 zonemgr_setsize(dns_zonemgr_t *zmgr, unsigned int workers) {
 	isc_result_t result;
 
+	zmgr->workers = workers;
+
 	/* Create the zone tasks pool. */
 	REQUIRE(zmgr->zonetasks == NULL);
 	result = isc_pool_create(zmgr->mctx, workers, taskfree, zonetaskinit,
@@ -23713,3 +23723,13 @@ dns_zonemgr_set_tlsctx_cache(dns_zonemgr_t *zmgr,
 
 	RWUNLOCK(&zmgr->rwlock, isc_rwlocktype_write);
 }
+
+isc_mem_t *
+dns_zone_getmem(dns_zone_t *zone) {
+	return (zone->mctx);
+}
+
+unsigned int
+dns_zone_gettid(dns_zone_t *zone) {
+	return (zone->tid);
+}

lib/isc/include/isc/pool.h

@@ -81,7 +81,7 @@ isc_pool_create(isc_mem_t *mctx, unsigned int count, isc_pooldeallocator_t free,
  */
 
 void *
-isc_pool_get(isc_pool_t *pool);
+isc_pool_get(isc_pool_t *pool, unsigned int tid);
 /*%<
  * Returns a pointer to an object from the pool. Currently the object
  * is chosen from the pool at random.

lib/isc/pool.c

@@ -73,8 +73,10 @@ isc_pool_create(isc_mem_t *mctx, unsigned int count,
 }
 
 void *
-isc_pool_get(isc_pool_t *pool) {
-	return (pool->pool[isc_random_uniform(pool->count)]);
+isc_pool_get(isc_pool_t *pool, unsigned int tid) {
+	REQUIRE(tid < pool->count);
+
+	return (pool->pool[tid]);
 }
 
 void

lib/ns/tests/nstest.c

@@ -412,7 +412,7 @@ ns_test_makezone(const char *name, dns_zone_t **zonep, dns_view_t *view,
 
 	zone = *zonep;
 	if (zone == NULL) {
-		CHECK(dns_zone_create(&zone, mctx));
+		CHECK(dns_zone_create(&zone, mctx, 0));
 	}
 
 	isc_buffer_constinit(&buffer, name, strlen(name));