diff --git a/lib/dns/db_p.h b/lib/dns/db_p.h index 3a982252a6..2d920a823e 100644 --- a/lib/dns/db_p.h +++ b/lib/dns/db_p.h @@ -21,9 +21,6 @@ #include #include -#define GLUETABLE_INIT_SIZE 1 << 2 -#define GLUETABLE_MIN_SIZE 1 << 8 - #define RDATATYPE_NCACHEANY DNS_TYPEPAIR_VALUE(0, dns_rdatatype_any) #ifdef STRONG_RWLOCK_CHECK @@ -128,6 +125,9 @@ struct dns_glue { dns_rdataset_t sigrdataset_a; dns_rdataset_t rdataset_aaaa; dns_rdataset_t sigrdataset_aaaa; + + isc_mem_t *mctx; + struct rcu_head rcu_head; }; typedef struct { diff --git a/lib/dns/include/dns/db.h b/lib/dns/include/dns/db.h index e38f4a3aca..254b7d38e5 100644 --- a/lib/dns/include/dns/db.h +++ b/lib/dns/include/dns/db.h @@ -177,8 +177,8 @@ typedef struct dns_dbmethods { void (*locknode)(dns_db_t *db, dns_dbnode_t *node, isc_rwlocktype_t t); void (*unlocknode)(dns_db_t *db, dns_dbnode_t *node, isc_rwlocktype_t t); - void (*addglue)(dns_db_t *db, dns_dbversion_t *version, - dns_rdataset_t *rdataset, dns_message_t *msg); + isc_result_t (*addglue)(dns_db_t *db, dns_dbversion_t *version, + dns_rdataset_t *rdataset, dns_message_t *msg); void (*expiredata)(dns_db_t *db, dns_dbnode_t *node, void *data); void (*deletedata)(dns_db_t *db, dns_dbnode_t *node, void *data); isc_result_t (*nodefullname)(dns_db_t *db, dns_dbnode_t *node, diff --git a/lib/dns/include/dns/rdataslab.h b/lib/dns/include/dns/rdataslab.h index e5ecbebf33..1a46d12d0f 100644 --- a/lib/dns/include/dns/rdataslab.h +++ b/lib/dns/include/dns/rdataslab.h @@ -132,7 +132,9 @@ struct dns_slabheader { */ unsigned char upper[32]; - isc_heap_t *heap; + isc_heap_t *heap; + dns_glue_t *glue_list; + struct cds_wfs_node wfs_node; }; enum { diff --git a/lib/dns/qpzone.c b/lib/dns/qpzone.c index 687aa40847..3acb6b7ea8 100644 --- a/lib/dns/qpzone.c +++ b/lib/dns/qpzone.c @@ -143,7 +143,7 @@ struct qpz_version { uint64_t records; uint64_t xfrsize; - struct cds_lfht *glue_table; + struct cds_wfs_stack glue_stack; }; typedef ISC_LIST(qpz_version_t) qpz_versionlist_t; @@ -214,17 +214,6 @@ typedef struct { isc_stdtime_t now; } qpz_search_t; -typedef struct dns_gluenode_t { - isc_mem_t *mctx; - - struct dns_glue *glue; - - qpznode_t *node; - - struct cds_lfht_node ht_node; - struct rcu_head rcu_head; -} dns_gluenode_t; - /*% * Load Context */ @@ -381,21 +370,61 @@ set_index(void *what, unsigned int idx) { } static void -free_gluenode(dns_gluenode_t *gluenode); +freeglue(dns_glue_t *glue_list) { + if (glue_list == (void *)-1) { + return; + } + + dns_glue_t *glue = glue_list; + while (glue != NULL) { + dns_glue_t *next = glue->next; + + if (dns_rdataset_isassociated(&glue->rdataset_a)) { + dns_rdataset_disassociate(&glue->rdataset_a); + } + if (dns_rdataset_isassociated(&glue->sigrdataset_a)) { + dns_rdataset_disassociate(&glue->sigrdataset_a); + } + + if (dns_rdataset_isassociated(&glue->rdataset_aaaa)) { + dns_rdataset_disassociate(&glue->rdataset_aaaa); + } + if (dns_rdataset_isassociated(&glue->sigrdataset_aaaa)) { + dns_rdataset_disassociate(&glue->sigrdataset_aaaa); + } + + dns_rdataset_invalidate(&glue->rdataset_a); + dns_rdataset_invalidate(&glue->sigrdataset_a); + dns_rdataset_invalidate(&glue->rdataset_aaaa); + dns_rdataset_invalidate(&glue->sigrdataset_aaaa); + + isc_mem_putanddetach(&glue->mctx, glue, sizeof(*glue)); + + glue = next; + } +} static void -free_gluetable(struct cds_lfht *glue_table) { - struct cds_lfht_iter iter; - dns_gluenode_t *gluenode = NULL; +free_gluelist_rcu(struct rcu_head *rcu_head) { + dns_glue_t *glue = caa_container_of(rcu_head, dns_glue_t, rcu_head); + + freeglue(glue); +} + +static void +free_gluetable(struct cds_wfs_stack *glue_stack) { + struct cds_wfs_head *head = __cds_wfs_pop_all(glue_stack); + struct cds_wfs_node *node = NULL, *next = NULL; rcu_read_lock(); - cds_lfht_for_each_entry(glue_table, &iter, gluenode, ht_node) { - INSIST(!cds_lfht_del(glue_table, &gluenode->ht_node)); - free_gluenode(gluenode); + cds_wfs_for_each_blocking_safe(head, node, next) { + dns_slabheader_t *header = + caa_container_of(node, dns_slabheader_t, wfs_node); + dns_glue_t *glue = rcu_xchg_pointer(&header->glue_list, NULL); + + call_rcu(&glue->rcu_head, free_gluelist_rcu); } rcu_read_unlock(); - - cds_lfht_destroy(glue_table, NULL); } static void @@ -442,6 +471,7 @@ free_qpdb(qpzonedb_t *qpdb, bool log) { isc_refcount_destroy(&qpdb->current_version->references); UNLINK(qpdb->open_versions, qpdb->current_version, link); + cds_wfs_destroy(&qpdb->current_version->glue_stack); isc_rwlock_destroy(&qpdb->current_version->rwlock); isc_mem_put(qpdb->common.mctx, qpdb->current_version, sizeof(*qpdb->current_version)); @@ -483,7 +513,7 @@ qpdb_destroy(dns_db_t *arg) { * node count below. */ if (qpdb->current_version != NULL) { - free_gluetable(qpdb->current_version->glue_table); + free_gluetable(&qpdb->current_version->glue_stack); } /* @@ -555,11 +585,9 @@ allocate_version(isc_mem_t *mctx, uint32_t serial, unsigned int references, .changed_list = ISC_LIST_INITIALIZER, .resigned_list = ISC_LIST_INITIALIZER, .link = ISC_LINK_INITIALIZER, - .glue_table = cds_lfht_new(GLUETABLE_INIT_SIZE, - GLUETABLE_MIN_SIZE, 0, - CDS_LFHT_AUTO_RESIZE, NULL), }; + cds_wfs_init(&version->glue_stack); isc_rwlock_init(&version->rwlock); isc_refcount_init(&version->references, references); @@ -1447,7 +1475,8 @@ closeversion(dns_db_t *db, dns_dbversion_t **versionp, if (cleanup_version != NULL) { isc_refcount_destroy(&cleanup_version->references); INSIST(EMPTY(cleanup_version->changed_list)); - free_gluetable(cleanup_version->glue_table); + free_gluetable(&cleanup_version->glue_stack); + cds_wfs_destroy(&cleanup_version->glue_stack); isc_rwlock_destroy(&cleanup_version->rwlock); isc_mem_put(qpdb->common.mctx, cleanup_version, sizeof(*cleanup_version)); @@ -3991,6 +4020,10 @@ deletedata(dns_db_t *db ISC_ATTR_UNUSED, dns_dbnode_t *node ISC_ATTR_UNUSED, RWUNLOCK(&qpdb->lock, isc_rwlocktype_write); } header->heap_index = 0; + + if (header->glue_list) { + freeglue(header->glue_list); + } } /* @@ -4960,6 +4993,7 @@ new_gluelist(isc_mem_t *mctx, dns_name_t *name) { *glue = (dns_glue_t){ 0 }; dns_name_t *gluename = dns_fixedname_initname(&glue->fixedname); + isc_mem_attach(mctx, &glue->mctx); dns_name_copy(name, gluename); return glue; @@ -5172,11 +5206,11 @@ addglue_to_message(dns_glue_t *ge, dns_message_t *msg) { } static dns_glue_t * -newglue(dns_db_t *db, qpz_version_t *version, qpznode_t *node, +newglue(qpzonedb_t *qpdb, qpz_version_t *version, qpznode_t *node, dns_rdataset_t *rdataset) { dns_fixedname_t nodename; dns_glue_additionaldata_ctx_t ctx = { - .db = db, + .db = (dns_db_t *)qpdb, .version = (dns_dbversion_t *)version, .nodename = dns_fixedname_initname(&nodename), }; @@ -5195,160 +5229,59 @@ newglue(dns_db_t *db, qpz_version_t *version, qpznode_t *node, return ctx.glue_list; } -static dns_gluenode_t * -new_gluenode(dns_db_t *db, qpz_version_t *version, qpznode_t *node, - dns_rdataset_t *rdataset) { - dns_gluenode_t *gluenode = isc_mem_get(db->mctx, sizeof(*gluenode)); - *gluenode = (dns_gluenode_t){ - .glue = newglue(db, version, node, rdataset), - }; - - isc_mem_attach(db->mctx, &gluenode->mctx); - qpznode_attach(node, &gluenode->node); - - return gluenode; -} - -static void -freeglue(isc_mem_t *mctx, dns_glue_t *glue) { - while (glue != NULL) { - dns_glue_t *next = glue->next; - - if (dns_rdataset_isassociated(&glue->rdataset_a)) { - dns_rdataset_disassociate(&glue->rdataset_a); - } - if (dns_rdataset_isassociated(&glue->sigrdataset_a)) { - dns_rdataset_disassociate(&glue->sigrdataset_a); - } - - if (dns_rdataset_isassociated(&glue->rdataset_aaaa)) { - dns_rdataset_disassociate(&glue->rdataset_aaaa); - } - if (dns_rdataset_isassociated(&glue->sigrdataset_aaaa)) { - dns_rdataset_disassociate(&glue->sigrdataset_aaaa); - } - - dns_rdataset_invalidate(&glue->rdataset_a); - dns_rdataset_invalidate(&glue->sigrdataset_a); - dns_rdataset_invalidate(&glue->rdataset_aaaa); - dns_rdataset_invalidate(&glue->sigrdataset_aaaa); - - isc_mem_put(mctx, glue, sizeof(*glue)); - - glue = next; - } -} - -static void -free_gluenode_rcu(struct rcu_head *rcu_head) { - dns_gluenode_t *gluenode = caa_container_of(rcu_head, dns_gluenode_t, - rcu_head); - - freeglue(gluenode->mctx, gluenode->glue); - - qpznode_detach(&gluenode->node); - - isc_mem_putanddetach(&gluenode->mctx, gluenode, sizeof(*gluenode)); -} - -static void -free_gluenode(dns_gluenode_t *gluenode) { - call_rcu(&gluenode->rcu_head, free_gluenode_rcu); -} - -static uint32_t -qpznode_hash(const qpznode_t *node) { - return isc_hash32(&node, sizeof(node), true); -} - -static int -qpznode_match(struct cds_lfht_node *ht_node, const void *key) { - const dns_gluenode_t *gluenode = - caa_container_of(ht_node, dns_gluenode_t, ht_node); - const qpznode_t *node = key; - - return gluenode->node == node; -} - -static uint32_t -gluenode_hash(const dns_gluenode_t *gluenode) { - return qpznode_hash(gluenode->node); -} - -static int -gluenode_match(struct cds_lfht_node *ht_node, const void *key) { - const dns_gluenode_t *gluenode = key; - - return qpznode_match(ht_node, gluenode->node); -} - -static void +static isc_result_t addglue(dns_db_t *db, dns_dbversion_t *dbversion, dns_rdataset_t *rdataset, dns_message_t *msg) { qpzonedb_t *qpdb = (qpzonedb_t *)db; qpz_version_t *version = dbversion; qpznode_t *node = (qpznode_t *)rdataset->slab.node; - dns_gluenode_t *gluenode = NULL; + dns_slabheader_t *header = dns_slabheader_fromrdataset(rdataset); REQUIRE(rdataset->type == dns_rdatatype_ns); REQUIRE(qpdb == (qpzonedb_t *)rdataset->slab.db); REQUIRE(qpdb == version->qpdb); REQUIRE(!IS_STUB(qpdb)); - /* - * The glue table cache that forms a part of the DB version - * structure is not explicitly bounded and there's no cache - * cleaning. The zone data size itself is an implicit bound. - * - * The key into the glue hashtable is the node pointer. This is - * because the glue hashtable is a property of the DB version, - * and the glue is keyed for the ownername/NS tuple. We don't - * bother with using an expensive dns_name_t comparison here as - * the node pointer is a fixed value that won't change for a DB - * version and can be compared directly. - */ - rcu_read_lock(); - struct cds_lfht_iter iter; - cds_lfht_lookup(version->glue_table, qpznode_hash(node), qpznode_match, - node, &iter); - - gluenode = cds_lfht_entry(cds_lfht_iter_get_node(&iter), dns_gluenode_t, - ht_node); - if (gluenode == NULL) { + dns_glue_t *glue = rcu_dereference(header->glue_list); + if (glue == NULL) { /* No cached glue was found in the table. Get new glue. */ - gluenode = new_gluenode(db, version, node, rdataset); + glue = newglue(qpdb, version, node, rdataset); - struct cds_lfht_node *ht_node = cds_lfht_add_unique( - version->glue_table, gluenode_hash(gluenode), - gluenode_match, gluenode, &gluenode->ht_node); - - if (ht_node != &gluenode->ht_node) { - free_gluenode_rcu(&gluenode->rcu_head); - - gluenode = cds_lfht_entry(ht_node, dns_gluenode_t, - ht_node); + /* Cache the glue or (void *)-1 if no glue was found. */ + dns_glue_t *old_glue = rcu_cmpxchg_pointer( + &header->glue_list, NULL, (glue) ? glue : (void *)-1); + if (old_glue != NULL) { + /* Somebody else was faster */ + freeglue(glue); + glue = old_glue; + } else if (glue != NULL) { + cds_wfs_push(&version->glue_stack, &header->wfs_node); } } - INSIST(gluenode != NULL); + /* We have a cached result. Add it to the message and return. */ - dns_glue_t *glue = gluenode->glue; - isc_statscounter_t counter = dns_gluecachestatscounter_hits_present; + if (qpdb->gluecachestats != NULL) { + isc_stats_increment( + qpdb->gluecachestats, + (glue == (void *)-1) + ? dns_gluecachestatscounter_hits_absent + : dns_gluecachestatscounter_hits_present); + } - if (glue != NULL) { - /* We have a cached result. Add it to the message and return. */ + /* + * (void *)-1 is a special value that means no glue is present in the + * zone. + */ + if (glue != (void *)-1) { addglue_to_message(glue, msg); - } else { - counter = dns_gluecachestatscounter_hits_absent; } rcu_read_unlock(); - if (qpdb->gluecachestats != NULL) { - isc_stats_increment(qpdb->gluecachestats, counter); - } + return ISC_R_SUCCESS; } static void diff --git a/lib/dns/rbt-zonedb.c b/lib/dns/rbt-zonedb.c index bed5c29527..ba7d46d241 100644 --- a/lib/dns/rbt-zonedb.c +++ b/lib/dns/rbt-zonedb.c @@ -2097,6 +2097,7 @@ new_gluelist(isc_mem_t *mctx, dns_name_t *name) { *glue = (dns_glue_t){ 0 }; dns_name_t *gluename = dns_fixedname_initname(&glue->fixedname); + isc_mem_attach(mctx, &glue->mctx); dns_name_copy(name, gluename); return glue; @@ -2310,12 +2311,12 @@ addglue_to_message(dns_glue_t *ge, dns_message_t *msg) { } static dns_glue_t * -newglue(dns_db_t *db, dns_dbversion_t *dbversion, dns_rbtnode_t *node, - dns_rdataset_t *rdataset) { +newglue(dns_rbtdb_t *rbtdb, dns_rbtdb_version_t *rbtversion, + dns_rbtnode_t *node, dns_rdataset_t *rdataset) { dns_fixedname_t nodename; dns_glue_additionaldata_ctx_t ctx = { - .db = db, - .version = dbversion, + .db = (dns_db_t *)rbtdb, + .version = (dns_dbversion_t *)rbtversion, .nodename = dns_fixedname_initname(&nodename), }; @@ -2325,7 +2326,7 @@ newglue(dns_db_t *db, dns_dbversion_t *dbversion, dns_rbtnode_t *node, * determining which NS records in the delegation are * in-bailiwick). */ - dns__rbtdb_nodefullname(db, node, ctx.nodename); + dns__rbtdb_nodefullname((dns_db_t *)rbtdb, node, ctx.nodename); (void)dns_rdataset_additionaldata(rdataset, dns_rootname, glue_nsdname_cb, &ctx); @@ -2333,117 +2334,60 @@ newglue(dns_db_t *db, dns_dbversion_t *dbversion, dns_rbtnode_t *node, return ctx.glue_list; } -/* FIXME: Perhaps we can squash dns_gluenode_t with - * dns_glue_additionaldata_ctx_t */ - -static dns_gluenode_t * -new_gluenode(dns_db_t *db, dns_dbversion_t *dbversion, dns_rbtnode_t *node, - dns_rdataset_t *rdataset) { - dns_gluenode_t *gluenode = isc_mem_get(db->mctx, sizeof(*gluenode)); - *gluenode = (dns_gluenode_t){ - .glue = newglue(db, dbversion, node, rdataset), - .db = db, - }; - - isc_mem_attach(db->mctx, &gluenode->mctx); - dns_db_attachnode(db, node, (dns_dbnode_t **)&gluenode->node); - - return gluenode; -} - -static uint32_t -rbtnode_hash(const dns_rbtnode_t *node) { - return isc_hash32(&node, sizeof(node), true); -} - -static int -rbtnode_match(struct cds_lfht_node *ht_node, const void *key) { - const dns_gluenode_t *gluenode = - caa_container_of(ht_node, dns_gluenode_t, ht_node); - const dns_rbtnode_t *node = key; - - return gluenode->node == node; -} - -static uint32_t -gluenode_hash(const dns_gluenode_t *gluenode) { - return rbtnode_hash(gluenode->node); -} - -static int -gluenode_match(struct cds_lfht_node *ht_node, const void *key) { - const dns_gluenode_t *gluenode = key; - - return rbtnode_match(ht_node, gluenode->node); -} - -static void -addglue(dns_db_t *db, dns_dbversion_t *dbversion, dns_rdataset_t *rdataset, +static isc_result_t +addglue(dns_db_t *db, dns_dbversion_t *version, dns_rdataset_t *rdataset, dns_message_t *msg) { dns_rbtdb_t *rbtdb = (dns_rbtdb_t *)db; - dns_rbtdb_version_t *version = dbversion; + dns_rbtdb_version_t *rbtversion = version; dns_rbtnode_t *node = (dns_rbtnode_t *)rdataset->slab.node; - dns_gluenode_t *gluenode = NULL; + dns_slabheader_t *header = dns_slabheader_fromrdataset(rdataset); REQUIRE(rdataset->type == dns_rdatatype_ns); REQUIRE(rbtdb == (dns_rbtdb_t *)rdataset->slab.db); - REQUIRE(rbtdb == version->rbtdb); + REQUIRE(rbtdb == rbtversion->rbtdb); REQUIRE(!IS_CACHE(rbtdb) && !IS_STUB(rbtdb)); - /* - * The glue table cache that forms a part of the DB version - * structure is not explicitly bounded and there's no cache - * cleaning. The zone data size itself is an implicit bound. - * - * The key into the glue hashtable is the node pointer. This is - * because the glue hashtable is a property of the DB version, - * and the glue is keyed for the ownername/NS tuple. We don't - * bother with using an expensive dns_name_t comparison here as - * the node pointer is a fixed value that won't change for a DB - * version and can be compared directly. - */ - rcu_read_lock(); - struct cds_lfht_iter iter; - cds_lfht_lookup(version->glue_table, rbtnode_hash(node), rbtnode_match, - node, &iter); - - gluenode = cds_lfht_entry(cds_lfht_iter_get_node(&iter), dns_gluenode_t, - ht_node); - if (gluenode == NULL) { + dns_glue_t *glue = rcu_dereference(header->glue_list); + if (glue == NULL) { /* No cached glue was found in the table. Get new glue. */ - gluenode = new_gluenode(db, version, node, rdataset); + glue = newglue(rbtdb, rbtversion, node, rdataset); - struct cds_lfht_node *ht_node = cds_lfht_add_unique( - version->glue_table, gluenode_hash(gluenode), - gluenode_match, gluenode, &gluenode->ht_node); - - if (ht_node != &gluenode->ht_node) { - dns__rbtdb_free_gluenode_rcu(&gluenode->rcu_head); - - gluenode = cds_lfht_entry(ht_node, dns_gluenode_t, - ht_node); + /* Cache the glue or (void *)-1 if no glue was found. */ + dns_glue_t *old_glue = rcu_cmpxchg_pointer( + &header->glue_list, NULL, (glue) ? glue : (void *)-1); + if (old_glue != NULL) { + /* Somebody else was faster */ + dns__rbtdb_freeglue(glue); + glue = old_glue; + } else if (glue != NULL) { + cds_wfs_push(&rbtversion->glue_stack, + &header->wfs_node); } } - INSIST(gluenode != NULL); + /* We have a cached result. Add it to the message and return. */ - dns_glue_t *glue = gluenode->glue; - isc_statscounter_t counter = dns_gluecachestatscounter_hits_present; + if (rbtdb->gluecachestats != NULL) { + isc_stats_increment( + rbtdb->gluecachestats, + (glue == (void *)-1) + ? dns_gluecachestatscounter_hits_absent + : dns_gluecachestatscounter_hits_present); + } - if (glue != NULL) { - /* We have a cached result. Add it to the message and return. */ + /* + * (void *)-1 is a special value that means no glue is present in the + * zone. + */ + if (glue != (void *)-1) { addglue_to_message(glue, msg); - } else { - counter = dns_gluecachestatscounter_hits_absent; } rcu_read_unlock(); - if (rbtdb->gluecachestats != NULL) { - isc_stats_increment(rbtdb->gluecachestats, counter); - } + return ISC_R_SUCCESS; } dns_dbmethods_t dns__rbtdb_zonemethods = { diff --git a/lib/dns/rbtdb.c b/lib/dns/rbtdb.c index 7d97640f84..fc4ac5068e 100644 --- a/lib/dns/rbtdb.c +++ b/lib/dns/rbtdb.c @@ -167,7 +167,7 @@ delete_callback(void *data, void *arg); static void prune_tree(void *arg); static void -free_gluetable(struct cds_lfht *glue_table); +free_gluetable(dns_rbtdb_version_t *version); static void rdatasetiter_destroy(dns_rdatasetiter_t **iteratorp DNS__DB_FLARG); @@ -466,6 +466,7 @@ free_rbtdb(dns_rbtdb_t *rbtdb, bool log) { isc_refcount_destroy(&rbtdb->current_version->references); UNLINK(rbtdb->open_versions, rbtdb->current_version, link); + cds_wfs_destroy(&rbtdb->current_version->glue_stack); isc_rwlock_destroy(&rbtdb->current_version->rwlock); isc_mem_put(rbtdb->common.mctx, rbtdb->current_version, sizeof(*rbtdb->current_version)); @@ -622,7 +623,7 @@ dns__rbtdb_destroy(dns_db_t *arg) { * node count below. */ if (rbtdb->current_version != NULL) { - free_gluetable(rbtdb->current_version->glue_table); + free_gluetable(rbtdb->current_version); } /* @@ -688,12 +689,10 @@ allocate_version(isc_mem_t *mctx, uint32_t serial, unsigned int references, .changed_list = ISC_LIST_INITIALIZER, .resigned_list = ISC_LIST_INITIALIZER, .link = ISC_LINK_INITIALIZER, - .glue_table = cds_lfht_new(GLUETABLE_INIT_SIZE, - GLUETABLE_MIN_SIZE, 0, - CDS_LFHT_AUTO_RESIZE, NULL), }; - isc_rwlock_init(&version->rwlock); + cds_wfs_init(&version->glue_stack); + isc_refcount_init(&version->references, references); return version; @@ -730,6 +729,7 @@ dns__rbtdb_newversion(dns_db_t *db, dns_dbversion_t **versionp) { version->salt_length = 0; memset(version->salt, 0, sizeof(version->salt)); } + isc_rwlock_init(&version->rwlock); RWLOCK(&rbtdb->current_version->rwlock, isc_rwlocktype_read); version->records = rbtdb->current_version->records; version->xfrsize = rbtdb->current_version->xfrsize; @@ -1947,7 +1947,8 @@ dns__rbtdb_closeversion(dns_db_t *db, dns_dbversion_t **versionp, if (cleanup_version != NULL) { isc_refcount_destroy(&cleanup_version->references); INSIST(EMPTY(cleanup_version->changed_list)); - free_gluetable(cleanup_version->glue_table); + free_gluetable(cleanup_version); + cds_wfs_destroy(&cleanup_version->glue_stack); isc_rwlock_destroy(&cleanup_version->rwlock); isc_mem_put(rbtdb->common.mctx, cleanup_version, sizeof(*cleanup_version)); @@ -4084,6 +4085,7 @@ dns__rbtdb_create(isc_mem_t *mctx, const dns_name_t *origin, dns_dbtype_t type, */ rbtdb->current_version = allocate_version(mctx, 1, 1, false); rbtdb->current_version->rbtdb = rbtdb; + isc_rwlock_init(&rbtdb->current_version->rwlock); /* * Keep the current version in the open list so that list operation @@ -4871,8 +4873,13 @@ dbiterator_origin(dns_dbiterator_t *iterator, dns_name_t *name) { return ISC_R_SUCCESS; } -static void -freeglue(isc_mem_t *mctx, dns_glue_t *glue) { +void +dns__rbtdb_freeglue(dns_glue_t *glue_list) { + if (glue_list == (void *)-1) { + return; + } + + dns_glue_t *glue = glue_list; while (glue != NULL) { dns_glue_t *next = glue->next; @@ -4895,42 +4902,33 @@ freeglue(isc_mem_t *mctx, dns_glue_t *glue) { dns_rdataset_invalidate(&glue->rdataset_aaaa); dns_rdataset_invalidate(&glue->sigrdataset_aaaa); - isc_mem_put(mctx, glue, sizeof(*glue)); + isc_mem_putanddetach(&glue->mctx, glue, sizeof(*glue)); glue = next; } } -void -dns__rbtdb_free_gluenode_rcu(struct rcu_head *rcu_head) { - dns_gluenode_t *gluenode = caa_container_of(rcu_head, dns_gluenode_t, - rcu_head); +static void +free_gluelist_rcu(struct rcu_head *rcu_head) { + dns_glue_t *glue = caa_container_of(rcu_head, dns_glue_t, rcu_head); - freeglue(gluenode->mctx, gluenode->glue); - - dns_db_detachnode(gluenode->db, (dns_dbnode_t **)&gluenode->node); - - isc_mem_putanddetach(&gluenode->mctx, gluenode, sizeof(*gluenode)); -} - -void -dns__rbtdb_free_gluenode(dns_gluenode_t *gluenode) { - call_rcu(&gluenode->rcu_head, dns__rbtdb_free_gluenode_rcu); + dns__rbtdb_freeglue(glue); } static void -free_gluetable(struct cds_lfht *glue_table) { - struct cds_lfht_iter iter; - dns_gluenode_t *gluenode = NULL; +free_gluetable(dns_rbtdb_version_t *rbtversion) { + struct cds_wfs_head *head = __cds_wfs_pop_all(&rbtversion->glue_stack); + struct cds_wfs_node *node = NULL, *next = NULL; rcu_read_lock(); - cds_lfht_for_each_entry(glue_table, &iter, gluenode, ht_node) { - INSIST(!cds_lfht_del(glue_table, &gluenode->ht_node)); - dns__rbtdb_free_gluenode(gluenode); + cds_wfs_for_each_blocking_safe(head, node, next) { + dns_slabheader_t *header = + caa_container_of(node, dns_slabheader_t, wfs_node); + dns_glue_t *glue = rcu_xchg_pointer(&header->glue_list, NULL); + + call_rcu(&glue->rcu_head, free_gluelist_rcu); } rcu_read_unlock(); - - cds_lfht_destroy(glue_table, NULL); } void @@ -4960,6 +4958,10 @@ dns__rbtdb_deletedata(dns_db_t *db ISC_ATTR_UNUSED, if (header->closest != NULL) { dns_slabheader_freeproof(db->mctx, &header->closest); } + } else { + if (header->glue_list) { + dns__rbtdb_freeglue(header->glue_list); + } } } diff --git a/lib/dns/rbtdb_p.h b/lib/dns/rbtdb_p.h index 6552ee9143..b28863eac7 100644 --- a/lib/dns/rbtdb_p.h +++ b/lib/dns/rbtdb_p.h @@ -15,7 +15,6 @@ #include #include -#include #include #include @@ -91,7 +90,7 @@ struct dns_rbtdb_version { uint64_t records; uint64_t xfrsize; - struct cds_lfht *glue_table; + struct cds_wfs_stack glue_stack; }; typedef ISC_LIST(dns_rbtdb_version_t) rbtdb_versionlist_t; @@ -214,18 +213,6 @@ typedef struct { extern dns_dbmethods_t dns__rbtdb_zonemethods; extern dns_dbmethods_t dns__rbtdb_cachemethods; -typedef struct dns_gluenode_t { - isc_mem_t *mctx; - - struct dns_glue *glue; - - dns_db_t *db; - dns_rbtnode_t *node; - - struct cds_lfht_node ht_node; - struct rcu_head rcu_head; -} dns_gluenode_t; - /* * Common DB implementation methods shared by both cache and zone RBT * databases: @@ -383,9 +370,7 @@ isc_result_t dns__rbtdb_nodefullname(dns_db_t *db, dns_dbnode_t *node, dns_name_t *name); void -dns__rbtdb_free_gluenode_rcu(struct rcu_head *rcu_head); -void -dns__rbtdb_free_gluenode(dns_gluenode_t *gluenode); +dns__rbtdb_freeglue(dns_glue_t *glue_list); void dns__rbtdb_newref(dns_rbtdb_t *rbtdb, dns_rbtnode_t *node, diff --git a/lib/dns/rdataslab.c b/lib/dns/rdataslab.c index 3da416e2ef..29fc2debba 100644 --- a/lib/dns/rdataslab.c +++ b/lib/dns/rdataslab.c @@ -1081,12 +1081,15 @@ dns_slabheader_reset(dns_slabheader_t *h, dns_db_t *db, dns_dbnode_t *node) { ISC_LINK_INIT(h, link); h->heap_index = 0; h->heap = NULL; + h->glue_list = NULL; h->db = db; h->node = node; atomic_init(&h->attributes, 0); atomic_init(&h->last_refresh_fail_ts, 0); + cds_wfs_node_init(&h->wfs_node); + STATIC_ASSERT((sizeof(h->attributes) == 2), "The .attributes field of dns_slabheader_t needs to be " "16-bit int type exactly.");