From dcc4c3e3ec8a3c2a4317ea4e3871a8cccdcac09a Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Tue, 25 Oct 2022 14:38:53 -0700 Subject: [PATCH 1/3] Refactor dns_master_loadfileinc() to use loopmgr instead of tasks Incremental file loads now use loopmgr events instead of task events. The dns_master_loadstreaminc(), _loadbufferinc(), _loadlexer() and _loadlexerinc() functions were not used in BIND, and have been removed. --- lib/dns/include/dns/events.h | 42 +++--- lib/dns/include/dns/master.h | 37 +----- lib/dns/master.c | 250 +++++++---------------------------- lib/dns/zone.c | 26 ++-- 4 files changed, 83 insertions(+), 272 deletions(-) diff --git a/lib/dns/include/dns/events.h b/lib/dns/include/dns/events.h index 7a514af530..19655aa581 100644 --- a/lib/dns/include/dns/events.h +++ b/lib/dns/include/dns/events.h @@ -46,30 +46,30 @@ #define DNS_EVENT_NOTIFYSENDTOADDR (ISC_EVENTCLASS_DNS + 23) #define DNS_EVENT_ZONE (ISC_EVENTCLASS_DNS + 24) #define DNS_EVENT_ZONESTARTXFRIN (ISC_EVENTCLASS_DNS + 25) -#define DNS_EVENT_MASTERQUANTUM (ISC_EVENTCLASS_DNS + 26) -#define DNS_EVENT_CACHEOVERMEM (ISC_EVENTCLASS_DNS + 27) -#define DNS_EVENT_MASTERNEXTZONE (ISC_EVENTCLASS_DNS + 28) -#define DNS_EVENT_IOREADY (ISC_EVENTCLASS_DNS + 29) -#define DNS_EVENT_LOOKUPDONE (ISC_EVENTCLASS_DNS + 30) -#define DNS_EVENT_RBTDEADNODES (ISC_EVENTCLASS_DNS + 31) -#define DNS_EVENT_DISPATCHCONTROL (ISC_EVENTCLASS_DNS + 32) -#define DNS_EVENT_REQUESTCONTROL (ISC_EVENTCLASS_DNS + 33) -#define DNS_EVENT_DUMPQUANTUM (ISC_EVENTCLASS_DNS + 34) +/* #define DNS_EVENT_MASTERQUANTUM (ISC_EVENTCLASS_DNS + 26) */ +#define DNS_EVENT_CACHEOVERMEM (ISC_EVENTCLASS_DNS + 27) +#define DNS_EVENT_MASTERNEXTZONE (ISC_EVENTCLASS_DNS + 28) +#define DNS_EVENT_IOREADY (ISC_EVENTCLASS_DNS + 29) +#define DNS_EVENT_LOOKUPDONE (ISC_EVENTCLASS_DNS + 30) +#define DNS_EVENT_RBTDEADNODES (ISC_EVENTCLASS_DNS + 31) +#define DNS_EVENT_DISPATCHCONTROL (ISC_EVENTCLASS_DNS + 32) +#define DNS_EVENT_REQUESTCONTROL (ISC_EVENTCLASS_DNS + 33) +#define DNS_EVENT_DUMPQUANTUM (ISC_EVENTCLASS_DNS + 34) /* #define DNS_EVENT_IMPORTRECVDONE (ISC_EVENTCLASS_DNS + 35) */ #define DNS_EVENT_FREESTORAGE (ISC_EVENTCLASS_DNS + 36) /* #define DNS_EVENT_VIEWACACHESHUTDOWN (ISC_EVENTCLASS_DNS + 37) */ -#define DNS_EVENT_ACACHECONTROL (ISC_EVENTCLASS_DNS + 38) -#define DNS_EVENT_ACACHECLEAN (ISC_EVENTCLASS_DNS + 39) -#define DNS_EVENT_ACACHEOVERMEM (ISC_EVENTCLASS_DNS + 40) -#define DNS_EVENT_RBTPRUNE (ISC_EVENTCLASS_DNS + 41) -#define DNS_EVENT_MANAGEKEYS (ISC_EVENTCLASS_DNS + 42) -#define DNS_EVENT_CLIENTRESDONE (ISC_EVENTCLASS_DNS + 43) -#define DNS_EVENT_CLIENTREQDONE (ISC_EVENTCLASS_DNS + 44) -#define DNS_EVENT_ADBGROWENTRIES (ISC_EVENTCLASS_DNS + 45) -#define DNS_EVENT_ADBGROWNAMES (ISC_EVENTCLASS_DNS + 46) -#define DNS_EVENT_ZONESECURESERIAL (ISC_EVENTCLASS_DNS + 47) -#define DNS_EVENT_ZONESECUREDB (ISC_EVENTCLASS_DNS + 48) -#define DNS_EVENT_ZONELOAD (ISC_EVENTCLASS_DNS + 49) +#define DNS_EVENT_ACACHECONTROL (ISC_EVENTCLASS_DNS + 38) +#define DNS_EVENT_ACACHECLEAN (ISC_EVENTCLASS_DNS + 39) +#define DNS_EVENT_ACACHEOVERMEM (ISC_EVENTCLASS_DNS + 40) +#define DNS_EVENT_RBTPRUNE (ISC_EVENTCLASS_DNS + 41) +#define DNS_EVENT_MANAGEKEYS (ISC_EVENTCLASS_DNS + 42) +#define DNS_EVENT_CLIENTRESDONE (ISC_EVENTCLASS_DNS + 43) +#define DNS_EVENT_CLIENTREQDONE (ISC_EVENTCLASS_DNS + 44) +#define DNS_EVENT_ADBGROWENTRIES (ISC_EVENTCLASS_DNS + 45) +#define DNS_EVENT_ADBGROWNAMES (ISC_EVENTCLASS_DNS + 46) +#define DNS_EVENT_ZONESECURESERIAL (ISC_EVENTCLASS_DNS + 47) +#define DNS_EVENT_ZONESECUREDB (ISC_EVENTCLASS_DNS + 48) +/* #define DNS_EVENT_ZONELOAD (ISC_EVENTCLASS_DNS + 49) */ #define DNS_EVENT_KEYDONE (ISC_EVENTCLASS_DNS + 50) #define DNS_EVENT_SETNSEC3PARAM (ISC_EVENTCLASS_DNS + 51) #define DNS_EVENT_SETSERIAL (ISC_EVENTCLASS_DNS + 52) diff --git a/lib/dns/include/dns/master.h b/lib/dns/include/dns/master.h index 2edb12a308..175ba5f23a 100644 --- a/lib/dns/include/dns/master.h +++ b/lib/dns/include/dns/master.h @@ -135,45 +135,19 @@ dns_master_loadbuffer(isc_buffer_t *buffer, dns_name_t *top, dns_name_t *origin, dns_rdataclass_t zclass, unsigned int options, dns_rdatacallbacks_t *callbacks, isc_mem_t *mctx); -isc_result_t -dns_master_loadlexer(isc_lex_t *lex, dns_name_t *top, dns_name_t *origin, - dns_rdataclass_t zclass, unsigned int options, - dns_rdatacallbacks_t *callbacks, isc_mem_t *mctx); - isc_result_t dns_master_loadfileinc(const char *master_file, dns_name_t *top, dns_name_t *origin, dns_rdataclass_t zclass, unsigned int options, uint32_t resign, - dns_rdatacallbacks_t *callbacks, isc_task_t *task, + dns_rdatacallbacks_t *callbacks, isc_loop_t *loop, dns_loaddonefunc_t done, void *done_arg, dns_loadctx_t **ctxp, dns_masterincludecb_t include_cb, void *include_arg, isc_mem_t *mctx, dns_masterformat_t format, uint32_t maxttl); -isc_result_t -dns_master_loadstreaminc(FILE *stream, dns_name_t *top, dns_name_t *origin, - dns_rdataclass_t zclass, unsigned int options, - dns_rdatacallbacks_t *callbacks, isc_task_t *task, - dns_loaddonefunc_t done, void *done_arg, - dns_loadctx_t **ctxp, isc_mem_t *mctx); - -isc_result_t -dns_master_loadbufferinc(isc_buffer_t *buffer, dns_name_t *top, - dns_name_t *origin, dns_rdataclass_t zclass, - unsigned int options, dns_rdatacallbacks_t *callbacks, - isc_task_t *task, dns_loaddonefunc_t done, - void *done_arg, dns_loadctx_t **ctxp, isc_mem_t *mctx); - -isc_result_t -dns_master_loadlexerinc(isc_lex_t *lex, dns_name_t *top, dns_name_t *origin, - dns_rdataclass_t zclass, unsigned int options, - dns_rdatacallbacks_t *callbacks, isc_task_t *task, - dns_loaddonefunc_t done, void *done_arg, - dns_loadctx_t **ctxp, isc_mem_t *mctx); - /*%< - * Loads a RFC1305 master file from a file, stream, buffer, or existing - * lexer into rdatasets and then calls 'callbacks->commit' to commit the + * Loads a RFC1305 master file from a file, stream, or buffer + * into rdatasets and then calls 'callbacks->commit' to commit the * rdatasets. Rdata memory belongs to dns_master_load and will be * reused / released when the callback completes. dns_load_master will * abort if callbacks->commit returns any value other than ISC_R_SUCCESS. @@ -193,14 +167,13 @@ dns_master_loadlexerinc(isc_lex_t *lex, dns_name_t *top, dns_name_t *origin, * * Requires: *\li 'master_file' points to a valid string. - *\li 'lexer' points to a valid lexer. *\li 'top' points to a valid name. *\li 'origin' points to a valid name. *\li 'callbacks->commit' points to a valid function. *\li 'callbacks->error' points to a valid function. *\li 'callbacks->warn' points to a valid function. *\li 'mctx' points to a valid memory context. - *\li 'task' and 'done' to be valid. + *\li 'loop' and 'done' to be valid. *\li 'lmgr' to be valid. *\li 'ctxp != NULL && ctxp == NULL'. * @@ -215,7 +188,7 @@ dns_master_loadlexerinc(isc_lex_t *lex, dns_name_t *top, dns_name_t *origin, *\li DNS_R_NOOWNER failed to specify a ownername. *\li DNS_R_NOTTL failed to specify a ttl. *\li DNS_R_BADCLASS record class did not match zone class. - *\li DNS_R_CONTINUE load still in progress (dns_master_load*inc() only). + *\li DNS_R_CONTINUE load still in progress (dns_master_loadfileinc() only). *\li Any dns_rdata_fromtext() error code. *\li Any error code from callbacks->commit(). */ diff --git a/lib/dns/master.c b/lib/dns/master.c index 8ff0de68a3..9f577a3293 100644 --- a/lib/dns/master.c +++ b/lib/dns/master.c @@ -16,9 +16,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -28,7 +30,6 @@ #include #include #include -#include #include #include @@ -104,7 +105,7 @@ struct dns_loadctx { dns_masterformat_t format; dns_rdatacallbacks_t *callbacks; - isc_task_t *task; + isc_loop_t *loop; dns_loaddonefunc_t done; void *done_arg; @@ -206,10 +207,7 @@ grow_rdata(int, dns_rdata_t *, int, rdatalist_head_t *, rdatalist_head_t *, isc_mem_t *); static void -load_quantum(isc_task_t *task, isc_event_t *event); - -static isc_result_t -task_send(dns_loadctx_t *lctx); +load_quantum(void *arg); static void loadctx_destroy(dns_loadctx_t *lctx); @@ -455,8 +453,8 @@ loadctx_destroy(dns_loadctx_t *lctx) { isc_lex_destroy(&lctx->lex); } - if (lctx->task != NULL) { - isc_task_detach(&lctx->task); + if (lctx->loop != NULL) { + isc_loop_detach(&lctx->loop); } isc_mem_putanddetach(&lctx->mctx, lctx, sizeof(*lctx)); @@ -499,10 +497,10 @@ static isc_result_t loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, uint32_t resign, dns_name_t *top, dns_rdataclass_t zclass, dns_name_t *origin, dns_rdatacallbacks_t *callbacks, - isc_task_t *task, dns_loaddonefunc_t done, void *done_arg, + isc_loop_t *loop, dns_loaddonefunc_t done, void *done_arg, dns_masterincludecb_t include_cb, void *include_arg, isc_lex_t *lex, dns_loadctx_t **lctxp) { - dns_loadctx_t *lctx; + dns_loadctx_t *lctx = NULL; isc_result_t result; isc_region_t r; isc_lexspecials_t specials; @@ -515,20 +513,35 @@ loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, REQUIRE(mctx != NULL); REQUIRE(dns_name_isabsolute(top)); REQUIRE(dns_name_isabsolute(origin)); - REQUIRE((task == NULL && done == NULL) || - (task != NULL && done != NULL)); + REQUIRE((loop == NULL && done == NULL) || + (loop != NULL && done != NULL)); lctx = isc_mem_get(mctx, sizeof(*lctx)); + *lctx = (dns_loadctx_t){ + .format = format, + .ttl_known = ((options & DNS_MASTER_NOTTL) != 0), + .default_ttl_known = ((options & DNS_MASTER_NOTTL) != 0), + .warn_1035 = true, + .warn_tcr = true, + .warn_sigexpired = true, + .options = options, + .zclass = zclass, + .resign = resign, + .include_cb = include_cb, + .include_arg = include_arg, + .first = true, + .done = done, + .callbacks = callbacks, + .done_arg = done_arg, + .loop_cnt = (done != NULL) ? 100 : 0, + + }; - lctx->inc = NULL; result = incctx_create(mctx, origin, &lctx->inc); if (result != ISC_R_SUCCESS) { goto cleanup_ctx; } - lctx->maxttl = 0; - - lctx->format = format; switch (format) { case dns_masterformat_text: lctx->openfile = openfile_text; @@ -562,43 +575,20 @@ loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, isc_lex_setcomments(lctx->lex, ISC_LEXCOMMENT_DNSMASTERFILE); } - lctx->ttl_known = ((options & DNS_MASTER_NOTTL) != 0); - lctx->ttl = 0; - lctx->default_ttl_known = lctx->ttl_known; - lctx->default_ttl = 0; - lctx->warn_1035 = true; /* XXX Argument? */ - lctx->warn_tcr = true; /* XXX Argument? */ - lctx->warn_sigexpired = true; /* XXX Argument? */ - lctx->options = options; - lctx->seen_include = false; - lctx->zclass = zclass; - lctx->resign = resign; - lctx->result = ISC_R_SUCCESS; - lctx->include_cb = include_cb; - lctx->include_arg = include_arg; isc_stdtime_get(&lctx->now); lctx->top = dns_fixedname_initname(&lctx->fixed_top); dns_name_toregion(top, &r); dns_name_fromregion(lctx->top, &r); - lctx->f = NULL; - lctx->first = true; dns_master_initrawheader(&lctx->header); - lctx->loop_cnt = (done != NULL) ? 100 : 0; - lctx->callbacks = callbacks; - lctx->task = NULL; - if (task != NULL) { - isc_task_attach(task, &lctx->task); + if (loop != NULL) { + isc_loop_attach(loop, &lctx->loop); } - lctx->done = done; - lctx->done_arg = done_arg; - atomic_init(&lctx->canceled, false); - lctx->mctx = NULL; - isc_mem_attach(mctx, &lctx->mctx); isc_refcount_init(&lctx->references, 1); /* Implicit attach. */ + isc_mem_attach(mctx, &lctx->mctx); lctx->magic = DNS_LCTX_MAGIC; *lctxp = lctx; @@ -2128,7 +2118,7 @@ load_text(dns_loadctx_t *lctx) { } if (!done) { - INSIST(lctx->done != NULL && lctx->task != NULL); + INSIST(lctx->done != NULL && lctx->loop != NULL); result = DNS_R_CONTINUE; } else if (result == ISC_R_SUCCESS && lctx->result != ISC_R_SUCCESS) { result = lctx->result; @@ -2623,7 +2613,7 @@ load_raw(dns_loadctx_t *lctx) { } if (!done) { - INSIST(lctx->done != NULL && lctx->task != NULL); + INSIST(lctx->done != NULL && lctx->loop != NULL); result = DNS_R_CONTINUE; } else if (result == ISC_R_SUCCESS && lctx->result != ISC_R_SUCCESS) { result = lctx->result; @@ -2685,7 +2675,7 @@ isc_result_t dns_master_loadfileinc(const char *master_file, dns_name_t *top, dns_name_t *origin, dns_rdataclass_t zclass, unsigned int options, uint32_t resign, - dns_rdatacallbacks_t *callbacks, isc_task_t *task, + dns_rdatacallbacks_t *callbacks, isc_loop_t *loop, dns_loaddonefunc_t done, void *done_arg, dns_loadctx_t **lctxp, dns_masterincludecb_t include_cb, void *include_arg, isc_mem_t *mctx, @@ -2693,11 +2683,11 @@ dns_master_loadfileinc(const char *master_file, dns_name_t *top, dns_loadctx_t *lctx = NULL; isc_result_t result; - REQUIRE(task != NULL); + REQUIRE(loop != NULL); REQUIRE(done != NULL); result = loadctx_create(format, mctx, options, resign, top, zclass, - origin, callbacks, task, done, done_arg, + origin, callbacks, loop, done, done_arg, include_cb, include_arg, NULL, &lctx); if (result != ISC_R_SUCCESS) { return (result); @@ -2707,18 +2697,13 @@ dns_master_loadfileinc(const char *master_file, dns_name_t *top, result = (lctx->openfile)(lctx, master_file); if (result != ISC_R_SUCCESS) { - goto cleanup; + dns_loadctx_detach(&lctx); + return (result); } - result = task_send(lctx); - if (result == ISC_R_SUCCESS) { - dns_loadctx_attach(lctx, lctxp); - return (DNS_R_CONTINUE); - } - -cleanup: - dns_loadctx_detach(&lctx); - return (result); + isc_async_run(loop, load_quantum, lctx); + dns_loadctx_attach(lctx, lctxp); + return (DNS_R_CONTINUE); } isc_result_t @@ -2739,7 +2724,8 @@ dns_master_loadstream(FILE *stream, dns_name_t *top, dns_name_t *origin, result = isc_lex_openstream(lctx->lex, stream); if (result != ISC_R_SUCCESS) { - goto cleanup; + dns_loadctx_detach(&lctx); + return (result); } result = (lctx->load)(lctx); @@ -2752,44 +2738,6 @@ cleanup: return (result); } -isc_result_t -dns_master_loadstreaminc(FILE *stream, dns_name_t *top, dns_name_t *origin, - dns_rdataclass_t zclass, unsigned int options, - dns_rdatacallbacks_t *callbacks, isc_task_t *task, - dns_loaddonefunc_t done, void *done_arg, - dns_loadctx_t **lctxp, isc_mem_t *mctx) { - isc_result_t result; - dns_loadctx_t *lctx = NULL; - - REQUIRE(stream != NULL); - REQUIRE(task != NULL); - REQUIRE(done != NULL); - - result = loadctx_create(dns_masterformat_text, mctx, options, 0, top, - zclass, origin, callbacks, task, done, done_arg, - NULL, NULL, NULL, &lctx); - if (result != ISC_R_SUCCESS) { - goto cleanup; - } - - result = isc_lex_openstream(lctx->lex, stream); - if (result != ISC_R_SUCCESS) { - goto cleanup; - } - - result = task_send(lctx); - if (result == ISC_R_SUCCESS) { - dns_loadctx_attach(lctx, lctxp); - return (DNS_R_CONTINUE); - } - -cleanup: - if (lctx != NULL) { - dns_loadctx_detach(&lctx); - } - return (result); -} - isc_result_t dns_master_loadbuffer(isc_buffer_t *buffer, dns_name_t *top, dns_name_t *origin, dns_rdataclass_t zclass, unsigned int options, @@ -2819,96 +2767,6 @@ cleanup: return (result); } -isc_result_t -dns_master_loadbufferinc(isc_buffer_t *buffer, dns_name_t *top, - dns_name_t *origin, dns_rdataclass_t zclass, - unsigned int options, dns_rdatacallbacks_t *callbacks, - isc_task_t *task, dns_loaddonefunc_t done, - void *done_arg, dns_loadctx_t **lctxp, - isc_mem_t *mctx) { - isc_result_t result; - dns_loadctx_t *lctx = NULL; - - REQUIRE(buffer != NULL); - REQUIRE(task != NULL); - REQUIRE(done != NULL); - - result = loadctx_create(dns_masterformat_text, mctx, options, 0, top, - zclass, origin, callbacks, task, done, done_arg, - NULL, NULL, NULL, &lctx); - if (result != ISC_R_SUCCESS) { - return (result); - } - - result = isc_lex_openbuffer(lctx->lex, buffer); - if (result != ISC_R_SUCCESS) { - goto cleanup; - } - - result = task_send(lctx); - if (result == ISC_R_SUCCESS) { - dns_loadctx_attach(lctx, lctxp); - return (DNS_R_CONTINUE); - } - -cleanup: - dns_loadctx_detach(&lctx); - return (result); -} - -isc_result_t -dns_master_loadlexer(isc_lex_t *lex, dns_name_t *top, dns_name_t *origin, - dns_rdataclass_t zclass, unsigned int options, - dns_rdatacallbacks_t *callbacks, isc_mem_t *mctx) { - isc_result_t result; - dns_loadctx_t *lctx = NULL; - - REQUIRE(lex != NULL); - - result = loadctx_create(dns_masterformat_text, mctx, options, 0, top, - zclass, origin, callbacks, NULL, NULL, NULL, - NULL, NULL, lex, &lctx); - if (result != ISC_R_SUCCESS) { - return (result); - } - - result = (lctx->load)(lctx); - INSIST(result != DNS_R_CONTINUE); - - dns_loadctx_detach(&lctx); - return (result); -} - -isc_result_t -dns_master_loadlexerinc(isc_lex_t *lex, dns_name_t *top, dns_name_t *origin, - dns_rdataclass_t zclass, unsigned int options, - dns_rdatacallbacks_t *callbacks, isc_task_t *task, - dns_loaddonefunc_t done, void *done_arg, - dns_loadctx_t **lctxp, isc_mem_t *mctx) { - isc_result_t result; - dns_loadctx_t *lctx = NULL; - - REQUIRE(lex != NULL); - REQUIRE(task != NULL); - REQUIRE(done != NULL); - - result = loadctx_create(dns_masterformat_text, mctx, options, 0, top, - zclass, origin, callbacks, task, done, done_arg, - NULL, NULL, lex, &lctx); - if (result != ISC_R_SUCCESS) { - return (result); - } - - result = task_send(lctx); - if (result == ISC_R_SUCCESS) { - dns_loadctx_attach(lctx, lctxp); - return (DNS_R_CONTINUE); - } - - dns_loadctx_detach(&lctx); - return (result); -} - /* * Grow the slab of dns_rdatalist_t structures. * Re-link glue and current list. @@ -3154,12 +3012,10 @@ is_glue(rdatalist_head_t *head, dns_name_t *owner) { } static void -load_quantum(isc_task_t *task, isc_event_t *event) { +load_quantum(void *arg) { isc_result_t result; - dns_loadctx_t *lctx; + dns_loadctx_t *lctx = (dns_loadctx_t *)arg; - REQUIRE(event != NULL); - lctx = event->ev_arg; REQUIRE(DNS_LCTX_VALID(lctx)); if (atomic_load_acquire(&lctx->canceled)) { @@ -3168,25 +3024,13 @@ load_quantum(isc_task_t *task, isc_event_t *event) { result = (lctx->load)(lctx); } if (result == DNS_R_CONTINUE) { - event->ev_arg = lctx; - isc_task_send(task, &event); + isc_async_run(lctx->loop, load_quantum, lctx); } else { (lctx->done)(lctx->done_arg, result); - isc_event_free(&event); dns_loadctx_detach(&lctx); } } -static isc_result_t -task_send(dns_loadctx_t *lctx) { - isc_event_t *event; - - event = isc_event_allocate(lctx->mctx, NULL, DNS_EVENT_MASTERQUANTUM, - load_quantum, lctx, sizeof(*event)); - isc_task_send(lctx->task, &event); - return (ISC_R_SUCCESS); -} - void dns_loadctx_cancel(dns_loadctx_t *lctx) { REQUIRE(DNS_LCTX_VALID(lctx)); diff --git a/lib/dns/zone.c b/lib/dns/zone.c index 1970f0ac1f..fa6bdb0e7c 100644 --- a/lib/dns/zone.c +++ b/lib/dns/zone.c @@ -2415,17 +2415,13 @@ dns_zone_load(dns_zone_t *zone, bool newonly) { } static void -zone_asyncload(isc_task_t *task, isc_event_t *event) { - dns_asyncload_t *asl = event->ev_arg; +zone_asyncload(void *arg) { + dns_asyncload_t *asl = arg; dns_zone_t *zone = asl->zone; isc_result_t result; - UNUSED(task); - REQUIRE(DNS_ZONE_VALID(zone)); - isc_event_free(&event); - LOCK_ZONE(zone); result = zone_load(zone, asl->flags, true); if (result != DNS_R_CONTINUE) { @@ -2435,7 +2431,7 @@ zone_asyncload(isc_task_t *task, isc_event_t *event) { /* Inform the zone table we've finished loading */ if (asl->loaded != NULL) { - (asl->loaded)(asl->loaded_arg, zone, task); + (asl->loaded)(asl->loaded_arg, zone, zone->task); } isc_mem_put(zone->mctx, asl, sizeof(*asl)); @@ -2445,7 +2441,6 @@ zone_asyncload(isc_task_t *task, isc_event_t *event) { isc_result_t dns_zone_asyncload(dns_zone_t *zone, bool newonly, dns_zt_zoneloaded_t done, void *arg) { - isc_event_t *e; dns_asyncload_t *asl = NULL; REQUIRE(DNS_ZONE_VALID(zone)); @@ -2468,12 +2463,9 @@ dns_zone_asyncload(dns_zone_t *zone, bool newonly, dns_zt_zoneloaded_t done, asl->loaded = done; asl->loaded_arg = arg; - e = isc_event_allocate(zone->zmgr->mctx, zone->zmgr, DNS_EVENT_ZONELOAD, - zone_asyncload, asl, sizeof(isc_event_t)); - zone_iattach(zone, &asl->zone); DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_LOADPENDING); - isc_task_send(zone->loadtask, &e); + isc_async_run(zone->loop, zone_asyncload, asl); UNLOCK_ZONE(zone); return (ISC_R_SUCCESS); @@ -2612,6 +2604,8 @@ zone_gotreadhandle(isc_task_t *task, isc_event_t *event) { REQUIRE(DNS_LOAD_VALID(load)); + UNUSED(task); + if ((event->ev_attributes & ISC_EVENTATTR_CANCELED) != 0) { result = ISC_R_CANCELED; } @@ -2625,9 +2619,9 @@ zone_gotreadhandle(isc_task_t *task, isc_event_t *event) { result = dns_master_loadfileinc( load->zone->masterfile, dns_db_origin(load->db), dns_db_origin(load->db), load->zone->rdclass, options, 0, - &load->callbacks, task, zone_loaddone, load, &load->zone->lctx, - zone_registerinclude, load->zone, load->zone->mctx, - load->zone->masterformat, load->zone->maxttl); + &load->callbacks, load->zone->loop, zone_loaddone, load, + &load->zone->lctx, zone_registerinclude, load->zone, + load->zone->mctx, load->zone->masterformat, load->zone->maxttl); if (result != ISC_R_SUCCESS && result != DNS_R_CONTINUE && result != DNS_R_SEENINCLUDE) { @@ -2762,7 +2756,7 @@ zone_startload(dns_db_t *db, dns_zone_t *zone, isc_time_t loadtime) { options |= DNS_MASTER_MANYERRORS; } - if (zone->zmgr != NULL && zone->db != NULL && zone->loadtask != NULL) { + if (zone->zmgr != NULL && zone->db != NULL) { load = isc_mem_get(zone->mctx, sizeof(*load)); load->mctx = NULL; From 77aeed6231946fd30e62812b9f5c68d343af8d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Wed, 26 Oct 2022 12:40:43 +0200 Subject: [PATCH 2/3] Move the zone loading to the offloaded threads Instead of doing incremental zone loading with fixed quantum - 100 loaded lines per event, move the zone loading process to the offloaded libuv threads using isc_work_enqueue() API. This has the advantage that the thread scheduling is given back to the operating system that understands blocking operations, and the zone loading operation doesn't block the networking threads directly. --- lib/dns/include/dns/master.h | 19 ++-- lib/dns/master.c | 192 ++++++++++++----------------------- lib/dns/zone.c | 8 +- 3 files changed, 78 insertions(+), 141 deletions(-) diff --git a/lib/dns/include/dns/master.h b/lib/dns/include/dns/master.h index 175ba5f23a..2e9966f8d0 100644 --- a/lib/dns/include/dns/master.h +++ b/lib/dns/include/dns/master.h @@ -136,17 +136,17 @@ dns_master_loadbuffer(isc_buffer_t *buffer, dns_name_t *top, dns_name_t *origin, dns_rdatacallbacks_t *callbacks, isc_mem_t *mctx); isc_result_t -dns_master_loadfileinc(const char *master_file, dns_name_t *top, - dns_name_t *origin, dns_rdataclass_t zclass, - unsigned int options, uint32_t resign, - dns_rdatacallbacks_t *callbacks, isc_loop_t *loop, - dns_loaddonefunc_t done, void *done_arg, - dns_loadctx_t **ctxp, dns_masterincludecb_t include_cb, - void *include_arg, isc_mem_t *mctx, - dns_masterformat_t format, uint32_t maxttl); +dns_master_loadfileasync(const char *master_file, dns_name_t *top, + dns_name_t *origin, dns_rdataclass_t zclass, + unsigned int options, uint32_t resign, + dns_rdatacallbacks_t *callbacks, isc_loop_t *loop, + dns_loaddonefunc_t done, void *done_arg, + dns_loadctx_t **ctxp, dns_masterincludecb_t include_cb, + void *include_arg, isc_mem_t *mctx, + dns_masterformat_t format, uint32_t maxttl); /*%< - * Loads a RFC1305 master file from a file, stream, or buffer + * Loads a RFC1035 master file from a file, stream, or buffer * into rdatasets and then calls 'callbacks->commit' to commit the * rdatasets. Rdata memory belongs to dns_master_load and will be * reused / released when the callback completes. dns_load_master will @@ -188,7 +188,6 @@ dns_master_loadfileinc(const char *master_file, dns_name_t *top, *\li DNS_R_NOOWNER failed to specify a ownername. *\li DNS_R_NOTTL failed to specify a ttl. *\li DNS_R_BADCLASS record class did not match zone class. - *\li DNS_R_CONTINUE load still in progress (dns_master_loadfileinc() only). *\li Any dns_rdata_fromtext() error code. *\li Any error code from callbacks->commit(). */ diff --git a/lib/dns/master.c b/lib/dns/master.c index 9f577a3293..b28ff98e82 100644 --- a/lib/dns/master.c +++ b/lib/dns/master.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -105,7 +106,6 @@ struct dns_loadctx { dns_masterformat_t format; dns_rdatacallbacks_t *callbacks; - isc_loop_t *loop; dns_loaddonefunc_t done; void *done_arg; @@ -138,8 +138,6 @@ struct dns_loadctx { dns_masterrawheader_t header; /* Which fixed buffers we are using? */ - unsigned int loop_cnt; /*% records per quantum, - * 0 => all. */ isc_result_t result; /* Atomic */ @@ -206,9 +204,6 @@ static dns_rdata_t * grow_rdata(int, dns_rdata_t *, int, rdatalist_head_t *, rdatalist_head_t *, isc_mem_t *); -static void -load_quantum(void *arg); - static void loadctx_destroy(dns_loadctx_t *lctx); @@ -293,11 +288,10 @@ loadctx_destroy(dns_loadctx_t *lctx); ((result != ISC_R_SUCCESS) && (result != ISC_R_IOERROR) && \ ((lctx)->options & DNS_MASTER_MANYERRORS) != 0) -#define SETRESULT(lctx, r) \ - do { \ - if ((lctx)->result == ISC_R_SUCCESS) \ - (lctx)->result = r; \ - } while (0) +#define SETRESULT(lctx, r) \ + if ((lctx)->result == ISC_R_SUCCESS) { \ + (lctx)->result = r; \ + } #define LOGITFILE(result, filename) \ if (result == ISC_R_INVALIDFILE || result == ISC_R_FILENOTFOUND || \ @@ -453,14 +447,10 @@ loadctx_destroy(dns_loadctx_t *lctx) { isc_lex_destroy(&lctx->lex); } - if (lctx->loop != NULL) { - isc_loop_detach(&lctx->loop); - } - isc_mem_putanddetach(&lctx->mctx, lctx, sizeof(*lctx)); } -static isc_result_t +static void incctx_create(isc_mem_t *mctx, dns_name_t *origin, dns_incctx_t **ictxp) { dns_incctx_t *ictx; isc_region_t r; @@ -490,20 +480,17 @@ incctx_create(isc_mem_t *mctx, dns_name_t *origin, dns_incctx_t **ictxp) { ictx->origin_changed = true; *ictxp = ictx; - return (ISC_R_SUCCESS); } -static isc_result_t +static void loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, uint32_t resign, dns_name_t *top, dns_rdataclass_t zclass, dns_name_t *origin, dns_rdatacallbacks_t *callbacks, - isc_loop_t *loop, dns_loaddonefunc_t done, void *done_arg, + dns_loaddonefunc_t done, void *done_arg, dns_masterincludecb_t include_cb, void *include_arg, isc_lex_t *lex, dns_loadctx_t **lctxp) { dns_loadctx_t *lctx = NULL; - isc_result_t result; isc_region_t r; - isc_lexspecials_t specials; REQUIRE(lctxp != NULL && *lctxp == NULL); REQUIRE(callbacks != NULL); @@ -513,8 +500,6 @@ loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, REQUIRE(mctx != NULL); REQUIRE(dns_name_isabsolute(top)); REQUIRE(dns_name_isabsolute(origin)); - REQUIRE((loop == NULL && done == NULL) || - (loop != NULL && done != NULL)); lctx = isc_mem_get(mctx, sizeof(*lctx)); *lctx = (dns_loadctx_t){ @@ -533,14 +518,9 @@ loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, .done = done, .callbacks = callbacks, .done_arg = done_arg, - .loop_cnt = (done != NULL) ? 100 : 0, - }; - result = incctx_create(mctx, origin, &lctx->inc); - if (result != ISC_R_SUCCESS) { - goto cleanup_ctx; - } + incctx_create(mctx, origin, &lctx->inc); switch (format) { case dns_masterformat_text: @@ -559,6 +539,7 @@ loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, lctx->lex = lex; lctx->keep_lex = true; } else { + isc_lexspecials_t specials; lctx->lex = NULL; isc_lex_create(mctx, TOKENSIZ, &lctx->lex); lctx->keep_lex = false; @@ -583,20 +564,12 @@ loadctx_create(dns_masterformat_t format, isc_mem_t *mctx, unsigned int options, dns_master_initrawheader(&lctx->header); - if (loop != NULL) { - isc_loop_attach(loop, &lctx->loop); - } - isc_refcount_init(&lctx->references, 1); /* Implicit attach. */ isc_mem_attach(mctx, &lctx->mctx); lctx->magic = DNS_LCTX_MAGIC; *lctxp = lctx; - return (ISC_R_SUCCESS); - -cleanup_ctx: - isc_mem_put(mctx, lctx, sizeof(*lctx)); - return (result); + return; } static const char *hex = "0123456789abcdef0123456789ABCDEF"; @@ -1037,7 +1010,6 @@ load_text(dns_loadctx_t *lctx) { uint32_t ttl_offset = 0; dns_name_t *new_name; bool current_has_delegation = false; - bool done = false; bool finish_origin = false; bool finish_include = false; bool read_till_eol = false; @@ -1065,7 +1037,6 @@ load_text(dns_loadctx_t *lctx) { unsigned char *target_mem = NULL; int target_size = TSIZ; int new_in_use; - unsigned int loop_cnt = 0; isc_mem_t *mctx; dns_rdatacallbacks_t *callbacks; dns_incctx_t *ictx; @@ -1109,7 +1080,12 @@ load_text(dns_loadctx_t *lctx) { options |= DNS_RDATA_CHECKMXFAIL; } source = isc_lex_getsourcename(lctx->lex); - do { + while (true) { + if (atomic_load_acquire(&lctx->canceled)) { + result = ISC_R_CANCELED; + goto log_and_cleanup; + } + initialws = false; line = isc_lex_getsourceline(lctx->lex); GETTOKEN(lctx->lex, ISC_LEXOPT_INITIALWS | ISC_LEXOPT_QSTRING, @@ -1134,8 +1110,7 @@ load_text(dns_loadctx_t *lctx) { ictx = lctx->inc; continue; } - done = true; - continue; + break; } if (token.type == isc_tokentype_eol) { @@ -1688,7 +1663,7 @@ load_text(dns_loadctx_t *lctx) { SETRESULT(lctx, result); read_till_eol = true; continue; - } else if (result != ISC_R_SUCCESS) { + } else { goto insist_and_cleanup; } } @@ -2097,7 +2072,7 @@ load_text(dns_loadctx_t *lctx) { COMMITALL; } next_line:; - } while (!done && (lctx->loop_cnt == 0 || loop_cnt++ < lctx->loop_cnt)); + } /* * Commit what has not yet been committed. @@ -2115,16 +2090,12 @@ load_text(dns_loadctx_t *lctx) { SETRESULT(lctx, result); } else if (result != ISC_R_SUCCESS) { goto insist_and_cleanup; - } - - if (!done) { - INSIST(lctx->done != NULL && lctx->loop != NULL); - result = DNS_R_CONTINUE; - } else if (result == ISC_R_SUCCESS && lctx->result != ISC_R_SUCCESS) { + } else if (lctx->result != ISC_R_SUCCESS) { result = lctx->result; - } else if (result == ISC_R_SUCCESS && lctx->seen_include) { + } else if (lctx->seen_include) { result = DNS_R_SEENINCLUDE; } + goto cleanup; log_and_cleanup: @@ -2165,6 +2136,7 @@ cleanup: if (rhs != NULL) { isc_mem_free(mctx, rhs); } + return (result); } @@ -2181,10 +2153,7 @@ pushfile(const char *master_file, dns_name_t *origin, dns_loadctx_t *lctx) { ictx = lctx->inc; lctx->seen_include = true; - result = incctx_create(lctx->mctx, origin, &newctx); - if (result != ISC_R_SUCCESS) { - return (result); - } + incctx_create(lctx->mctx, origin, &newctx); /* * Push origin_changed. @@ -2341,8 +2310,6 @@ openfile_raw(dns_loadctx_t *lctx, const char *master_file) { static isc_result_t load_raw(dns_loadctx_t *lctx) { isc_result_t result = ISC_R_SUCCESS; - bool done = false; - unsigned int loop_cnt = 0; dns_rdatacallbacks_t *callbacks; unsigned char namebuf[DNS_NAME_MAXWIRE]; dns_fixedname_t fixed; @@ -2387,9 +2354,7 @@ load_raw(dns_loadctx_t *lctx) { * in this format, and so trying to continue parsing erroneous data * does not really make sense. */ - for (loop_cnt = 0; (lctx->loop_cnt == 0 || loop_cnt < lctx->loop_cnt); - loop_cnt++) - { + while (true) { unsigned int i, rdcount; uint16_t namelen; uint32_t totallen; @@ -2403,7 +2368,6 @@ load_raw(dns_loadctx_t *lctx) { lctx->f, NULL); if (result == ISC_R_EOF) { result = ISC_R_SUCCESS; - done = true; break; } if (result != ISC_R_SUCCESS) { @@ -2612,10 +2576,7 @@ load_raw(dns_loadctx_t *lctx) { } } - if (!done) { - INSIST(lctx->done != NULL && lctx->loop != NULL); - result = DNS_R_CONTINUE; - } else if (result == ISC_R_SUCCESS && lctx->result != ISC_R_SUCCESS) { + if (result == ISC_R_SUCCESS && lctx->result != ISC_R_SUCCESS) { result = lctx->result; } @@ -2630,7 +2591,7 @@ cleanup: if (target_mem != NULL) { isc_mem_put(mctx, target_mem, target_size); } - if (result != ISC_R_SUCCESS && result != DNS_R_CONTINUE) { + if (result != ISC_R_SUCCESS) { (*callbacks->error)(callbacks, "dns_master_load: %s", isc_result_totext(result)); } @@ -2649,12 +2610,9 @@ dns_master_loadfile(const char *master_file, dns_name_t *top, dns_loadctx_t *lctx = NULL; isc_result_t result; - result = loadctx_create(format, mctx, options, resign, top, zclass, - origin, callbacks, NULL, NULL, NULL, include_cb, - include_arg, NULL, &lctx); - if (result != ISC_R_SUCCESS) { - return (result); - } + loadctx_create(format, mctx, options, resign, top, zclass, origin, + callbacks, NULL, NULL, include_cb, include_arg, NULL, + &lctx); lctx->maxttl = maxttl; @@ -2671,27 +2629,39 @@ cleanup: return (result); } +static void +load(void *arg) { + dns_loadctx_t *lctx = arg; + lctx->result = (lctx->load)(lctx); +} + +static void +load_done(void *arg) { + dns_loadctx_t *lctx = arg; + + (lctx->done)(lctx->done_arg, lctx->result); + dns_loadctx_detach(&lctx); +} + isc_result_t -dns_master_loadfileinc(const char *master_file, dns_name_t *top, - dns_name_t *origin, dns_rdataclass_t zclass, - unsigned int options, uint32_t resign, - dns_rdatacallbacks_t *callbacks, isc_loop_t *loop, - dns_loaddonefunc_t done, void *done_arg, - dns_loadctx_t **lctxp, dns_masterincludecb_t include_cb, - void *include_arg, isc_mem_t *mctx, - dns_masterformat_t format, uint32_t maxttl) { +dns_master_loadfileasync(const char *master_file, dns_name_t *top, + dns_name_t *origin, dns_rdataclass_t zclass, + unsigned int options, uint32_t resign, + dns_rdatacallbacks_t *callbacks, isc_loop_t *loop, + dns_loaddonefunc_t done, void *done_arg, + dns_loadctx_t **lctxp, + dns_masterincludecb_t include_cb, void *include_arg, + isc_mem_t *mctx, dns_masterformat_t format, + uint32_t maxttl) { dns_loadctx_t *lctx = NULL; isc_result_t result; REQUIRE(loop != NULL); REQUIRE(done != NULL); - result = loadctx_create(format, mctx, options, resign, top, zclass, - origin, callbacks, loop, done, done_arg, - include_cb, include_arg, NULL, &lctx); - if (result != ISC_R_SUCCESS) { - return (result); - } + loadctx_create(format, mctx, options, resign, top, zclass, origin, + callbacks, done, done_arg, include_cb, include_arg, NULL, + &lctx); lctx->maxttl = maxttl; @@ -2701,9 +2671,10 @@ dns_master_loadfileinc(const char *master_file, dns_name_t *top, return (result); } - isc_async_run(loop, load_quantum, lctx); dns_loadctx_attach(lctx, lctxp); - return (DNS_R_CONTINUE); + isc_work_enqueue(loop, load, load_done, lctx); + + return (ISC_R_SUCCESS); } isc_result_t @@ -2715,26 +2686,19 @@ dns_master_loadstream(FILE *stream, dns_name_t *top, dns_name_t *origin, REQUIRE(stream != NULL); - result = loadctx_create(dns_masterformat_text, mctx, options, 0, top, - zclass, origin, callbacks, NULL, NULL, NULL, - NULL, NULL, NULL, &lctx); - if (result != ISC_R_SUCCESS) { - goto cleanup; - } + loadctx_create(dns_masterformat_text, mctx, options, 0, top, zclass, + origin, callbacks, NULL, NULL, NULL, NULL, NULL, &lctx); result = isc_lex_openstream(lctx->lex, stream); if (result != ISC_R_SUCCESS) { - dns_loadctx_detach(&lctx); - return (result); + goto cleanup; } result = (lctx->load)(lctx); INSIST(result != DNS_R_CONTINUE); cleanup: - if (lctx != NULL) { - dns_loadctx_detach(&lctx); - } + dns_loadctx_detach(&lctx); return (result); } @@ -2747,12 +2711,8 @@ dns_master_loadbuffer(isc_buffer_t *buffer, dns_name_t *top, dns_name_t *origin, REQUIRE(buffer != NULL); - result = loadctx_create(dns_masterformat_text, mctx, options, 0, top, - zclass, origin, callbacks, NULL, NULL, NULL, - NULL, NULL, NULL, &lctx); - if (result != ISC_R_SUCCESS) { - return (result); - } + loadctx_create(dns_masterformat_text, mctx, options, 0, top, zclass, + origin, callbacks, NULL, NULL, NULL, NULL, NULL, &lctx); result = isc_lex_openbuffer(lctx->lex, buffer); if (result != ISC_R_SUCCESS) { @@ -3011,26 +2971,6 @@ is_glue(rdatalist_head_t *head, dns_name_t *owner) { return (false); } -static void -load_quantum(void *arg) { - isc_result_t result; - dns_loadctx_t *lctx = (dns_loadctx_t *)arg; - - REQUIRE(DNS_LCTX_VALID(lctx)); - - if (atomic_load_acquire(&lctx->canceled)) { - result = ISC_R_CANCELED; - } else { - result = (lctx->load)(lctx); - } - if (result == DNS_R_CONTINUE) { - isc_async_run(lctx->loop, load_quantum, lctx); - } else { - (lctx->done)(lctx->done_arg, result); - dns_loadctx_detach(&lctx); - } -} - void dns_loadctx_cancel(dns_loadctx_t *lctx) { REQUIRE(DNS_LCTX_VALID(lctx)); diff --git a/lib/dns/zone.c b/lib/dns/zone.c index fa6bdb0e7c..6880380d46 100644 --- a/lib/dns/zone.c +++ b/lib/dns/zone.c @@ -2616,19 +2616,17 @@ zone_gotreadhandle(isc_task_t *task, isc_event_t *event) { options = get_primary_options(load->zone); - result = dns_master_loadfileinc( + result = dns_master_loadfileasync( load->zone->masterfile, dns_db_origin(load->db), dns_db_origin(load->db), load->zone->rdclass, options, 0, &load->callbacks, load->zone->loop, zone_loaddone, load, &load->zone->lctx, zone_registerinclude, load->zone, load->zone->mctx, load->zone->masterformat, load->zone->maxttl); - if (result != ISC_R_SUCCESS && result != DNS_R_CONTINUE && - result != DNS_R_SEENINCLUDE) - { + if (result != ISC_R_SUCCESS) { goto fail; } - return; + return; fail: zone_loaddone(load, result); } From c59750bfbc488853b6b029805f22e7810cdb6af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= Date: Wed, 26 Oct 2022 13:26:23 +0200 Subject: [PATCH 3/3] Add CHANGES note for [GL #3625] --- CHANGES | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES b/CHANGES index f405acc871..8a86367eee 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +6005. [func] The zone loading has been moved to the offload + threadpool instead of doing incremental repeated + tasks, so zone loading scheduling is now driven + by the operating system scheduler rather than fixed + (100) quantum. [GL #3625] + 6004. [func] Add check-svcb to control the checking of additional constraints on SVBC records. This change impacts on named, named-checkconf, named-checkzone,