diff --git a/CHANGES b/CHANGES index 0196f3d530..da493efd19 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +4613. [func] By default, the maximum size of a zone journal file + is now twice the size of the zone's contents (there + is little benefit to a journal larger than this). + This can be overridden by setting "max-journal-size" + to "unlimited" or to an explicit value up to 2G. + Thanks to Tony Finch. [RT #38324] + 4612. [bug] Silence 'may be use uninitalised' warning and simplify the code in lwres/getaddinfo:process_answer. [RT #45158] diff --git a/bin/named/config.c b/bin/named/config.c index cd5488bc74..771faff5e8 100644 --- a/bin/named/config.c +++ b/bin/named/config.c @@ -233,7 +233,7 @@ options {\n\ sig-signing-type 65534;\n\ inline-signing no;\n\ zone-statistics terse;\n\ - max-journal-size unlimited;\n\ + max-journal-size default;\n\ ixfr-from-differences false;\n\ check-wildcard yes;\n\ check-sibling yes;\n\ diff --git a/bin/named/zoneconf.c b/bin/named/zoneconf.c index 0ac1c4221e..96318ba278 100644 --- a/bin/named/zoneconf.c +++ b/bin/named/zoneconf.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1211,12 +1212,16 @@ ns_zone_configure(const cfg_obj_t *config, const cfg_obj_t *vconfig, dns_zone_setjournalsize(zone, -1); if (cfg_obj_isstring(obj)) { const char *str = cfg_obj_asstring(obj); - INSIST(strcasecmp(str, "unlimited") == 0); - journal_size = ISC_UINT32_MAX / 2; + if (strcasecmp(str, "unlimited") == 0) { + journal_size = DNS_JOURNAL_SIZE_MAX; + } else { + INSIST(strcasecmp(str, "default") == 0); + journal_size = -1; + } } else { isc_resourcevalue_t value; value = cfg_obj_asuint64(obj); - if (value > ISC_UINT32_MAX / 2) { + if (value > DNS_JOURNAL_SIZE_MAX) { cfg_obj_log(obj, ns_g_lctx, ISC_LOG_ERROR, "'max-journal-size " @@ -1331,12 +1336,16 @@ ns_zone_configure(const cfg_obj_t *config, const cfg_obj_t *vconfig, dns_zone_setjournalsize(zone, -1); if (cfg_obj_isstring(obj)) { const char *str = cfg_obj_asstring(obj); - INSIST(strcasecmp(str, "unlimited") == 0); - journal_size = ISC_UINT32_MAX / 2; + if (strcasecmp(str, "unlimited") == 0) { + journal_size = DNS_JOURNAL_SIZE_MAX; + } else { + INSIST(strcasecmp(str, "default") == 0); + journal_size = -1; + } } else { isc_resourcevalue_t value; value = cfg_obj_asuint64(obj); - if (value > ISC_UINT32_MAX / 2) { + if (value > DNS_JOURNAL_SIZE_MAX) { cfg_obj_log(obj, ns_g_lctx, ISC_LOG_ERROR, "'max-journal-size " diff --git a/bin/tests/system/nsupdate/clean.sh b/bin/tests/system/nsupdate/clean.sh index a72ef61bbe..be8d8f3485 100644 --- a/bin/tests/system/nsupdate/clean.sh +++ b/bin/tests/system/nsupdate/clean.sh @@ -19,6 +19,7 @@ rm -f ns*/named.lock rm -f ns1/*.jnl ns2/*.jnl ns3/*.jnl rm -f ns1/example.db ns1/unixtime.db ns1/yyyymmddvv.db ns1/update.db ns1/other.db ns1/keytests.db rm -f ns1/many.test.db +rm -f ns1/maxjournal.db rm -f ns1/md5.key ns1/sha1.key ns1/sha224.key ns1/sha256.key ns1/sha384.key rm -f ns1/sha512.key ns1/ddns.key rm -f ns2/example.bk diff --git a/bin/tests/system/nsupdate/ns1/many.test.db.in b/bin/tests/system/nsupdate/ns1/many.test.db.in index f00461c67e..a4d28d7469 100644 --- a/bin/tests/system/nsupdate/ns1/many.test.db.in +++ b/bin/tests/system/nsupdate/ns1/many.test.db.in @@ -4,14 +4,13 @@ ; License, v. 2.0. If a copy of the MPL was not distributed with this ; file, You can obtain one at http://mozilla.org/MPL/2.0/. -$ORIGIN . $TTL 300 ; 5 minutes -many.test IN SOA ns1.example.nil. hostmaster.example.nil. ( +@ IN SOA ns1.example.nil. hostmaster.example.nil. ( 1 ; serial 2000 ; refresh (2000 seconds) 2000 ; retry (2000 seconds) 1814400 ; expire (3 weeks) 3600 ; minimum (1 hour) ) -many.test NS ns1.example.nil. -many.test NS ns2.example.nil. +@ NS ns1.example.nil. + NS ns2.example.nil. diff --git a/bin/tests/system/nsupdate/ns1/maxjournal.db.in b/bin/tests/system/nsupdate/ns1/maxjournal.db.in new file mode 100644 index 0000000000..9ad1e43ee1 --- /dev/null +++ b/bin/tests/system/nsupdate/ns1/maxjournal.db.in @@ -0,0 +1,15 @@ +; Copyright (C) 2014, 2016 Internet Systems Consortium, Inc. ("ISC") +; +; This Source Code Form is subject to the terms of the Mozilla Public +; License, v. 2.0. If a copy of the MPL was not distributed with this +; file, You can obtain one at http://mozilla.org/MPL/2.0/. + +$TTL 300 ; 5 minutes +@ IN SOA ns1.example.nil. hostmaster.example.nil. ( + 1 ; serial + 2000 ; refresh (2000 seconds) + 2000 ; retry (2000 seconds) + 1814400 ; expire (3 weeks) + 3600 ; minimum (1 hour) + ) +@ NS ns1.example.nil. diff --git a/bin/tests/system/nsupdate/ns1/named.conf b/bin/tests/system/nsupdate/ns1/named.conf index bdb008754f..cb88f44dc2 100644 --- a/bin/tests/system/nsupdate/ns1/named.conf +++ b/bin/tests/system/nsupdate/ns1/named.conf @@ -129,3 +129,10 @@ zone "sample" { allow-update { any; }; file "sample.db"; }; + +zone "maxjournal.test" { + type master; + allow-update { any; }; + file "maxjournal.db"; + max-journal-size default; +}; diff --git a/bin/tests/system/nsupdate/setup.sh b/bin/tests/system/nsupdate/setup.sh index e1139b6034..872d26200a 100644 --- a/bin/tests/system/nsupdate/setup.sh +++ b/bin/tests/system/nsupdate/setup.sh @@ -61,3 +61,6 @@ rm -f ns1/many.test.db.jnl cp ns1/sample.db.in ns1/sample.db cp ns2/sample.db.in ns2/sample.db + +cp -f ns1/maxjournal.db.in ns1/maxjournal.db +rm -f ns1/maxjournal.db.jnl diff --git a/bin/tests/system/nsupdate/tests.sh b/bin/tests/system/nsupdate/tests.sh index 5460bf2f35..7da54e2ce2 100755 --- a/bin/tests/system/nsupdate/tests.sh +++ b/bin/tests/system/nsupdate/tests.sh @@ -6,8 +6,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# $Id: tests.sh,v 1.42 2011/12/16 23:01:17 each Exp $ - SYSTEMTESTTOP=.. . $SYSTEMTESTTOP/conf.sh @@ -583,6 +581,7 @@ if [ $ret -ne 0 ]; then fi n=`expr $n + 1` +ret=0 echo "I:check that yyyymmddvv serial number is correctly generated ($n)" oldserial=`$DIG +short yyyymmddvv.nil. soa @10.53.0.1 -p 5300 | awk '{print $3}'` || ret=1 $NSUPDATE < /dev/null 2>&1 || ret=1 @@ -630,6 +629,46 @@ test ${lines:-0} -eq 64 || ret=1 [ $ret = 0 ] || { echo I:failed; status=1; } fi +n=`expr $n + 1` +echo "I:check max-journal-size limits ($n)" +ret=0 +rm -f nsupdate.out1-$n +# add one record +$NSUPDATE << EOF >> nsupdate.out1-$n 2>&1 +server 10.53.0.1 5300 +zone maxjournal.test +update add z.maxjournal.test 300 IN A 10.20.30.40 +send +EOF +for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do + # repeatedly add and remove the same set of records to fill up + # the journal file without changing the zone content + $NSUPDATE << EOF >> nsupdate.out1-$n 2>&1 +server 10.53.0.1 5300 +zone maxjournal.test +update add a.maxjournal.test 300 IN A 1.2.3.4 +update add b.maxjournal.test 300 IN A 1.2.3.4 +update add c.maxjournal.test 300 IN A 1.2.3.4 +update add d.maxjournal.test 300 IN A 1.2.3.4 +send +update del a.maxjournal.test +update del b.maxjournal.test +update del c.maxjournal.test +update del d.maxjournal.test +send +EOF +done +# check that the journal is big enough to require truncation. +size=`$PERL -e 'use File::stat; my $sb = stat(@ARGV[0]); printf("%s\n", $sb->size);' ns1/maxjournal.db.jnl` +[ "$size" -gt 6000 ] || ret=1 +sleep 1 +$RNDC -c ../common/rndc.conf -s 10.53.0.1 -p 9953 sync maxjournal.test +sleep 1 + +size=`$PERL -e 'use File::stat; my $sb = stat(@ARGV[0]); printf("%s\n", $sb->size);' ns1/maxjournal.db.jnl` +[ "$size" -lt 5000 ] || ret=1 +[ $ret = 0 ] || { echo I:failed; status=1; } + n=`expr $n + 1` echo "I:check check-names processing ($n)" ret=0 diff --git a/doc/arm/Bv9ARM-book.xml b/doc/arm/Bv9ARM-book.xml index 2facfe03e5..1929db1e65 100644 --- a/doc/arm/Bv9ARM-book.xml +++ b/doc/arm/Bv9ARM-book.xml @@ -8492,16 +8492,24 @@ avoid-v6-udp-ports { 40000; range 50000 60000; }; max-journal-size - Sets a maximum size for each journal file - (see ). When the journal file - approaches - the specified size, some of the oldest transactions in the - journal - will be automatically removed. The largest permitted - value is 2 gigabytes. The default is - unlimited, which also - means 2 gigabytes. - This may also be set on a per-zone basis. + Sets a maximum size for each journal file (see + ), expressed in bytes + or, if followed by an optional unit suffix ('k', + 'm', or 'g'), in kilobytes, megabytes, or gigabytes. + When the journal file approaches the specified size, + some of the oldest transactions in the journal + will be automatically removed. The largest + permitted value is 2 gigabytes. Very small + values are rounded up to 4096 bytes. You + can specify unlimited, which + also means 2 gigabytes. If you set the limit to + default or leave it unset, the + journal is allowed to grow up to twice as large as + the zone. (There is little benefit in storing + larger journals.) + + + This option may also be set on a per-zone basis. diff --git a/doc/arm/notes.xml b/doc/arm/notes.xml index d35280abd8..f532068325 100644 --- a/doc/arm/notes.xml +++ b/doc/arm/notes.xml @@ -151,6 +151,16 @@
New Features + + + Setting max-journal-size to + default limits journal sizes to twice the + size of the zone contents. This can be overridden by setting + max-journal-size to unlimited + or to an explicit value up to 2G. Thanks to Tony Finch for + the contribution. [RT #38324] + + The new-zones-directory option allows diff --git a/lib/dns/include/dns/journal.h b/lib/dns/include/dns/journal.h index 3a682aec21..046f968489 100644 --- a/lib/dns/include/dns/journal.h +++ b/lib/dns/include/dns/journal.h @@ -41,6 +41,9 @@ #define DNS_JOURNAL_CREATE 0x00000001 /* ISC_TRUE */ #define DNS_JOURNAL_WRITE 0x00000002 +#define DNS_JOURNAL_SIZE_MAX ISC_INT32_MAX +#define DNS_JOURNAL_SIZE_MIN 4096 + /*** *** Types ***/ @@ -169,6 +172,12 @@ dns_journal_write_transaction(dns_journal_t *j, dns_diff_t *diff); * Reading transactions from journals. */ +isc_boolean_t +dns_journal_empty(dns_journal_t *j); +/*< + * Find out if a journal is empty. + */ + isc_uint32_t dns_journal_first_serial(dns_journal_t *j); isc_uint32_t diff --git a/lib/dns/journal.c b/lib/dns/journal.c index 1210a1ab87..6666c70fcf 100644 --- a/lib/dns/journal.c +++ b/lib/dns/journal.c @@ -6,8 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/* $Id: journal.c,v 1.120 2011/12/22 07:32:41 each Exp $ */ - #include #include @@ -1536,6 +1534,11 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { /* * Miscellaneous accessors. */ +isc_boolean_t +dns_journal_empty(dns_journal_t *j) { + return (JOURNAL_EMPTY(&j->header)); +} + isc_uint32_t dns_journal_first_serial(dns_journal_t *j) { return (j->header.begin.serial); @@ -2139,6 +2142,8 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, isc_uint32_t serial, */ indexend = sizeof(journal_rawheader_t) + j->header.index_size * sizeof(journal_rawpos_t); + if (target_size < DNS_JOURNAL_SIZE_MIN) + target_size = DNS_JOURNAL_SIZE_MIN; if (target_size < indexend * 2) target_size = target_size/2 + indexend; diff --git a/lib/dns/win32/libdns.def.in b/lib/dns/win32/libdns.def.in index ca689240ee..a6e6b8d01d 100644 --- a/lib/dns/win32/libdns.def.in +++ b/lib/dns/win32/libdns.def.in @@ -405,6 +405,7 @@ dns_journal_commit dns_journal_compact dns_journal_current_rr dns_journal_destroy +dns_journal_empty dns_journal_first_rr dns_journal_first_serial dns_journal_get_sourceserial diff --git a/lib/dns/zone.c b/lib/dns/zone.c index d4c54d16a9..0a6c485ba4 100644 --- a/lib/dns/zone.c +++ b/lib/dns/zone.c @@ -4385,11 +4385,13 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, ! DNS_ZONE_OPTION(zone, DNS_ZONEOPT_IXFRFROMDIFFS)) { isc_uint32_t jserial; dns_journal_t *journal = NULL; + isc_boolean_t empty = ISC_FALSE; result = dns_journal_open(zone->mctx, zone->journal, DNS_JOURNAL_READ, &journal); if (result == ISC_R_SUCCESS) { jserial = dns_journal_last_serial(journal); + empty = dns_journal_empty(journal); dns_journal_destroy(&journal); } else { jserial = serial; @@ -4397,9 +4399,10 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, } if (jserial != serial) { - dns_zone_log(zone, ISC_LOG_INFO, - "journal file is out of date: " - "removing journal file"); + if (!empty) + dns_zone_log(zone, ISC_LOG_INFO, + "journal file is out of date: " + "removing journal file"); if (remove(zone->journal) < 0 && errno != ENOENT) { char strbuf[ISC_STRERRORSIZE]; isc__strerror(errno, strbuf, sizeof(strbuf)); @@ -9756,6 +9759,52 @@ dns_zone_refresh(dns_zone_t *zone) { UNLOCK_ZONE(zone); } +static void +zone_journal_compact(dns_zone_t *zone, dns_db_t *db, isc_uint32_t serial) { + isc_result_t result; + isc_int32_t journalsize; + dns_dbversion_t *ver = NULL; + isc_uint64_t dbsize; + + INSIST(LOCKED_ZONE(zone)); + if (inline_raw(zone)) + INSIST(LOCKED_ZONE(zone->secure)); + + journalsize = zone->journalsize; + if (journalsize == -1) { + journalsize = DNS_JOURNAL_SIZE_MAX; + dns_db_currentversion(db, &ver); + result = dns_db_getsize(db, ver, NULL, &dbsize); + dns_db_closeversion(db, &ver, ISC_FALSE); + if (result != ISC_R_SUCCESS) { + dns_zone_log(zone, ISC_LOG_ERROR, + "zone_journal_compact: " + "could not get zone size: %s", + isc_result_totext(result)); + } else if (dbsize < DNS_JOURNAL_SIZE_MAX / 2) { + journalsize = (isc_int32_t)dbsize * 2; + } + } + zone_debuglog(zone, "zone_journal_compact", 1, + "target journal size %d", journalsize); + result = dns_journal_compact(zone->mctx, zone->journal, + serial, journalsize); + switch (result) { + case ISC_R_SUCCESS: + case ISC_R_NOSPACE: + case ISC_R_NOTFOUND: + dns_zone_log(zone, ISC_LOG_DEBUG(3), + "dns_journal_compact: %s", + dns_result_totext(result)); + break; + default: + dns_zone_log(zone, ISC_LOG_ERROR, + "dns_journal_compact failed: %s", + dns_result_totext(result)); + break; + } +} + isc_result_t dns_zone_flush(dns_zone_t *zone) { isc_result_t result = ISC_R_SUCCESS; @@ -9773,7 +9822,7 @@ dns_zone_flush(dns_zone_t *zone) { dumping = ISC_TRUE; UNLOCK_ZONE(zone); if (!dumping) - result = zone_dump(zone, ISC_FALSE); /* Unknown task. */ + result = zone_dump(zone, ISC_TRUE); /* Unknown task. */ return (result); } @@ -9841,8 +9890,7 @@ dump_done(void *arg, isc_result_t result) { ENTER; - if (result == ISC_R_SUCCESS && zone->journal != NULL && - zone->journalsize != -1) { + if (result == ISC_R_SUCCESS && zone->journal != NULL) { /* * We don't own these, zone->dctx must stay valid. */ @@ -9887,37 +9935,19 @@ dump_done(void *arg, isc_result_t result) { } ZONEDB_UNLOCK(&secure->dblock, isc_rwlocktype_read); } - if (secure != NULL) - UNLOCK_ZONE(secure); - UNLOCK_ZONE(zone); - - /* - * Note: we are task locked here so we can test - * zone->xfr safely. - */ if (tresult == ISC_R_SUCCESS && zone->xfr == NULL) { - tresult = dns_journal_compact(zone->mctx, - zone->journal, - serial, - zone->journalsize); - switch (tresult) { - case ISC_R_SUCCESS: - case ISC_R_NOSPACE: - case ISC_R_NOTFOUND: - dns_zone_log(zone, ISC_LOG_DEBUG(3), - "dns_journal_compact: %s", - dns_result_totext(tresult)); - break; - default: - dns_zone_log(zone, ISC_LOG_ERROR, - "dns_journal_compact failed: %s", - dns_result_totext(tresult)); - break; + dns_db_t *zdb = NULL; + if (dns_zone_getdb(zone, &zdb) == ISC_R_SUCCESS) { + zone_journal_compact(zone, zdb, serial); + dns_db_detach(&db); } } else if (tresult == ISC_R_SUCCESS) { compact = ISC_TRUE; zone->compact_serial = serial; } + if (secure != NULL) + UNLOCK_ZONE(secure); + UNLOCK_ZONE(zone); } LOCK_ZONE(zone); @@ -13112,7 +13142,6 @@ dns_zone_setzeronosoattl(dns_zone_t *zone, isc_boolean_t state) { void dns_zone_setchecknames(dns_zone_t *zone, dns_severity_t severity) { - REQUIRE(DNS_ZONE_VALID(zone)); zone->check_names = severity; @@ -13120,7 +13149,6 @@ dns_zone_setchecknames(dns_zone_t *zone, dns_severity_t severity) { dns_severity_t dns_zone_getchecknames(dns_zone_t *zone) { - REQUIRE(DNS_ZONE_VALID(zone)); return (zone->check_names); @@ -13128,7 +13156,6 @@ dns_zone_getchecknames(dns_zone_t *zone) { void dns_zone_setjournalsize(dns_zone_t *zone, isc_int32_t size) { - REQUIRE(DNS_ZONE_VALID(zone)); zone->journalsize = size; @@ -13136,7 +13163,6 @@ dns_zone_setjournalsize(dns_zone_t *zone, isc_int32_t size) { isc_int32_t dns_zone_getjournalsize(dns_zone_t *zone) { - REQUIRE(DNS_ZONE_VALID(zone)); return (zone->journalsize); @@ -14445,7 +14471,7 @@ zone_replacedb(dns_zone_t *zone, dns_db_t *db, isc_boolean_t dump) { unsigned int nscount = 0; /* - * 'zone' and 'zonedb' locked by caller. + * 'zone' and 'zone->db' locked by caller. */ REQUIRE(DNS_ZONE_VALID(zone)); REQUIRE(LOCKED_ZONE(zone)); @@ -14530,24 +14556,8 @@ zone_replacedb(dns_zone_t *zone, dns_db_t *db, isc_boolean_t dump) { goto fail; if (dump) zone_needdump(zone, DNS_DUMP_DELAY); - else if (zone->journalsize != -1) { - result = dns_journal_compact(zone->mctx, zone->journal, - serial, zone->journalsize); - switch (result) { - case ISC_R_SUCCESS: - case ISC_R_NOSPACE: - case ISC_R_NOTFOUND: - dns_zone_log(zone, ISC_LOG_DEBUG(3), - "dns_journal_compact: %s", - dns_result_totext(result)); - break; - default: - dns_zone_log(zone, ISC_LOG_ERROR, - "dns_journal_compact failed: %s", - dns_result_totext(result)); - break; - } - } + else + zone_journal_compact(zone, zone->db, serial); if (zone->type == dns_zone_master && inline_raw(zone)) zone_send_secureserial(zone, serial); } else { @@ -14860,24 +14870,12 @@ zone_xfrdone(dns_zone_t *zone, isc_result_t result) { * Handle any deferred journal compaction. */ if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDCOMPACT)) { - result = dns_journal_compact(zone->mctx, zone->journal, - zone->compact_serial, - zone->journalsize); - switch (result) { - case ISC_R_SUCCESS: - case ISC_R_NOSPACE: - case ISC_R_NOTFOUND: - dns_zone_log(zone, ISC_LOG_DEBUG(3), - "dns_journal_compact: %s", - dns_result_totext(result)); - break; - default: - dns_zone_log(zone, ISC_LOG_ERROR, - "dns_journal_compact failed: %s", - dns_result_totext(result)); - break; + dns_db_t *db = NULL; + if (dns_zone_getdb(zone, &db) == ISC_R_SUCCESS) { + zone_journal_compact(zone, db, zone->compact_serial); + dns_db_detach(&db); + DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT); } - DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT); } if (secure != NULL) diff --git a/lib/isccfg/namedconf.c b/lib/isccfg/namedconf.c index f13178f5af..7c9bc429bf 100644 --- a/lib/isccfg/namedconf.c +++ b/lib/isccfg/namedconf.c @@ -2008,7 +2008,7 @@ zone_clauses[] = { { "masterfile-format", &cfg_type_masterformat, 0 }, { "masterfile-style", &cfg_type_masterstyle, 0 }, { "max-ixfr-log-size", &cfg_type_size, CFG_CLAUSEFLAG_OBSOLETE }, - { "max-journal-size", &cfg_type_sizenodefault, 0 }, + { "max-journal-size", &cfg_type_size, 0 }, { "max-records", &cfg_type_uint32, 0 }, { "max-refresh-time", &cfg_type_uint32, 0 }, { "max-retry-time", &cfg_type_uint32, 0 },