upstream/bind9
1
0
Fork 0
mirror of https://github.com/isc-projects/bind9.git synced 2026-03-06 15:30:29 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[master] automatically tune max-journal-size

Browse source 
4613.	[func]		By default, the maximum size of a zone journal file
			is now twice the size of the zone's contents (there
			is little benefit to a journal larger than this).
			This can be overridden by setting "max-journal-size"
			to "unlimited" or to an explicit value up to 2G.
			Thanks to Tony Finch. [RT #38324]
This commit is contained in:
Evan Hunt 2017-05-02 13:23:08 -07:00
parent 4c97cb13bd
commit d39ab7440e
16 changed files with 208 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CHANGES
View file

@ -1,3 +1,10 @@
4613. [func] By default, the maximum size of a zone journal file
is now twice the size of the zone's contents (there
is little benefit to a journal larger than this).
This can be overridden by setting "max-journal-size"
to "unlimited" or to an explicit value up to 2G.
Thanks to Tony Finch. [RT #38324]
4612. [bug] Silence 'may be use uninitalised' warning and simplify
the code in lwres/getaddinfo:process_answer.
[RT #45158]

2
bin/named/config.c
View file

@ -233,7 +233,7 @@ options {\n\
sig-signing-type 65534;\n\
inline-signing no;\n\
zone-statistics terse;\n\
max-journal-size unlimited;\n\
max-journal-size default;\n\
ixfr-from-differences false;\n\
check-wildcard yes;\n\
check-sibling yes;\n\

21
bin/named/zoneconf.c
View file

@ -20,6 +20,7 @@
#include <dns/db.h>
#include <dns/ipkeylist.h>
#include <dns/fixedname.h>
#include <dns/journal.h>
#include <dns/log.h>
#include <dns/name.h>
#include <dns/masterdump.h>
@ -1211,12 +1212,16 @@ ns_zone_configure(const cfg_obj_t *config, const cfg_obj_t *vconfig,
dns_zone_setjournalsize(zone, -1);
if (cfg_obj_isstring(obj)) {
const char *str = cfg_obj_asstring(obj);
INSIST(strcasecmp(str, "unlimited") == 0);
journal_size = ISC_UINT32_MAX / 2;
if (strcasecmp(str, "unlimited") == 0) {
journal_size = DNS_JOURNAL_SIZE_MAX;
} else {
INSIST(strcasecmp(str, "default") == 0);
journal_size = -1;
}
} else {
isc_resourcevalue_t value;
value = cfg_obj_asuint64(obj);
if (value > ISC_UINT32_MAX / 2) {
if (value > DNS_JOURNAL_SIZE_MAX) {
cfg_obj_log(obj, ns_g_lctx,
ISC_LOG_ERROR,
"'max-journal-size "
@ -1331,12 +1336,16 @@ ns_zone_configure(const cfg_obj_t *config, const cfg_obj_t *vconfig,
dns_zone_setjournalsize(zone, -1);
if (cfg_obj_isstring(obj)) {
const char *str = cfg_obj_asstring(obj);
INSIST(strcasecmp(str, "unlimited") == 0);
journal_size = ISC_UINT32_MAX / 2;
if (strcasecmp(str, "unlimited") == 0) {
journal_size = DNS_JOURNAL_SIZE_MAX;
} else {
INSIST(strcasecmp(str, "default") == 0);
journal_size = -1;
}
} else {
isc_resourcevalue_t value;
value = cfg_obj_asuint64(obj);
if (value > ISC_UINT32_MAX / 2) {
if (value > DNS_JOURNAL_SIZE_MAX) {
cfg_obj_log(obj, ns_g_lctx,
ISC_LOG_ERROR,
"'max-journal-size "

1
bin/tests/system/nsupdate/clean.sh
View file

@ -19,6 +19,7 @@ rm -f ns*/named.lock
rm -f ns1/*.jnl ns2/*.jnl ns3/*.jnl
rm -f ns1/example.db ns1/unixtime.db ns1/yyyymmddvv.db ns1/update.db ns1/other.db ns1/keytests.db
rm -f ns1/many.test.db
rm -f ns1/maxjournal.db
rm -f ns1/md5.key ns1/sha1.key ns1/sha224.key ns1/sha256.key ns1/sha384.key
rm -f ns1/sha512.key ns1/ddns.key
rm -f ns2/example.bk

7
bin/tests/system/nsupdate/ns1/many.test.db.in
View file

@ -4,14 +4,13 @@
; License, v. 2.0. If a copy of the MPL was not distributed with this
; file, You can obtain one at http://mozilla.org/MPL/2.0/.
$ORIGIN .
$TTL 300 ; 5 minutes
many.test IN SOA ns1.example.nil. hostmaster.example.nil. (
@ IN SOA ns1.example.nil. hostmaster.example.nil. (
1 ; serial
2000 ; refresh (2000 seconds)
2000 ; retry (2000 seconds)
1814400 ; expire (3 weeks)
3600 ; minimum (1 hour)
)
many.test NS ns1.example.nil.
many.test NS ns2.example.nil.
@ NS ns1.example.nil.
NS ns2.example.nil.

15
bin/tests/system/nsupdate/ns1/maxjournal.db.in Normal file
View file

@ -0,0 +1,15 @@
; Copyright (C) 2014, 2016 Internet Systems Consortium, Inc. ("ISC")
;
; This Source Code Form is subject to the terms of the Mozilla Public
; License, v. 2.0. If a copy of the MPL was not distributed with this
; file, You can obtain one at http://mozilla.org/MPL/2.0/.
$TTL 300 ; 5 minutes
@ IN SOA ns1.example.nil. hostmaster.example.nil. (
1 ; serial
2000 ; refresh (2000 seconds)
2000 ; retry (2000 seconds)
1814400 ; expire (3 weeks)
3600 ; minimum (1 hour)
)
@ NS ns1.example.nil.

7
bin/tests/system/nsupdate/ns1/named.conf
View file

@ -129,3 +129,10 @@ zone "sample" {
allow-update { any; };
file "sample.db";
};
zone "maxjournal.test" {
type master;
allow-update { any; };
file "maxjournal.db";
max-journal-size default;
};

3
bin/tests/system/nsupdate/setup.sh
View file

@ -61,3 +61,6 @@ rm -f ns1/many.test.db.jnl
cp ns1/sample.db.in ns1/sample.db
cp ns2/sample.db.in ns2/sample.db
cp -f ns1/maxjournal.db.in ns1/maxjournal.db
rm -f ns1/maxjournal.db.jnl

43
bin/tests/system/nsupdate/tests.sh
View file

@ -6,8 +6,6 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# $Id: tests.sh,v 1.42 2011/12/16 23:01:17 each Exp $
SYSTEMTESTTOP=..
. $SYSTEMTESTTOP/conf.sh
@ -583,6 +581,7 @@ if [ $ret -ne 0 ]; then
fi
n=`expr $n + 1`
ret=0
echo "I:check that yyyymmddvv serial number is correctly generated ($n)"
oldserial=`$DIG +short yyyymmddvv.nil. soa @10.53.0.1 -p 5300 | awk '{print $3}'` || ret=1
$NSUPDATE <<END > /dev/null 2>&1 || ret=1
@ -630,6 +629,46 @@ test ${lines:-0} -eq 64 || ret=1
[ $ret = 0 ] || { echo I:failed; status=1; }
fi
n=`expr $n + 1`
echo "I:check max-journal-size limits ($n)"
ret=0
rm -f nsupdate.out1-$n
# add one record
$NSUPDATE << EOF >> nsupdate.out1-$n 2>&1
server 10.53.0.1 5300
zone maxjournal.test
update add z.maxjournal.test 300 IN A 10.20.30.40
send
EOF
for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
# repeatedly add and remove the same set of records to fill up
# the journal file without changing the zone content
$NSUPDATE << EOF >> nsupdate.out1-$n 2>&1
server 10.53.0.1 5300
zone maxjournal.test
update add a.maxjournal.test 300 IN A 1.2.3.4
update add b.maxjournal.test 300 IN A 1.2.3.4
update add c.maxjournal.test 300 IN A 1.2.3.4
update add d.maxjournal.test 300 IN A 1.2.3.4
send
update del a.maxjournal.test
update del b.maxjournal.test
update del c.maxjournal.test
update del d.maxjournal.test
send
EOF
done
# check that the journal is big enough to require truncation.
size=`$PERL -e 'use File::stat; my $sb = stat(@ARGV[0]); printf("%s\n", $sb->size);' ns1/maxjournal.db.jnl`
[ "$size" -gt 6000 ] || ret=1
sleep 1
$RNDC -c ../common/rndc.conf -s 10.53.0.1 -p 9953 sync maxjournal.test
sleep 1
size=`$PERL -e 'use File::stat; my $sb = stat(@ARGV[0]); printf("%s\n", $sb->size);' ns1/maxjournal.db.jnl`
[ "$size" -lt 5000 ] || ret=1
[ $ret = 0 ] || { echo I:failed; status=1; }
n=`expr $n + 1`
echo "I:check check-names processing ($n)"
ret=0

28
doc/arm/Bv9ARM-book.xml
View file

@ -8492,16 +8492,24 @@ avoid-v6-udp-ports { 40000; range 50000 60000; };
<term><command>max-journal-size</command></term>
<listitem>
<para>
Sets a maximum size for each journal file
(see <xref linkend="journal"/>). When the journal file
approaches
the specified size, some of the oldest transactions in the
journal
will be automatically removed. The largest permitted
value is 2 gigabytes. The default is
<literal>unlimited</literal>, which also
means 2 gigabytes.
This may also be set on a per-zone basis.
Sets a maximum size for each journal file (see
<xref linkend="journal"/>), expressed in bytes
or, if followed by an optional unit suffix ('k',
'm', or 'g'), in kilobytes, megabytes, or gigabytes.
When the journal file approaches the specified size,
some of the oldest transactions in the journal
will be automatically removed. The largest
permitted value is 2 gigabytes. Very small
values are rounded up to 4096 bytes. You
can specify <literal>unlimited</literal>, which
also means 2 gigabytes. If you set the limit to
<literal>default</literal> or leave it unset, the
journal is allowed to grow up to twice as large as
the zone. (There is little benefit in storing
larger journals.)
</para>
<para>
This option may also be set on a per-zone basis.
</para>
</listitem>
</varlistentry>

10
doc/arm/notes.xml
View file

@ -151,6 +151,16 @@
<section xml:id="relnotes_features"><info><title>New Features</title></info>
<itemizedlist>
<listitem>
<para>
Setting <command>max-journal-size</command> to
<literal>default</command> limits journal sizes to twice the
size of the zone contents. This can be overridden by setting
<command>max-journal-size</command> to <literal>unlimited</literal>
or to an explicit value up to 2G. Thanks to Tony Finch for
the contribution. [RT #38324]
</para>
</listitem>
<listitem>
<para>
The <command>new-zones-directory</command> option allows

9
lib/dns/include/dns/journal.h
View file

@ -41,6 +41,9 @@
#define DNS_JOURNAL_CREATE 0x00000001 /* ISC_TRUE */
#define DNS_JOURNAL_WRITE 0x00000002
#define DNS_JOURNAL_SIZE_MAX ISC_INT32_MAX
#define DNS_JOURNAL_SIZE_MIN 4096
/***
*** Types
***/
@ -169,6 +172,12 @@ dns_journal_write_transaction(dns_journal_t *j, dns_diff_t *diff);
* Reading transactions from journals.
*/
isc_boolean_t
dns_journal_empty(dns_journal_t *j);
/*<
* Find out if a journal is empty.
*/
isc_uint32_t
dns_journal_first_serial(dns_journal_t *j);
isc_uint32_t

9
lib/dns/journal.c
View file

@ -6,8 +6,6 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
/* $Id: journal.c,v 1.120 2011/12/22 07:32:41 each Exp $ */
#include <config.h>
#include <stdlib.h>
@ -1536,6 +1534,11 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) {
/*
* Miscellaneous accessors.
*/
isc_boolean_t
dns_journal_empty(dns_journal_t *j) {
return (JOURNAL_EMPTY(&j->header));
}
isc_uint32_t
dns_journal_first_serial(dns_journal_t *j) {
return (j->header.begin.serial);
@ -2139,6 +2142,8 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, isc_uint32_t serial,
*/
indexend = sizeof(journal_rawheader_t) +
j->header.index_size * sizeof(journal_rawpos_t);
if (target_size < DNS_JOURNAL_SIZE_MIN)
target_size = DNS_JOURNAL_SIZE_MIN;
if (target_size < indexend * 2)
target_size = target_size/2 + indexend;

1
lib/dns/win32/libdns.def.in
View file

@ -405,6 +405,7 @@ dns_journal_commit
dns_journal_compact
dns_journal_current_rr
dns_journal_destroy
dns_journal_empty
dns_journal_first_rr
dns_journal_first_serial
dns_journal_get_sourceserial

140
lib/dns/zone.c
View file

@ -4385,11 +4385,13 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
! DNS_ZONE_OPTION(zone, DNS_ZONEOPT_IXFRFROMDIFFS)) {
isc_uint32_t jserial;
dns_journal_t *journal = NULL;
isc_boolean_t empty = ISC_FALSE;
result = dns_journal_open(zone->mctx, zone->journal,
DNS_JOURNAL_READ, &journal);
if (result == ISC_R_SUCCESS) {
jserial = dns_journal_last_serial(journal);
empty = dns_journal_empty(journal);
dns_journal_destroy(&journal);
} else {
jserial = serial;
@ -4397,9 +4399,10 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
}
if (jserial != serial) {
dns_zone_log(zone, ISC_LOG_INFO,
"journal file is out of date: "
"removing journal file");
if (!empty)
dns_zone_log(zone, ISC_LOG_INFO,
"journal file is out of date: "
"removing journal file");
if (remove(zone->journal) < 0 && errno != ENOENT) {
char strbuf[ISC_STRERRORSIZE];
isc__strerror(errno, strbuf, sizeof(strbuf));
@ -9756,6 +9759,52 @@ dns_zone_refresh(dns_zone_t *zone) {
UNLOCK_ZONE(zone);
}
static void
zone_journal_compact(dns_zone_t *zone, dns_db_t *db, isc_uint32_t serial) {
isc_result_t result;
isc_int32_t journalsize;
dns_dbversion_t *ver = NULL;
isc_uint64_t dbsize;
INSIST(LOCKED_ZONE(zone));
if (inline_raw(zone))
INSIST(LOCKED_ZONE(zone->secure));
journalsize = zone->journalsize;
if (journalsize == -1) {
journalsize = DNS_JOURNAL_SIZE_MAX;
dns_db_currentversion(db, &ver);
result = dns_db_getsize(db, ver, NULL, &dbsize);
dns_db_closeversion(db, &ver, ISC_FALSE);
if (result != ISC_R_SUCCESS) {
dns_zone_log(zone, ISC_LOG_ERROR,
"zone_journal_compact: "
"could not get zone size: %s",
isc_result_totext(result));
} else if (dbsize < DNS_JOURNAL_SIZE_MAX / 2) {
journalsize = (isc_int32_t)dbsize * 2;
}
}
zone_debuglog(zone, "zone_journal_compact", 1,
"target journal size %d", journalsize);
result = dns_journal_compact(zone->mctx, zone->journal,
serial, journalsize);
switch (result) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(result));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(result));
break;
}
}
isc_result_t
dns_zone_flush(dns_zone_t *zone) {
isc_result_t result = ISC_R_SUCCESS;
@ -9773,7 +9822,7 @@ dns_zone_flush(dns_zone_t *zone) {
dumping = ISC_TRUE;
UNLOCK_ZONE(zone);
if (!dumping)
result = zone_dump(zone, ISC_FALSE); /* Unknown task. */
result = zone_dump(zone, ISC_TRUE); /* Unknown task. */
return (result);
}
@ -9841,8 +9890,7 @@ dump_done(void *arg, isc_result_t result) {
ENTER;
if (result == ISC_R_SUCCESS && zone->journal != NULL &&
zone->journalsize != -1) {
if (result == ISC_R_SUCCESS && zone->journal != NULL) {
/*
* We don't own these, zone->dctx must stay valid.
*/
@ -9887,37 +9935,19 @@ dump_done(void *arg, isc_result_t result) {
}
ZONEDB_UNLOCK(&secure->dblock, isc_rwlocktype_read);
}
if (secure != NULL)
UNLOCK_ZONE(secure);
UNLOCK_ZONE(zone);
/*
* Note: we are task locked here so we can test
* zone->xfr safely.
*/
if (tresult == ISC_R_SUCCESS && zone->xfr == NULL) {
tresult = dns_journal_compact(zone->mctx,
zone->journal,
serial,
zone->journalsize);
switch (tresult) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(tresult));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(tresult));
break;
dns_db_t *zdb = NULL;
if (dns_zone_getdb(zone, &zdb) == ISC_R_SUCCESS) {
zone_journal_compact(zone, zdb, serial);
dns_db_detach(&db);
}
} else if (tresult == ISC_R_SUCCESS) {
compact = ISC_TRUE;
zone->compact_serial = serial;
}
if (secure != NULL)
UNLOCK_ZONE(secure);
UNLOCK_ZONE(zone);
}
LOCK_ZONE(zone);
@ -13112,7 +13142,6 @@ dns_zone_setzeronosoattl(dns_zone_t *zone, isc_boolean_t state) {
void
dns_zone_setchecknames(dns_zone_t *zone, dns_severity_t severity) {
REQUIRE(DNS_ZONE_VALID(zone));
zone->check_names = severity;
@ -13120,7 +13149,6 @@ dns_zone_setchecknames(dns_zone_t *zone, dns_severity_t severity) {
dns_severity_t
dns_zone_getchecknames(dns_zone_t *zone) {
REQUIRE(DNS_ZONE_VALID(zone));
return (zone->check_names);
@ -13128,7 +13156,6 @@ dns_zone_getchecknames(dns_zone_t *zone) {
void
dns_zone_setjournalsize(dns_zone_t *zone, isc_int32_t size) {
REQUIRE(DNS_ZONE_VALID(zone));
zone->journalsize = size;
@ -13136,7 +13163,6 @@ dns_zone_setjournalsize(dns_zone_t *zone, isc_int32_t size) {
isc_int32_t
dns_zone_getjournalsize(dns_zone_t *zone) {
REQUIRE(DNS_ZONE_VALID(zone));
return (zone->journalsize);
@ -14445,7 +14471,7 @@ zone_replacedb(dns_zone_t *zone, dns_db_t *db, isc_boolean_t dump) {
unsigned int nscount = 0;
/*
* 'zone' and 'zonedb' locked by caller.
* 'zone' and 'zone->db' locked by caller.
*/
REQUIRE(DNS_ZONE_VALID(zone));
REQUIRE(LOCKED_ZONE(zone));
@ -14530,24 +14556,8 @@ zone_replacedb(dns_zone_t *zone, dns_db_t *db, isc_boolean_t dump) {
goto fail;
if (dump)
zone_needdump(zone, DNS_DUMP_DELAY);
else if (zone->journalsize != -1) {
result = dns_journal_compact(zone->mctx, zone->journal,
serial, zone->journalsize);
switch (result) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(result));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(result));
break;
}
}
else
zone_journal_compact(zone, zone->db, serial);
if (zone->type == dns_zone_master && inline_raw(zone))
zone_send_secureserial(zone, serial);
} else {
@ -14860,24 +14870,12 @@ zone_xfrdone(dns_zone_t *zone, isc_result_t result) {
* Handle any deferred journal compaction.
*/
if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDCOMPACT)) {
result = dns_journal_compact(zone->mctx, zone->journal,
zone->compact_serial,
zone->journalsize);
switch (result) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(result));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(result));
break;
dns_db_t *db = NULL;
if (dns_zone_getdb(zone, &db) == ISC_R_SUCCESS) {
zone_journal_compact(zone, db, zone->compact_serial);
dns_db_detach(&db);
DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT);
}
DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT);
}
if (secure != NULL)

2
lib/isccfg/namedconf.c
View file

@ -2008,7 +2008,7 @@ zone_clauses[] = {
{ "masterfile-format", &cfg_type_masterformat, 0 },
{ "masterfile-style", &cfg_type_masterstyle, 0 },
{ "max-ixfr-log-size", &cfg_type_size, CFG_CLAUSEFLAG_OBSOLETE },
{ "max-journal-size", &cfg_type_sizenodefault, 0 },
{ "max-journal-size", &cfg_type_size, 0 },
{ "max-records", &cfg_type_uint32, 0 },
{ "max-refresh-time", &cfg_type_uint32, 0 },
{ "max-retry-time", &cfg_type_uint32, 0 },