harvest print output.

git-svn-id: file:///svn/unbound/trunk@1002 be551aaa-1e26-0410-a405-d3ace91eadb9

testcode/harvest.c

@@ -71,6 +71,9 @@
 #include <signal.h>
 #include "config.h"
 #include "libunbound/unbound.h"
+#ifdef HAVE_SYS_STAT_H
+#include "sys/stat.h"
+#endif
 struct todo_item;
 struct labdata;
 
@@ -103,6 +106,8 @@ struct harvest_data {
 	int maxlabels;
 	/** number of RRs stored */
 	int num_rrs;
+	/** number of zones written */
+	int num_zones;
 };
 
 /**
@@ -314,7 +319,7 @@ new_todo_item(struct harvest_data* data, ldns_rdf* qname, int qtype,
 	else data->todo_list = it;
 	data->todo_last = it;
 	data->numtodo ++;
-	if(hverb) {
+	if(hverb >= 2) {
 		printf("new todo: ");
 		ldns_rdf_print(stdout, it->qname);
 		if(ldns_rr_descript((uint16_t)it->qtype) && 
@@ -386,6 +391,29 @@ process_rr(struct harvest_data* data, ldns_rr* rr, int depth)
 			ldns_rr_rdf(rr, 0)), depth+1);
 	}
 	/* store it */
+	if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_NSEC) {
+		/* find correct zone to store NSEC in (for delegation zones) */
+		if(ldns_dname_compare(ldns_rr_rdf(rr, 0), ldns_rr_owner(rr))
+			== 0) {
+			/* store at the single name = apex */
+		} else if(!ldns_dname_is_subdomain(ldns_rr_rdf(rr, 0), 
+			ldns_rr_owner(rr)) && lab->parent) {
+			/* if   owner NSEC subdomain-of-owner then
+		 	 * store at owner (owner is apex or empty nonterminal).
+		 	 * Otherwise at owner parent. */
+			lab = lab->parent;
+		}
+	} else if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_DS) {
+		/* store DSes in parent zone */
+		if(lab->parent)
+			lab = lab->parent;
+	} else if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_NSEC3) {
+		/* store NSEC3s one label up at zone apex */
+		if(lab->parent)
+			lab = lab->parent;
+	}
+	/* we assume NS set is equal across parent-child border. */
+
 	if(!ldns_rr_list_contains_rr(lab->rrlist, rr)) {
 		if(hverb >= 2) {
 			printf("store RR ");
@@ -506,6 +534,237 @@ harvest_main(struct harvest_data* data)
 	}
 }
 
+/** create directory if it does not exist */
+static void
+hv_mkdir(char* dir)
+{
+	if(mkdir(dir, 0755) == -1) {
+		if(errno == EEXIST)
+			return;
+		perror(dir);
+		error_exit("mkdir failed");
+	}
+}
+
+
+/** see if rrlist contains a SOA record */
+static ldns_rr*
+has_SOA(ldns_rr_list* list)
+{
+	size_t i;
+	for(i=0; i<ldns_rr_list_rr_count(list); i++) {
+		if(ldns_rr_get_type(ldns_rr_list_rr(list, i)) 
+			== LDNS_RR_TYPE_SOA)
+			return ldns_rr_list_rr(list, i);
+	}
+	return NULL;
+}
+
+/** write moredata for a zone*/
+static void
+write_moredata(struct harvest_data* data, struct labdata* zone,
+	FILE *f, struct labdata* thislab, ldns_rr* nslist)
+{
+	struct labdata* lab;
+	size_t i;
+	ldns_rr* ns;
+	LDNS_RBTREE_FOR(lab, struct labdata*, thislab->sublabels) {
+		if(has_SOA(lab->rrlist)) {
+			/* copy only NS glue */
+			for(i=0; i<ldns_rr_list_rr_count(lab->rrlist); i++) {
+				ns = ldns_rr_list_rr(lab->rrlist, i);
+				if(ldns_rr_get_type(ns) == LDNS_RR_TYPE_NS) {
+					ldns_rr_print(f, ns);
+					if(ldns_dname_is_subdomain(
+						ldns_rr_ns_nsdname(ns), 
+						lab->name)) {
+						ldns_rr_push_rdf(nslist, 
+							ldns_rdf_clone(
+							ldns_rr_ns_nsdname(ns)));
+					}
+				}
+			}
+		} else {
+			/* copy all, recurse */
+			for(i=0; i<ldns_rr_list_rr_count(lab->rrlist); i++) {
+				ldns_rr_print(f, 
+					ldns_rr_list_rr(lab->rrlist, i));
+			}
+			write_moredata(data, zone, f, lab, nslist);
+		}
+	}
+}
+
+/** find and write glue into zone file */
+static void
+write_glue(struct harvest_data* data, struct labdata* thislab, FILE* f, 
+	ldns_rdf* name, int dep)
+{
+	size_t i;
+	struct labdata* lab;
+	ldns_rr* rr;
+	if(ldns_dname_compare(name, thislab->name) == 0) {
+		/* this is it! Did we go outside the zone? */
+		if(dep == 0)
+			return;
+		/* find A and AAAA */
+		for(i=0; i<ldns_rr_list_rr_count(thislab->rrlist); i++) {
+			rr = ldns_rr_list_rr(thislab->rrlist, i);
+			if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_A ||
+			   ldns_rr_get_type(rr) == LDNS_RR_TYPE_AAAA) {
+				ldns_rr_print(f, rr);
+			}
+		}
+		return;
+	}
+	/* recurse deeper */
+	LDNS_RBTREE_FOR(lab, struct labdata*, thislab->sublabels) {
+		if(has_SOA(lab->rrlist)) {
+			write_glue(data, lab, f, name, dep+1);
+		} else {
+			write_glue(data, lab, f, name, dep);
+		}
+	}
+}
+
+/** write zonefile for zone at this apex */
+static void
+write_zonefile(struct harvest_data* data, int dep, FILE* zlist, 
+	struct labdata* apex, ldns_rr* soa)
+{
+	FILE *f;
+	char fname[1024];
+	char* zname = ldns_rdf2str(apex->name);
+	time_t tm = time(NULL);
+	size_t i;
+	ldns_rr* nslist;
+	if(!zname) error_exit("out of mem ldns_rdf2str");
+	if(strcmp(zname, ".") == 0)
+		snprintf(fname, sizeof(fname), "l%d/root.zone", dep);
+	else	snprintf(fname, sizeof(fname), "l%d/%szone", dep, zname);
+
+	fprintf(zlist, "zone: name: \"%s\" %s%szonefile: \"%s\"\n",
+		zname, 
+		strlen(zname)/8<1?"\t":"",
+		strlen(zname)/8<2?"\t":"",
+		fname);
+
+	if(hverb) printf("writing %s\n", fname);
+	f = fopen(fname, "w");
+	if(!f) {
+		perror(fname);
+		error_exit("cannot open zone file");
+	}
+	fprintf(f, "; %s - generated by harvest program.\n", fname);
+	fprintf(f, "; zone name %s - this is a partial snapshot of "
+		"data relevant to the query list.\n", zname);
+	fprintf(f, "; created %u - date %s\n", (unsigned)tm, ctime(&tm));
+	ldns_rr_print(f, soa);
+	fprintf(f, "\n");
+	for(i=0; i<ldns_rr_list_rr_count(apex->rrlist); i++) {
+		if(ldns_rr_get_type(ldns_rr_list_rr(apex->rrlist, i))
+			== LDNS_RR_TYPE_SOA) continue;
+		ldns_rr_print(f, ldns_rr_list_rr(apex->rrlist, i));
+	}
+	/* search for more data - subdomains inside the zone, NS glue */
+	nslist = ldns_rr_new();
+	if(!nslist) error_exit("out of memory");
+	fprintf(f, "; end of apex, more data follows\n");
+	write_moredata(data, apex, f, apex, nslist);
+
+	/* add NS from apex that need glue too */
+	for(i=0; i<ldns_rr_list_rr_count(apex->rrlist); i++) {
+		if(ldns_rr_get_type(ldns_rr_list_rr(apex->rrlist, i)) !=
+			LDNS_RR_TYPE_NS)
+			continue;
+		/* these are only added again if in a subzone */
+		if(ldns_dname_is_subdomain(ldns_rr_ns_nsdname(
+			ldns_rr_list_rr(apex->rrlist, i)), apex->name)) {
+			ldns_rr_push_rdf(nslist, ldns_rdf_clone(
+				ldns_rr_ns_nsdname(ldns_rr_list_rr(
+				apex->rrlist, i))));
+		}
+	}
+
+	fprintf(f, "; glue data follows\n");
+	/* lookup and add glue (if not already in zone) */
+	for(i=0; i<ldns_rr_rd_count(nslist); i++) {
+		write_glue(data, apex, f, ldns_rr_rdf(nslist, i), 0);
+	}
+
+	fclose(f);
+	ldns_rr_free(nslist);
+	free(zname);
+}
+
+/** create zones at depth d in label tree */
+static void
+create_zones(struct harvest_data* data, int dep, FILE* zlist, 
+	struct labdata* labnow, int depnow)
+{
+	struct labdata* s;
+	ldns_rr* soa;
+	if(depnow == dep) {
+		/* see if this is a zone start - a SOA */
+		if((soa=has_SOA(labnow->rrlist))) {
+			write_zonefile(data, dep, zlist, labnow, soa);
+			data->num_zones++;
+		}
+		return;
+	}
+	/* recurse */
+	LDNS_RBTREE_FOR(s, struct labdata*, labnow->sublabels) {
+		create_zones(data, dep, zlist, s, depnow+1);
+	}
+}
+
+/** sort rrlists */
+static void
+harvest_sort(struct labdata* lab)
+{
+	struct labdata* s;
+	/* prettier output if sorted here */
+	ldns_rr_list_sort(lab->rrlist);
+	/* and recurse */
+	LDNS_RBTREE_FOR(s, struct labdata*, lab->sublabels) {
+		harvest_sort(s);
+	}
+}
+
+/** output harvested results */
+static void
+harvest_output(struct harvest_data* data)
+{
+	int d;
+	char buf[20];
+	FILE* zlist;
+	int lastzones;
+	hv_mkdir(data->resultdir);
+	if(chdir(data->resultdir) == -1) {
+		perror(data->resultdir);
+		error_exit("cannot chdir");
+	}
+	harvest_sort(data->root);
+	/* create zones */
+	for(d = 0; d<data->maxlabels; d++) {
+		lastzones = data->num_zones;
+		printf("creating zones %d\n", d);	
+		snprintf(buf, sizeof(buf), "l%d", d);
+		hv_mkdir(buf);
+		snprintf(buf, sizeof(buf), "l%d.zones", d);
+		zlist = fopen(buf, "w");
+		if(!zlist) {
+			perror(buf);
+			error_exit("cannot write zonelist file");
+		}
+		fprintf(zlist, "# partial zones at depth %d\n", d);
+		create_zones(data, d, zlist, data->root, 0);
+		fclose(zlist);
+		printf("creating zones %d - %d zones written\n", d,
+			data->num_zones - lastzones);	
+	}
+}
+
 /** getopt global, in case header files fail to declare it. */
 extern int optind;
 /** getopt global, in case header files fail to declare it. */
@@ -551,6 +810,7 @@ int main(int argc, char* argv[])
 	
 	/* harvest the data */
 	harvest_main(&data);
+	harvest_output(&data);
 
 	/* no cleanup except the context (to close open sockets) */
 	ub_ctx_delete(data.ctx);