-syncUUID search in syncrepl

-test017-syncreplication

servers/slapd/config.c

@@ -2896,51 +2896,9 @@ parse_syncrepl_line(
 			si->attrsonly = 1;
 		} else if ( !strncasecmp( cargv[ i ],
 				ATTRSSTR, sizeof( ATTRSSTR ) - 1 ) ) {
-			char **tmp;
 			val = cargv[ i ] + sizeof( ATTRSSTR );
-			for ( ; hp = strchr( val, ' ' ); val = ++hp ) {
-				*hp = '\0';
-				if ( *val != '\0' ) {
-					nr_attr++;
-					tmp = (char **) ch_realloc( si->attrs, nr_attr * sizeof( char * ));
-					if ( tmp == NULL ) {
-#ifdef NEW_LOGGING
-						LDAP_LOG( CONFIG, ERR, "out of memory\n", 0,0,0 );
-#else
-						Debug( LDAP_DEBUG_ANY, "out of memory\n", 0,0,0 );
-#endif
-						return -1;
-					}
-					si->attrs = tmp;
-					si->attrs[ nr_attr - 1 ] = ch_strdup( val );
-				}
-			}
-			if ( *val != '\0' ) {
-				nr_attr++;
-				tmp = (char **) ch_realloc( si->attrs, nr_attr * sizeof( char * ));
-				if ( tmp == NULL ) {
-#ifdef NEW_LOGGING
-					LDAP_LOG( CONFIG, ERR, "out of memory\n", 0,0,0 );
-#else
-					Debug( LDAP_DEBUG_ANY, "out of memory\n", 0,0,0 );
-#endif
-					return -1;
-				}
-				si->attrs = tmp;
-				si->attrs[ nr_attr - 1 ] = ch_strdup( val );
-			}
-			nr_attr++;
-			tmp = (char **) ch_realloc( si->attrs, nr_attr * sizeof( char * ));
-			if ( tmp == NULL ) {
-#ifdef NEW_LOGGING
-				LDAP_LOG( CONFIG, ERR, "out of memory\n", 0,0,0 );
-#else
-				Debug( LDAP_DEBUG_ANY, "out of memory\n", 0,0,0 );
-#endif
-				return -1;
-			}
-			si->attrs = tmp;
-			si->attrs[ nr_attr - 1 ] = NULL;
+			si->attrs = NULL;
+			si->attrs = str2clist( si->attrs, val, "," );
 			gots |= GOT_ATTRS;
 		} else if ( !strncasecmp( cargv[ i ],
 				TYPESTR, sizeof( TYPESTR ) - 1 ) ) {

servers/slapd/proto-slap.h

@@ -1127,6 +1127,8 @@ LDAP_SLAPD_V (struct runqueue_s) syncrepl_rq;
 
 LDAP_SLAPD_F (void) init_syncrepl LDAP_P(());
 LDAP_SLAPD_F (void*) do_syncrepl LDAP_P((void *, void *));
+
+LDAP_SLAPD_F (char **) str2clist( char **, char *, const char * );
 #endif
 
 LDAP_END_DECL

servers/slapd/slap.h

@@ -1323,6 +1323,7 @@ typedef struct syncinfo_s {
         int             tls;
         int             found;
         struct berval   *syncUUID;
+		struct berval	*syncUUID_ndn;
         struct berval   *syncCookie;
         Avlnode         *presentlist;
 		LDAP_LIST_HEAD(np, nonpresent_entry) nonpresentlist;

servers/slapd/syncrepl.c

@@ -69,6 +69,7 @@ slap_mods2entry_syncrepl( Modifications *, Entry **, int,
 
 /* callback functions */
 static int cookie_callback( struct slap_op *, struct slap_rep * );
+static int dn_callback( struct slap_op *, struct slap_rep * );
 static int nonpresent_callback( struct slap_op *, struct slap_rep * );
 static int null_callback( struct slap_op *, struct slap_rep * );
 
@@ -891,13 +892,12 @@ syncrepl_entry(
 	char csnbuf[ LDAP_LUTIL_CSNSTR_BUFSIZE ];
 
 	SlapReply	rs = {REP_RESULT};
-	int rc;
+	int rc = LDAP_SUCCESS;
+
+	struct berval base_bv = {0, NULL};
 
-#if 0 /* FIXME : UUID search required first */
 	char *filterstr;
-	struct berval filterstr_bv;
 	Filter *filter;
-#endif
 
 	Attribute *a;
 
@@ -919,74 +919,103 @@ syncrepl_entry(
 		attr_merge_one( e, slap_schema.si_ad_entryUUID, syncUUID, syncUUID );
 	}
 
-#if 0 /* FIXME : UUID search required first */
 	filterstr = (char *) sl_malloc( strlen("entryUUID=") + syncUUID->bv_len + 1, op->o_tmpmemctx ); 
 	strcpy( filterstr, "entryUUID=" );
 	strcat( filterstr, syncUUID->bv_val );
-#endif
 
 	si->e = e;
 	si->syncUUID = syncUUID;
+	si->syncUUID_ndn = NULL;
 
-#if 0 /* FIXME : UUID search required first */
 	filter = str2filter( filterstr );
-	ber_str2bv( filterstr, strlen(filterstr), 1, &filterstr_bv );
+	ber_str2bv( filterstr, strlen(filterstr), 1, &op->ors_filterstr );
 	ch_free( filterstr );
-#endif
+	op->ors_filter = filter;
+	op->ors_scope = LDAP_SCOPE_SUBTREE;
 
-	op->o_req_dn = e->e_name;
-	op->o_req_ndn = e->e_nname;
+	/* get syncrepl cookie of shadow replica from subentry */
+	ber_str2bv( si->base, strlen(si->base), 1, &base_bv ); 
+	dnPrettyNormal( 0, &base_bv, &op->o_req_dn, &op->o_req_ndn, op->o_tmpmemctx );
+	ch_free( base_bv.bv_val );
 
+	/* set callback function */
 	op->o_callback = &cb;
-	cb.sc_response = null_callback;
+	cb.sc_response = dn_callback;
 	cb.sc_private = si;
 
+	be->be_search( op, &rs );
+
+	ch_free( op->o_req_dn.bv_val );
+	ch_free( op->o_req_ndn.bv_val );
+	filter_free( op->ors_filter );
+	ch_free( op->ors_filterstr.bv_val );
+
+	cb.sc_response = null_callback;
+
+	if ( si->syncUUID_ndn )
+		printf("syncUUID_ndn = %s\n", si->syncUUID_ndn );
+
 	switch ( syncstate ) {
 	case LDAP_SYNC_ADD :
 	case LDAP_SYNC_MODIFY :
-sync_add_retry:
-		op->o_tag = LDAP_REQ_MODIFY;
-		op->orm_modlist = modlist;
-		rc = be->be_modify( op, &rs );
-		if ( rc != LDAP_SUCCESS ) {
-			if ( rc == LDAP_REFERRAL ||
-				 rc == LDAP_NO_SUCH_OBJECT ||
-				 rc == DB_NOTFOUND ) {
-				op->o_tag = LDAP_REQ_ADD;
-				op->ora_e = e;
-				rc = be->be_add( op, &rs );
-				if ( rc != LDAP_SUCCESS ) {
-					if ( rc == LDAP_ALREADY_EXISTS ) {
-						goto sync_add_retry;
-					} else if ( rc == LDAP_REFERRAL ||
-								rc == LDAP_NO_SUCH_OBJECT ||
-								rc == DB_NOTFOUND ) {
-						syncrepl_add_glue(ld, op, e,
-							modlist, syncstate,
-							syncUUID, syncCookie);
-					} else {
+
+		rc = LDAP_SUCCESS;
+
+		if ( si->syncUUID_ndn ) {
+			op->o_req_dn = *si->syncUUID_ndn;
+			op->o_req_ndn = *si->syncUUID_ndn;
+			op->o_tag = LDAP_REQ_DELETE;
+			rc = be->be_delete( op, &rs );
+		}
+
+		if ( rc == LDAP_SUCCESS ||
+			 rc == LDAP_REFERRAL ||
+			 rc == LDAP_NO_SUCH_OBJECT ||
+			 rc == DB_NOTFOUND ) {
+			op->o_tag = LDAP_REQ_ADD;
+			op->ora_e = e;
+			op->o_req_dn = e->e_name;
+			op->o_req_ndn = e->e_nname;
+			rc = be->be_add( op, &rs );
+			if ( rc != LDAP_SUCCESS ) {
+				if ( rc == LDAP_ALREADY_EXISTS ) {
 #ifdef NEW_LOGGING
-						LDAP_LOG( OPERATION, ERR,
-							"be_add failed (%d)\n",
-							rc, 0, 0 );
+					LDAP_LOG( OPERATION, ERR,
+						"be_add failed : already exists (%d)\n",
+						rc, 0, 0 );
 #else
-						Debug( LDAP_DEBUG_ANY,
-							"be_add failed (%d)\n",
-							rc, 0, 0 );
+					Debug( LDAP_DEBUG_ANY,
+						"be_add failed : already exists (%d)\n",
+						rc, 0, 0 );
 #endif
-					}
+				} else if ( rc == LDAP_REFERRAL ||
+							rc == LDAP_NO_SUCH_OBJECT ||
+							rc == DB_NOTFOUND ) {
+					syncrepl_add_glue(ld, op, e,
+						modlist, syncstate,
+						syncUUID, syncCookie);
 				} else {
-					return 0;
+#ifdef NEW_LOGGING
+					LDAP_LOG( OPERATION, ERR,
+						"be_add failed (%d)\n",
+						rc, 0, 0 );
+#else
+					Debug( LDAP_DEBUG_ANY,
+						"be_add failed (%d)\n",
+						rc, 0, 0 );
+#endif
 				}
 			} else {
-#ifdef NEW_LOGGING
-				LDAP_LOG( OPERATION, ERR,
-					"be_modify failed (%d)\n", rc, 0, 0 );
-#else
-				Debug( LDAP_DEBUG_ANY,
-					"be_modify failed (%d)\n", rc, 0, 0 );
-#endif
+				return 0;
 			}
+		} else {
+#ifdef NEW_LOGGING
+			LDAP_LOG( OPERATION, ERR,
+				"be_modify/be_delete failed (%d)\n", rc, 0, 0 );
+#else
+			Debug( LDAP_DEBUG_ANY,
+				"be_modify/be_delete failed (%d)\n", rc, 0, 0 );
+#endif
 		}
 
 		si->e = NULL;
@@ -1916,6 +1945,21 @@ cookie_callback(
 	return LDAP_SUCCESS;
 }
 
+static int
+dn_callback(
+	Operation*	op,
+	SlapReply*	rs
+)
+{
+	syncinfo_t *si = op->o_callback->sc_private;
+	
+	if ( rs->sr_type == REP_SEARCH ) {
+		si->syncUUID_ndn = &rs->sr_entry->e_nname;
+	}
+
+	return LDAP_SUCCESS;
+}
+
 static int
 nonpresent_callback(
 	Operation*	op,
@@ -1982,4 +2026,44 @@ null_callback(
 	return LDAP_SUCCESS;
 }
 
+
+char **
+str2clist( char **out, char *in, const char *brkstr )
+{
+	char	*str;
+	char	*s;
+	char	*lasts;
+	int	i, j;
+	const char *text;
+	char	**new;
+
+	/* find last element in list */
+	for (i = 0; out && out[i]; i++);
+	
+	/* protect the input string from strtok */
+	str = ch_strdup( in );
+
+	/* Count words in string */
+	j=1;
+	for ( s = str; *s; s++ ) {
+		if ( strchr( brkstr, *s ) != NULL ) {
+			j++;
+		}
+	}
+
+	out = ch_realloc( out, ( i + j + 1 ) * sizeof( char * ) );
+	new = out + i;
+	for ( s = ldap_pvt_strtok( str, brkstr, &lasts );
+		s != NULL;
+		s = ldap_pvt_strtok( NULL, brkstr, &lasts ) )
+	{
+		*new = ch_strdup( s );
+		new++;
+	}
+
+	*new = NULL;
+	free( str );
+	return( out );
+}
+
 #endif

servers/slapd/tools/mimic.c

@@ -263,4 +263,9 @@ void* do_syncrepl( void *ctx, void *arg )
 {
 	return NULL;
 }
+
+char** str2clist( char **out, char *in, const char *brkstr )
+{
+	return NULL;
+}
 #endif

tests/data/slapd-syncrepl-master.conf

@@ -0,0 +1,30 @@
+# $OpenLDAP$
+#
+# master slapd config -- for testing of SYNC replication
+#
+ucdata-path	./ucdata
+include ./schema/core.schema
+include ./schema/cosine.schema
+include ./schema/inetorgperson.schema
+include ./schema/openldap.schema
+#
+pidfile     ./test-db/slapd.pid
+argsfile    ./test-db/slapd.args
+
+modulepath      ../servers/slapd/back-@BACKEND@/
+@MODULELOAD@
+
+#######################################################################
+# ldbm database definitions
+#######################################################################
+
+database	@BACKEND@
+#ldbm#cachesize	0
+suffix		"o=University of Michigan,c=US"
+directory	./test-db
+rootdn		"cn=Manager,o=University of Michigan,c=US"
+rootpw		secret
+#ldbm#index		objectClass	eq
+#ldbm#index		cn,sn,uid	pres,eq,sub
+#bdb#index		objectClass	eq
+#bdb#index		cn,sn,uid	pres,eq,sub

tests/data/slapd-syncrepl-slave.conf

@@ -0,0 +1,45 @@
+# $OpenLDAP$
+#
+# slave slapd config -- for testing of SYNC replication
+#
+ucdata-path	./ucdata
+include ./schema/core.schema
+include ./schema/cosine.schema
+include ./schema/inetorgperson.schema
+include ./schema/openldap.schema
+#
+pidfile     ./test-repl/slapd.pid
+argsfile    ./test-repl/slapd.args
+
+modulepath      ../servers/slapd/back-@BACKEND@/
+@MODULELOAD@
+
+#######################################################################
+# ldbm database definitions
+#######################################################################
+
+database	@BACKEND@
+#ldbm#cachesize	0
+suffix		"o=University of Michigan,c=US"
+directory	./test-repl
+rootdn		"cn=Replica,o=University of Michigan,c=US"
+rootpw		secret
+#ldbm#index		objectClass	eq
+#ldbm#index		cn,sn,uid	pres,eq,sub
+#bdb#index		objectClass	eq
+#bdb#index		cn,sn,uid	pres,eq,sub
+
+# Don't change syncrepl spec yet
+syncrepl id=1
+		 master=ldap://localhost:9009
+		 updatedn="cn=Replica,o=University of Michigan,c=US"
+		 binddn="cn=Manager,o=University of Michigan,c=US"
+		 bindmethod=simple
+		 credentials=secret
+		 searchbase="o=University of Michigan,c=US"
+		 filter="objectClass=*"
+		 attrs="*"
+		 lastmod=req
+		 scope=sub
+		 type=refreshOnly
+		 interval=10

tests/scripts/defines.sh

@@ -12,7 +12,9 @@ PWCONF=$DATADIR/slapd-pw.conf
 ACLCONF=$DATADIR/slapd-acl.conf
 RCONF=$DATADIR/slapd-referrals.conf
 MASTERCONF=$DATADIR/slapd-repl-master.conf
+SRMASTERCONF=$DATADIR/slapd-syncrepl-master.conf
 SLAVECONF=$DATADIR/slapd-repl-slave.conf
+SRSLAVECONF=$DATADIR/slapd-syncrepl-slave.conf
 REFSLAVECONF=$DATADIR/slapd-ref-slave.conf
 SUBMASTERCONF=$DATADIR/slapd-repl-submaster.conf
 SUBSLAVECONF=$DATADIR/slapd-repl-subslave.conf

tests/scripts/test017-syncreplication

@@ -0,0 +1,212 @@
+#! /bin/sh
+# $OpenLDAP$
+
+SRCDIR="."
+if test $# -ge 1 ; then
+	SRCDIR=$1; shift
+fi
+
+. $SRCDIR/scripts/args.sh
+
+echo "running defines.sh"
+. $SRCDIR/scripts/defines.sh
+
+#
+# Test replication:
+# - start master
+# - start slave
+# - populate over ldap
+# - perform some modifies and deleted
+# - retrieve database over ldap and compare against expected results
+#
+
+echo "Cleaning up in $DBDIR..."
+rm -f $DBDIR/[!C]*
+echo "Cleaning up in $REPLDIR..."
+rm -rf $REPLDIR/[!C]*
+
+echo "Starting master slapd on TCP/IP port $PORT..."
+. $CONFFILTER $BACKEND $MONITORDB < $SRMASTERCONF > $DBCONF
+$SLAPD -f $DBCONF -h $MASTERURI -d $LVL $TIMING > $MASTERLOG 2>&1 &
+PID=$!
+if test $WAIT != 0 ; then
+    echo PID $PID
+    read foo
+fi
+
+echo "Starting slave slapd on TCP/IP port $SLAVEPORT..."
+. $CONFFILTER $BACKEND $MONITORDB < $SRSLAVECONF > $REPLCONF
+$SLAPD -f $REPLCONF -h $SLAVEURI -d $LVL $TIMING > $SLAVELOG 2>&1 &
+SLAVEPID=$!
+if test $WAIT != 0 ; then
+    echo SLAVEPID $SLAVEPID
+    read foo
+fi
+
+echo "Using ldapsearch to check that master slapd is running..."
+for i in 0 1 2 3 4 5; do
+	$LDAPSEARCH -s base -b "$MONITOR" -h $LOCALHOST -p $PORT \
+		'objectclass=*' > /dev/null 2>&1
+	RC=$?
+	if test $RC = 0 ; then
+		break
+	fi
+	echo "Waiting 5 seconds for slapd to start..."
+	sleep 5
+done
+
+echo "Using ldapsearch to check that slave slapd is running..."
+for i in 0 1 2 3 4 5; do
+	$LDAPSEARCH -s base -b "$MONITOR" -h $LOCALHOST -p $PORT \
+		'objectclass=*' > /dev/null 2>&1
+	RC=$?
+	if test $RC = 0 ; then
+		break
+	fi
+	echo "Waiting 5 seconds for slapd to start..."
+	sleep 5
+done
+
+echo "Using ldapadd to populate the master directory..."
+$LDAPADD -D "$MANAGERDN" -h $LOCALHOST -p $PORT -w $PASSWD < \
+	$LDIFORDERED > /dev/null 2>&1
+RC=$?
+if test $RC != 0 ; then
+	echo "ldapadd failed ($RC)!"
+	kill -HUP $PID $SLAVEPID
+	kill -HUP $SLURPPID
+	exit $RC
+fi
+
+echo "Waiting 30 seconds for syncrepl to receive changes..."
+sleep 30
+
+echo "Using ldapmodify to modify master directory..."
+
+#
+# Do some modifications
+#
+
+$LDAPMODIFY -v -D "$MANAGERDN" -h $LOCALHOST -p $PORT -w $PASSWD > \
+	$TESTOUT 2>&1 << EOMODS
+dn: cn=James A Jones 1, ou=Alumni Association, ou=People, o=University of Michigan, c=US
+changetype: modify
+add: drink
+drink: Orange Juice
+-
+delete: sn
+sn: Jones
+-
+add: sn
+sn: Jones
+
+dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, o=University of Michigan, c=US
+changetype: modify
+replace: drink
+drink: Iced Tea
+drink: Mad Dog 20/20
+
+dn: cn=ITD Staff,ou=Groups,o=University of Michigan,c=US
+changetype: modify
+delete: member
+member: cn=James A Jones 2, ou=Information Technology Division, ou=People, o=University of Michigan, c=US
+member: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, o=University of Michigan, c=US
+-
+add: member
+member: cn=Dorothy Stevens, ou=Alumni Association, ou=People, o=University of Michigan, c=US
+member: cn=James A Jones 1, ou=Alumni Association, ou=People, o=University of Michigan, c=US
+
+dn: cn=All Staff,ou=Groups,o=University of Michigan,c=US
+changetype: modify
+delete: description
+
+dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, o=University of Michigan, c=US
+changetype: add
+objectclass: OpenLDAPperson
+cn: Gern Jensen
+sn: Jensen
+uid: gjensen
+title: Chief Investigator, ITD
+postaladdress: ITD $ 535 W. William St $ Ann Arbor, MI 48103
+seealso: cn=All Staff, ou=Groups, o=University of Michigan, c=US
+drink: Coffee
+homepostaladdress: 844 Brown St. Apt. 4 $ Ann Arbor, MI 48104
+description: Very odd
+facsimiletelephonenumber: +1 313 555 7557
+telephonenumber: +1 313 555 8343
+mail: gjensen@mailgw.example.com
+homephone: +1 313 555 8844
+
+dn: ou=Retired, ou=People, o=University of Michigan, c=US
+changetype: add
+objectclass: organizationalUnit
+ou: Retired
+
+dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, o=University of Michigan, c=US
+changetype: add
+objectclass: OpenLDAPperson
+cn: Rosco P. Coltrane
+sn: Coltrane
+uid: rosco
+
+dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, o=University of Michigan, c=US
+changetype: modrdn
+newrdn: cn=Rosco P. Coltrane
+deleteoldrdn: 1
+newsuperior: ou=Retired, ou=People, o=University of Michigan, c=US
+
+dn: cn=James A Jones 2, ou=Information Technology Division, ou=People, o=University of Michigan, c=US
+changetype: delete
+
+EOMODS
+
+echo "Waiting 30 seconds for syncrepl to receive changes..."
+sleep 30
+
+echo "Using ldapsearch to read all the entries from the master..."
+$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT \
+	'objectclass=*' > $MASTEROUT 2>&1
+RC=$?
+
+if test $RC != 0 ; then
+	echo "ldapsearch failed at master ($RC)!"
+	kill -HUP $PID $SLAVEPID
+	kill -HUP $SLURPPID
+	exit $RC
+fi
+
+echo "Using ldapsearch to read all the entries from the slave..."
+$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $SLAVEPORT \
+	'objectclass=*' > $SLAVEOUT 2>&1
+RC=$?
+
+if test $RC != 0 ; then
+	echo "ldapsearch failed at slave ($RC)!"
+	kill -HUP $PID $SLAVEPID
+	kill -HUP $SLURPPID
+	exit $RC
+fi
+
+kill -HUP $PID $SLAVEPID
+kill -HUP $SLURPPID
+
+SEARCHOUT=$MASTEROUT
+LDIF=$SLAVEOUT
+
+echo "Filtering ldapsearch results..."
+. $LDIFFILTER < $SEARCHOUT > $SEARCHFLT
+echo "Filtering original ldif used to create database..."
+. $LDIFFILTER < $LDIF > $LDIFFLT
+
+echo "Comparing retrieved entries from master and slave..."
+$CMP $SEARCHFLT $LDIFFLT > $CMPOUT
+
+if test $? != 0 ; then
+	echo "test failed - master and slave databases differ"
+	exit 1
+fi
+
+echo ">>>>> Test succeeded"
+
+
+exit 0