diff --git a/CHANGES b/CHANGES
index 41928f2b81..eedf0ed9dc 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,6 @@
+6003.	[bug]		Fix an inheritance bug when setting the port on
+			remote servers in configuration. [GL #3627]
+
 6002.	[bug]		Fix a resolver prefetch bug when the record's TTL value
 			is equal to the configured prefetch eligibility value,
 			but the record was erroneously not treated as eligible
diff --git a/bin/named/config.c b/bin/named/config.c
index 7743645eb3..4c956a72a5 100644
--- a/bin/named/config.c
+++ b/bin/named/config.c
@@ -881,18 +881,22 @@ resume:
 			goto cleanup;
 		}
 
-		/* Set the default port or tls-port */
-		if (port == 0) {
-			if (tlss[i] != NULL) {
-				port = def_tlsport;
-			} else {
-				port = def_port;
+		/* If the port is unset, take it from one of the upper levels */
+		if (isc_sockaddr_getport(&addrs[i]) == 0) {
+			in_port_t addr_port = port;
+
+			/* If unset, use the default port or tls-port */
+			if (addr_port == 0) {
+				if (tlss[i] != NULL) {
+					addr_port = def_tlsport;
+				} else {
+					addr_port = def_port;
+				}
 			}
+
+			isc_sockaddr_setport(&addrs[i], addr_port);
 		}
 
-		if (isc_sockaddr_getport(&addrs[i]) == 0) {
-			isc_sockaddr_setport(&addrs[i], port);
-		}
 		i++;
 	}
 	if (pushed != 0) {
diff --git a/bin/tests/system/xfer/clean.sh b/bin/tests/system/xfer/clean.sh
index 2851553009..bf5016e8f9 100644
--- a/bin/tests/system/xfer/clean.sh
+++ b/bin/tests/system/xfer/clean.sh
@@ -24,10 +24,11 @@ rm -f axfr.out
 rm -f dig.out.*
 rm -f ns*/managed-keys.bind*
 rm -f ns*/named.lock
+rm -f ns1/dot-fallback.db
 rm -f ns1/edns-expire.db
 rm -f ns1/ixfr-too-big.db ns1/ixfr-too-big.db.jnl
 rm -f ns1/sec.db ns2/sec.db
-rm -f ns2/example.db ns2/tsigzone.db ns2/example.db.jnl
+rm -f ns2/example.db ns2/tsigzone.db ns2/example.db.jnl ns2/dot-fallback.db
 rm -f ns2/mapped.db
 rm -f ns3/example.bk ns3/xfer-stats.bk ns3/tsigzone.bk ns3/example.bk.jnl
 rm -f ns3/mapped.bk
diff --git a/bin/tests/system/xfer/dig3.good b/bin/tests/system/xfer/dig3.good
new file mode 100644
index 0000000000..d58c4216a3
--- /dev/null
+++ b/bin/tests/system/xfer/dig3.good
@@ -0,0 +1,6 @@
+dot-fallback.		5	IN	SOA	ns1.dot-fallback. hostmaster.dot-fallback. 1 3600 3600 3600 3600
+dot-fallback.		5	IN	NS	ns1.dot-fallback.
+a01.dot-fallback.	5	IN	A	1.1.1.1
+a02.dot-fallback.	5	IN	A	255.255.255.255
+ns1.dot-fallback.	5	IN	A	10.53.0.1
+dot-fallback.		5	IN	SOA	ns1.dot-fallback. hostmaster.dot-fallback. 1 3600 3600 3600 3600
diff --git a/bin/tests/system/xfer/ns1/dot-fallback.db.in b/bin/tests/system/xfer/ns1/dot-fallback.db.in
new file mode 100644
index 0000000000..997f5baf87
--- /dev/null
+++ b/bin/tests/system/xfer/ns1/dot-fallback.db.in
@@ -0,0 +1,19 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; SPDX-License-Identifier: MPL-2.0
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0.  If a copy of the MPL was not distributed with this
+; file, you can obtain one at https://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+$TTL 5
+
+@			IN SOA	ns1 hostmaster 1 3600 3600 3600 3600
+@			NS	ns1
+ns1			A	10.53.0.1
+a01			A	1.1.1.1
+a02			A	255.255.255.255
+
diff --git a/bin/tests/system/xfer/ns1/named.conf.in b/bin/tests/system/xfer/ns1/named.conf.in
index 3ff6cdf0c4..1cd72719f8 100644
--- a/bin/tests/system/xfer/ns1/named.conf.in
+++ b/bin/tests/system/xfer/ns1/named.conf.in
@@ -59,3 +59,8 @@ zone "xfer-stats" {
 	type primary;
 	file "xfer-stats.db";
 };
+
+zone "dot-fallback" {
+	type primary;
+	file "dot-fallback.db";
+};
diff --git a/bin/tests/system/xfer/ns2/named.conf.in b/bin/tests/system/xfer/ns2/named.conf.in
index 642b4bc15f..e6fb0c69b7 100644
--- a/bin/tests/system/xfer/ns2/named.conf.in
+++ b/bin/tests/system/xfer/ns2/named.conf.in
@@ -72,3 +72,12 @@ zone "mapped" {
 	masterfile-format text;
 	primaries { 10.53.0.100; };
 };
+
+zone "dot-fallback" {
+	type secondary;
+	file "dot-fallback.db";
+	primaries {
+		10.53.0.1 tls ephemeral;
+		10.53.0.1;
+	};
+};
diff --git a/bin/tests/system/xfer/setup.sh b/bin/tests/system/xfer/setup.sh
index d823f2d8b0..092d798799 100644
--- a/bin/tests/system/xfer/setup.sh
+++ b/bin/tests/system/xfer/setup.sh
@@ -32,6 +32,8 @@ copy_setports ns8/named.conf.in ns8/named.conf
 
 copy_setports ns4/named.conf.base ns4/named.conf
 
+cp ns1/dot-fallback.db.in ns1/dot-fallback.db
+
 cp ns2/sec.db.in ns2/sec.db
 touch -t 200101010000 ns2/sec.db
 
diff --git a/bin/tests/system/xfer/tests.sh b/bin/tests/system/xfer/tests.sh
index 9c6fcba974..2aaf4c015d 100755
--- a/bin/tests/system/xfer/tests.sh
+++ b/bin/tests/system/xfer/tests.sh
@@ -35,13 +35,24 @@ tmp=0
 # Spin to allow the zone to transfer.
 #
 wait_for_xfer () {
-	$DIG $DIGOPTS example. @10.53.0.3 axfr > dig.out.ns3.test$n || return 1
-	grep "^;" dig.out.ns3.test$n > /dev/null && return 1
+	ZONE=$1
+	SERVER=$2
+	$DIG $DIGOPTS $ZONE @$SERVER axfr > dig.out.test$n || return 1
+	grep "^;" dig.out.test$n > /dev/null && return 1
 	return 0
 }
-retry_quiet 25 wait_for_xfer || tmp=1
-grep "^;" dig.out.ns3.test$n | cat_i
-digcomp dig1.good dig.out.ns3.test$n || tmp=1
+retry_quiet 25 wait_for_xfer example. 10.53.0.3 || tmp=1
+grep "^;" dig.out.test$n | cat_i
+digcomp dig1.good dig.out.test$n || tmp=1
+if test $tmp != 0 ; then echo_i "failed"; fi
+status=$((status+tmp))
+
+n=$((n+1))
+echo_i "testing zone transfer functionality (fallback to DNS after DoT failed) ($n)"
+tmp=0
+retry_quiet 25 wait_for_xfer dot-fallback. 10.53.0.2 || tmp=1
+grep "^;" dig.out.test$n | cat_i
+digcomp dig3.good dig.out.test$n || tmp=1
 if test $tmp != 0 ; then echo_i "failed"; fi
 status=$((status+tmp))
 
diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst
index abbae59fcf..a501bfe4dc 100644
--- a/doc/notes/notes-current.rst
+++ b/doc/notes/notes-current.rst
@@ -50,3 +50,7 @@ Bug Fixes
 - In certain resolution scenarios quotas could be erroneously reached for
   servers, including the configured forwarders, resulting in SERVFAIL answers
   sent to the clients. This has been fixed. :gl:`#3598`
+
+- The port in remote servers such as in :any:`primaries` and
+  :any:`parental-agents` could be wrongly configured because of an inheritance
+  bug. :gl:`#3627`