Fix unbounded recursive handling of SSL/GSS in ProcessStartupPacket()

The handling of SSL and GSS negotiation messages in
ProcessStartupPacket() could cause a recursion of the backend,
ultimately crashing the server as the negotiation attempts were not
tracked across multiple calls processing startup packets.

A malicious client could therefore alternate rejected SSL and GSS
requests indefinitely, each adding a stack frame, until the backend
crashed with a stack overflow, taking down a server.

This commit addresses this issue by modifying ProcessStartupPacket() so
as processed negotiation attempts are tracked, preventing infinite
recursive attempts.  A TAP test is added to check this problem, where
multiple SSL and GSS negotiated attempts are stacked.

Reported-by: Calif.io in collaboration with Claude and Anthropic
Research
Author: Michael Paquier <michael@paquier.xyz>
Reviewed-by: Daniel Gustafsson <daniel@yesql.se>
Security: CVE-2026-6479
Backpatch-through: 14

src/backend/tcop/backend_startup.c

@@ -496,6 +496,7 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 	ProtocolVersion proto;
 	MemoryContext oldcontext;
 
+retry:
 	pq_startmsgread();
 
 	/*
@@ -616,6 +617,7 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 #endif
 
 		pfree(buf);
+		buf = NULL;
 
 		/*
 		 * At this point we should have no data already buffered.  If we do,
@@ -634,7 +636,16 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 		 * another SSL negotiation request, and a GSS request should only
 		 * follow if SSL was rejected (client may negotiate in either order)
 		 */
-		return ProcessStartupPacket(port, true, SSLok == 'S');
+		ssl_done = true;
+		if (SSLok == 'S')
+		{
+			/*
+			 * We are done with SSL and negotiated correctly, so consider the
+			 * same for GSS.
+			 */
+			gss_done = true;
+		}
+		goto retry;
 	}
 	else if (proto == NEGOTIATE_GSS_CODE && !gss_done)
 	{
@@ -672,6 +683,7 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 #endif
 
 		pfree(buf);
+		buf = NULL;
 
 		/*
 		 * At this point we should have no data already buffered.  If we do,
@@ -690,7 +702,16 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done)
 		 * another GSS negotiation request, and an SSL request should only
 		 * follow if GSS was rejected (client may negotiate in either order)
 		 */
-		return ProcessStartupPacket(port, GSSok == 'G', true);
+		gss_done = true;
+		if (GSSok == 'G')
+		{
+			/*
+			 * We are done with GSS and negotiated correctly, so consider the
+			 * same for SSL.
+			 */
+			ssl_done = true;
+		}
+		goto retry;
 	}
 
 	/* Could add additional special packet types here */

src/test/postmaster/meson.build

@@ -9,6 +9,7 @@ tests += {
       't/001_basic.pl',
       't/002_connection_limits.pl',
       't/003_start_stop.pl',
+      't/004_negotiate.pl',
     ],
   },
 }

src/test/postmaster/t/004_negotiate.pl

@@ -0,0 +1,83 @@
+# Copyright (c) 2026, PostgreSQL Global Development Group
+
+# Test the negotiation of combined SSL and GSS requests.  This test
+# relies on both SSL and GSS requests to be rejected first, followed
+# by more requests.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+use Time::HiRes qw(usleep);
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+$node->init;
+$node->append_conf('postgresql.conf', "log_min_messages = debug2");
+$node->append_conf('postgresql.conf',
+	"log_connections = 'receipt,authentication,authorization'");
+$node->append_conf('postgresql.conf', 'trace_connection_negotiation=on');
+$node->start;
+
+if (!$node->raw_connect_works())
+{
+	plan skip_all => "this test requires working raw_connect()";
+}
+
+my $sock = $node->raw_connect();
+
+# SSLRequest: packet length followed by NEGOTIATE_SSL_CODE.
+my $ssl_request = pack("Nnn", 8, 1234, 5679);
+
+# GSSENCRequest: packet length followed by NEGOTIATE_GSS_CODE.
+my $gss_request = pack("Nnn", 8, 1234, 5680);
+
+# Send SSLRequest, reject or bypass.
+$sock->send($ssl_request);
+my $reply = "";
+$sock->recv($reply, 1);
+if ($reply ne 'N')
+{
+	$sock->close();
+	plan skip_all =>
+	  "server accepted SSL; test requires SSL to be rejected";
+}
+
+# Send GSSENCRequest, reject or bypass test.
+$sock->send($gss_request);
+$reply = "";
+$sock->recv($reply, 1);
+if ($reply ne 'N')
+{
+	$sock->close();
+	plan skip_all =>
+	  "server accepted GSS; test requires GSS to be rejected";
+}
+
+my $log_offset = -s $node->logfile;
+
+# Send a second SSLRequest, now that we know that both SSL and GSS have
+# been rejected for this connection.  We are done with both requests, so
+# extra requests will be rejected and fail with an invalid protocol
+# version, and the connection should be closed by the server.
+$sock->send($ssl_request);
+
+# Try to read a response, there should be nothing, and certainly not an
+# extra 'N' message indicating a rejection.
+$reply = "";
+my $bytes = $sock->recv($reply, 1024);
+isnt($reply, 'N',
+	"server does not re-enter SSL negotiation after SSL+GSS were both tried");
+
+$sock->close();
+$node->wait_for_log(qr/FATAL: .* unsupported frontend protocol 1234.5679/,
+					$log_offset);
+
+# Check extra connection with a simple query.
+my $result = $node->safe_psql('postgres', 'select 1;');
+is($result, '1', 'server able to accept connection');
+ok($node->is_alive(), "server still running after negotiation attempt");
+
+$node->stop;
+
+done_testing();