Move shmem allocator's fields from PGShmemHeader to its own struct

For readability. It was a slight modularity violation to have fields
in PGShmemHeader that were only used by the allocator code in
shmem.c. And it was inconsistent that ShmemLock was nevertheless not
stored there. Moving all the allocator-related fields to a separate
struct makes it more consistent and modular, and removes the need to
allocate and pass ShmemLock separately via BackendParameters.

Merge InitShmemAccess() and InitShmemAllocation() into a single
function that initializes the struct when called from postmaster, and
when called from backends in EXEC_BACKEND mode, re-establishes the
global variables. That's similar to all the *ShmemInit() functions
that we have.

Co-authored-by: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com>
Discussion: https://www.postgresql.org/message-id/CAExHW5uNRB9oT4pdo54qAo025MXFX4MfYrD9K15OCqe-ExnNvg@mail.gmail.com

src/backend/port/sysv_shmem.c

@@ -855,7 +855,7 @@ PGSharedMemoryCreate(Size size,
 	 * Initialize space allocation status for segment.
 	 */
 	hdr->totalsize = size;
-	hdr->freeoffset = MAXALIGN(sizeof(PGShmemHeader));
+	hdr->content_offset = MAXALIGN(sizeof(PGShmemHeader));
 	*shim = hdr;
 
 	/* Save info for possible future use */

src/backend/port/win32_shmem.c

@@ -389,7 +389,7 @@ retry:
 	 * Initialize space allocation status for segment.
 	 */
 	hdr->totalsize = size;
-	hdr->freeoffset = MAXALIGN(sizeof(PGShmemHeader));
+	hdr->content_offset = MAXALIGN(sizeof(PGShmemHeader));
 	hdr->dsm_control = 0;
 
 	/* Save info for possible future use */

src/backend/postmaster/launch_backend.c

@@ -96,7 +96,6 @@ typedef struct
 	HANDLE		UsedShmemSegID;
 #endif
 	void	   *UsedShmemSegAddr;
-	slock_t    *ShmemLock;
 #ifdef USE_INJECTION_POINTS
 	struct InjectionPointsCtl *ActiveInjectionPoints;
 #endif
@@ -676,7 +675,7 @@ SubPostmasterMain(int argc, char *argv[])
 
 	/* Restore basic shared memory pointers */
 	if (UsedShmemSegAddr != NULL)
-		InitShmemAccess(UsedShmemSegAddr);
+		InitShmemAllocator(UsedShmemSegAddr);
 
 	/*
 	 * Run the appropriate Main function
@@ -724,8 +723,6 @@ save_backend_variables(BackendParameters *param,
 	param->UsedShmemSegID = UsedShmemSegID;
 	param->UsedShmemSegAddr = UsedShmemSegAddr;
 
-	param->ShmemLock = ShmemLock;
-
 #ifdef USE_INJECTION_POINTS
 	param->ActiveInjectionPoints = ActiveInjectionPoints;
 #endif
@@ -986,8 +983,6 @@ restore_backend_variables(BackendParameters *param)
 	UsedShmemSegID = param->UsedShmemSegID;
 	UsedShmemSegAddr = param->UsedShmemSegAddr;
 
-	ShmemLock = param->ShmemLock;
-
 #ifdef USE_INJECTION_POINTS
 	ActiveInjectionPoints = param->ActiveInjectionPoints;
 #endif

src/backend/storage/ipc/ipci.c

@@ -212,12 +212,10 @@ CreateSharedMemoryAndSemaphores(void)
 	Assert(strcmp("unknown",
 				  GetConfigOption("huge_pages_status", false, false)) != 0);
 
-	InitShmemAccess(seghdr);
-
 	/*
 	 * Set up shared memory allocation mechanism
 	 */
-	InitShmemAllocation();
+	InitShmemAllocator(seghdr);
 
 	/* Initialize subsystems */
 	CreateOrAttachShmemStructs();

src/backend/storage/ipc/shmem.c

@@ -76,20 +76,33 @@
 #include "storage/spin.h"
 #include "utils/builtins.h"
 
+/*
+ * This is the first data structure stored in the shared memory segment, at
+ * the offset that PGShmemHeader->content_offset points to.  Allocations by
+ * ShmemAlloc() are carved out of the space after this.
+ *
+ * For the base pointer and the total size of the shmem segment, we rely on
+ * the PGShmemHeader.
+ */
+typedef struct ShmemAllocatorData
+{
+	Size		free_offset;	/* offset to first free space from ShmemBase */
+	HTAB	   *index;			/* copy of ShmemIndex */
+
+	/* protects shared memory and LWLock allocation */
+	slock_t		shmem_lock;
+} ShmemAllocatorData;
+
 static void *ShmemAllocRaw(Size size, Size *allocated_size);
-static void *ShmemAllocUnlocked(Size size);
 
 /* shared memory global variables */
 
 static PGShmemHeader *ShmemSegHdr;	/* shared mem segment header */
-
 static void *ShmemBase;			/* start address of shared memory */
-
 static void *ShmemEnd;			/* end+1 address of shared memory */
 
-slock_t    *ShmemLock;			/* spinlock for shared memory and LWLock
+static ShmemAllocatorData *ShmemAllocator;
-								 * allocation */
+slock_t    *ShmemLock;			/* points to ShmemAllocator->shmem_lock */
-
 static HTAB *ShmemIndex = NULL; /* primary index hashtable for shmem */
 
 /* To get reliable results for NUMA inquiry we need to "touch pages" once */
@@ -98,49 +111,64 @@ static bool firstNumaTouch = true;
 Datum		pg_numa_available(PG_FUNCTION_ARGS);
 
 /*
- *	InitShmemAccess() --- set up basic pointers to shared memory.
+ *	InitShmemAllocator() --- set up basic pointers to shared memory.
+ *
+ * Called at postmaster or stand-alone backend startup, to initialize the
+ * allocator's data structure in the shared memory segment.  In EXEC_BACKEND,
+ * this is also called at backend startup, to set up pointers to the shared
+ * memory areas.
  */
 void
-InitShmemAccess(PGShmemHeader *seghdr)
+InitShmemAllocator(PGShmemHeader *seghdr)
 {
+	Assert(seghdr != NULL);
+
+	/*
+	 * We assume the pointer and offset are MAXALIGN.  Not a hard requirement,
+	 * but it's true today and keeps the math below simpler.
+	 */
+	Assert(seghdr == (void *) MAXALIGN(seghdr));
+	Assert(seghdr->content_offset == MAXALIGN(seghdr->content_offset));
+
 	ShmemSegHdr = seghdr;
 	ShmemBase = seghdr;
 	ShmemEnd = (char *) ShmemBase + seghdr->totalsize;
-}
 
-/*
+#ifndef EXEC_BACKEND
- *	InitShmemAllocation() --- set up shared-memory space allocation.
+	Assert(!IsUnderPostmaster);
- *
+#endif
- * This should be called only in the postmaster or a standalone backend.
+	if (IsUnderPostmaster)
- */
+	{
-void
+		PGShmemHeader *shmhdr = ShmemSegHdr;
-InitShmemAllocation(void)
-{
-	PGShmemHeader *shmhdr = ShmemSegHdr;
-	char	   *aligned;
 
-	Assert(shmhdr != NULL);
+		ShmemAllocator = (ShmemAllocatorData *) ((char *) shmhdr + shmhdr->content_offset);
+		ShmemLock = &ShmemAllocator->shmem_lock;
+	}
+	else
+	{
+		Size		offset;
 
-	/*
+		/*
-	 * Initialize the spinlock used by ShmemAlloc.  We must use
+		 * Allocations after this point should go through ShmemAlloc, which
-	 * ShmemAllocUnlocked, since obviously ShmemAlloc can't be called yet.
+		 * expects to allocate everything on cache line boundaries.  Make sure
-	 */
+		 * the first allocation begins on a cache line boundary.
-	ShmemLock = (slock_t *) ShmemAllocUnlocked(sizeof(slock_t));
+		 */
+		offset = CACHELINEALIGN(seghdr->content_offset + sizeof(ShmemAllocatorData));
+		if (offset > seghdr->totalsize)
+			ereport(ERROR,
+					(errcode(ERRCODE_OUT_OF_MEMORY),
+					 errmsg("out of shared memory (%zu bytes requested)",
+							offset)));
 
-	SpinLockInit(ShmemLock);
+		ShmemAllocator = (ShmemAllocatorData *) ((char *) seghdr + seghdr->content_offset);
 
-	/*
+		SpinLockInit(&ShmemAllocator->shmem_lock);
-	 * Allocations after this point should go through ShmemAlloc, which
+		ShmemLock = &ShmemAllocator->shmem_lock;
-	 * expects to allocate everything on cache line boundaries.  Make sure the
+		ShmemAllocator->free_offset = offset;
-	 * first allocation begins on a cache line boundary.
+		/* ShmemIndex can't be set up yet (need LWLocks first) */
-	 */
+		ShmemAllocator->index = NULL;
-	aligned = (char *)
+		ShmemIndex = (HTAB *) NULL;
-		(CACHELINEALIGN((((char *) shmhdr) + shmhdr->freeoffset)));
+	}
-	shmhdr->freeoffset = aligned - (char *) shmhdr;
-
-	/* ShmemIndex can't be set up yet (need LWLocks first) */
-	shmhdr->index = NULL;
-	ShmemIndex = (HTAB *) NULL;
 }
 
 /*
@@ -209,13 +237,13 @@ ShmemAllocRaw(Size size, Size *allocated_size)
 
 	SpinLockAcquire(ShmemLock);
 
-	newStart = ShmemSegHdr->freeoffset;
+	newStart = ShmemAllocator->free_offset;
 
 	newFree = newStart + size;
 	if (newFree <= ShmemSegHdr->totalsize)
 	{
 		newSpace = (char *) ShmemBase + newStart;
-		ShmemSegHdr->freeoffset = newFree;
+		ShmemAllocator->free_offset = newFree;
 	}
 	else
 		newSpace = NULL;
@@ -228,45 +256,6 @@ ShmemAllocRaw(Size size, Size *allocated_size)
 	return newSpace;
 }
 
-/*
- * ShmemAllocUnlocked -- allocate max-aligned chunk from shared memory
- *
- * Allocate space without locking ShmemLock.  This should be used for,
- * and only for, allocations that must happen before ShmemLock is ready.
- *
- * We consider maxalign, rather than cachealign, sufficient here.
- */
-static void *
-ShmemAllocUnlocked(Size size)
-{
-	Size		newStart;
-	Size		newFree;
-	void	   *newSpace;
-
-	/*
-	 * Ensure allocated space is adequately aligned.
-	 */
-	size = MAXALIGN(size);
-
-	Assert(ShmemSegHdr != NULL);
-
-	newStart = ShmemSegHdr->freeoffset;
-
-	newFree = newStart + size;
-	if (newFree > ShmemSegHdr->totalsize)
-		ereport(ERROR,
-				(errcode(ERRCODE_OUT_OF_MEMORY),
-				 errmsg("out of shared memory (%zu bytes requested)",
-						size)));
-	ShmemSegHdr->freeoffset = newFree;
-
-	newSpace = (char *) ShmemBase + newStart;
-
-	Assert(newSpace == (void *) MAXALIGN(newSpace));
-
-	return newSpace;
-}
-
 /*
  * ShmemAddrIsValid -- test if an address refers to shared memory
  *
@@ -395,16 +384,14 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
 
 	if (!ShmemIndex)
 	{
-		PGShmemHeader *shmemseghdr = ShmemSegHdr;
-
 		/* Must be trying to create/attach to ShmemIndex itself */
 		Assert(strcmp(name, "ShmemIndex") == 0);
 
 		if (IsUnderPostmaster)
 		{
 			/* Must be initializing a (non-standalone) backend */
-			Assert(shmemseghdr->index != NULL);
+			Assert(ShmemAllocator->index != NULL);
-			structPtr = shmemseghdr->index;
+			structPtr = ShmemAllocator->index;
 			*foundPtr = true;
 		}
 		else
@@ -417,9 +404,9 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
 			 * index has been initialized.  This should be OK because no other
 			 * process can be accessing shared memory yet.
 			 */
-			Assert(shmemseghdr->index == NULL);
+			Assert(ShmemAllocator->index == NULL);
 			structPtr = ShmemAlloc(size);
-			shmemseghdr->index = structPtr;
+			ShmemAllocator->index = structPtr;
 			*foundPtr = false;
 		}
 		LWLockRelease(ShmemIndexLock);
@@ -553,15 +540,15 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS)
 	/* output shared memory allocated but not counted via the shmem index */
 	values[0] = CStringGetTextDatum("<anonymous>");
 	nulls[1] = true;
-	values[2] = Int64GetDatum(ShmemSegHdr->freeoffset - named_allocated);
+	values[2] = Int64GetDatum(ShmemAllocator->free_offset - named_allocated);
 	values[3] = values[2];
 	tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
 
 	/* output as-of-yet unused shared memory */
 	nulls[0] = true;
-	values[1] = Int64GetDatum(ShmemSegHdr->freeoffset);
+	values[1] = Int64GetDatum(ShmemAllocator->free_offset);
 	nulls[1] = false;
-	values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemSegHdr->freeoffset);
+	values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemAllocator->free_offset);
 	values[3] = values[2];
 	tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
 

src/include/storage/pg_shmem.h

@@ -32,9 +32,9 @@ typedef struct PGShmemHeader	/* standard header for all Postgres shmem */
 #define PGShmemMagic  679834894
 	pid_t		creatorPID;		/* PID of creating process (set but unread) */
 	Size		totalsize;		/* total size of segment */
-	Size		freeoffset;		/* offset to first free space */
+	Size		content_offset; /* offset to the data, i.e. size of this
+								 * header */
 	dsm_handle	dsm_control;	/* ID of dynamic shared memory control seg */
-	void	   *index;			/* pointer to ShmemIndex table */
 #ifndef WIN32					/* Windows doesn't have useful inode#s */
 	dev_t		device;			/* device data directory is on */
 	ino_t		inode;			/* inode number of data directory */

src/include/storage/shmem.h

@@ -29,8 +29,7 @@
 extern PGDLLIMPORT slock_t *ShmemLock;
 typedef struct PGShmemHeader PGShmemHeader; /* avoid including
 											 * storage/pg_shmem.h here */
-extern void InitShmemAccess(PGShmemHeader *seghdr);
+extern void InitShmemAllocator(PGShmemHeader *seghdr);
-extern void InitShmemAllocation(void);
 extern void *ShmemAlloc(Size size);
 extern void *ShmemAllocNoError(Size size);
 extern bool ShmemAddrIsValid(const void *addr);

src/tools/pgindent/typedefs.list

@@ -2804,6 +2804,7 @@ SharedTypmodTableEntry
 Sharedsort
 ShellTypeInfo
 ShippableCacheEntry
+ShmemAllocatorData
 ShippableCacheKey
 ShmemIndexEnt
 ShutdownForeignScan_function