From 12cbd8fab5889ee909edac278e9866f20cfb1f9f Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Fri, 2 Aug 2024 19:34:06 +0900
Subject: [PATCH v34 12/16] Make smgrdounlinkall accept forknumbers

In a subsequent patch, crash-safe file deletion on a per-fork basis
will be required. To facilitate this, modify smgrdounlinkall(), which
efficiently removes multiple files, to accept fork numbers.
---
 src/backend/catalog/storage.c       |  2 +-
 src/backend/storage/buffer/bufmgr.c | 92 ++++++++++++++++++++++++-----
 src/backend/storage/smgr/md.c       |  2 +-
 src/backend/storage/smgr/smgr.c     | 28 ++++++---
 src/backend/utils/cache/relcache.c  |  2 +-
 src/include/common/relpath.h        | 11 ++++
 src/include/storage/bufmgr.h        |  2 +-
 src/include/storage/smgr.h          |  3 +-
 8 files changed, 114 insertions(+), 28 deletions(-)

diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c
index a00c59a274..71ae5f9b08 100644
--- a/src/backend/catalog/storage.c
+++ b/src/backend/catalog/storage.c
@@ -783,7 +783,7 @@ smgrDoPendingDeletes(bool isCommit)
 
 	if (nrels > 0)
 	{
-		smgrdounlinkall(srels, nrels, false);
+		smgrdounlinkall(srels, NULL, nrels, false);
 
 		for (int i = 0; i < nrels; i++)
 			smgrclose(srels[i]);
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 6de8e00bdc..3eb4a0c792 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -164,6 +164,16 @@ struct copy_storage_using_buffer_read_stream_private
 	BlockNumber nblocks;
 };
 
+/*
+ * Helper struct for handling RelFileNOde and ForkNumber together in
+ * DropRelationsAllBuffers.
+ */
+typedef struct RelFileForks
+{
+	RelFileLocator rloc;	 /* key member for qsort */
+	ForkBitmap	   forks;	 /* fork number in bitmap */
+} RelFileForks;
+
 /*
  * Callback function to get next block for read stream object used in
  * RelationCopyStorageUsingBuffer() function.
@@ -4498,24 +4508,32 @@ CheckIfPersistenceChanged(void)
  *		This function removes from the buffer pool all the pages of all
  *		forks of the specified relations.  It's equivalent to calling
  *		DropRelationBuffers once per fork per relation with firstDelBlock = 0.
+ *		The additional parameter forks is used to identify forks if
+ *		provided.
  *		--------------------------------------------------------------------
  */
 void
-DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
+DropRelationsAllBuffers(SMgrRelation *smgr_reln, ForkBitmap *pforks,
+						int nlocators)
 {
 	int			i;
 	int			n = 0;
 	SMgrRelation *rels;
 	BlockNumber (*block)[MAX_FORKNUM + 1];
 	uint64		nBlocksToInvalidate = 0;
-	RelFileLocator *locators;
+	ForkBitmap  *forks = NULL;
+	RelFileForks *locators;
 	bool		cached = true;
 	bool		use_bsearch;
 
 	if (nlocators == 0)
 		return;
 
-	rels = palloc(sizeof(SMgrRelation) * nlocators);	/* non-local relations */
+	/* storages for non-local relations */
+	rels = palloc(sizeof(SMgrRelation) * nlocators);
+
+	if (pforks)
+		forks = palloc(sizeof(ForkBitmap) * nlocators);
 
 	/* If it's a local relation, it's localbuf.c's problem. */
 	for (i = 0; i < nlocators; i++)
@@ -4526,7 +4544,12 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 				DropRelationAllLocalBuffers(smgr_reln[i]->smgr_rlocator.locator);
 		}
 		else
-			rels[n++] = smgr_reln[i];
+		{
+			rels[n] = smgr_reln[i];
+			if (forks)
+				forks[n] = pforks[i];
+			n++;
+		}
 	}
 
 	/*
@@ -4536,6 +4559,10 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 	if (n == 0)
 	{
 		pfree(rels);
+
+		if (forks)
+			pfree(forks);
+
 		return;
 	}
 
@@ -4554,6 +4581,13 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 	{
 		for (int j = 0; j <= MAX_FORKNUM; j++)
 		{
+			/* Consider only the specified fork, if provided. */
+			if (forks && !FORKBITMAP_ISSET(forks[i], j))
+			{
+				block[i][j] = InvalidBlockNumber;
+				continue;
+			}
+
 			/* Get the number of blocks for a relation's fork. */
 			block[i][j] = smgrnblocks_cached(rels[i], j);
 
@@ -4581,7 +4615,7 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 		{
 			for (int j = 0; j <= MAX_FORKNUM; j++)
 			{
-				/* ignore relation forks that doesn't exist */
+				/* ignore relation forks that doesn't exist or is ignored */
 				if (!BlockNumberIsValid(block[i][j]))
 					continue;
 
@@ -4597,9 +4631,13 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 	}
 
 	pfree(block);
-	locators = palloc(sizeof(RelFileLocator) * n);	/* non-local relations */
+	locators = palloc(sizeof(RelFileForks) * n);	/* non-local relations */
+
 	for (i = 0; i < n; i++)
-		locators[i] = rels[i]->smgr_rlocator.locator;
+	{
+		locators[i].rloc  = rels[i]->smgr_rlocator.locator;
+		locators[i].forks = (forks ? forks[i] : FORKBITMAP_ALLFORKS());
+	}
 
 	/*
 	 * For low number of relations to drop just use a simple walk through, to
@@ -4609,13 +4647,34 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 	 */
 	use_bsearch = n > RELS_BSEARCH_THRESHOLD;
 
-	/* sort the list of rlocators if necessary */
-	if (use_bsearch)
-		qsort(locators, n, sizeof(RelFileLocator), rlocator_comparator);
+	/*
+	 * Sort and compress the list of RelFileForks if necessary. We believe the
+	 * caller passed unique rlocators if forks are not specified.
+	 */
+	if (use_bsearch || forks)
+	{
+		int j = 0;
+
+		qsort(locators, n, sizeof(RelFileForks), rlocator_comparator);
+
+		/*
+		 * Now the list is in rlocator increasing order, compress the list by
+		 * merging fork bitmaps so that all elements have unique rlocators.
+		 */
+		for (i = 1 ; i < n ; i++)
+		{
+			if (RelFileLocatorEquals(locators[j].rloc, locators[i].rloc))
+				locators[j].forks |= locators[i].forks;
+			else
+				locators[++j] = locators[i];
+		}
+
+		n = j + 1;
+	}
 
 	for (i = 0; i < NBuffers; i++)
 	{
-		RelFileLocator *rlocator = NULL;
+		RelFileForks *rlocator = NULL;
 		BufferDesc *bufHdr = GetBufferDescriptor(i);
 		uint32		buf_state;
 
@@ -4630,7 +4689,8 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 
 			for (j = 0; j < n; j++)
 			{
-				if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j]))
+				if (BufTagMatchesRelFileLocator(&bufHdr->tag,
+												&locators[j].rloc))
 				{
 					rlocator = &locators[j];
 					break;
@@ -4643,16 +4703,18 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
 
 			locator = BufTagGetRelFileLocator(&bufHdr->tag);
 			rlocator = bsearch((const void *) &(locator),
-							   locators, n, sizeof(RelFileLocator),
+							   locators, n, sizeof(RelFileForks),
 							   rlocator_comparator);
 		}
 
 		/* buffer doesn't belong to any of the given relfilelocators; skip it */
-		if (rlocator == NULL)
+		if (rlocator == NULL ||
+			!FORKBITMAP_ISSET(rlocator->forks, BufTagGetForkNum(&bufHdr->tag)))
 			continue;
 
 		buf_state = LockBufHdr(bufHdr);
-		if (BufTagMatchesRelFileLocator(&bufHdr->tag, rlocator))
+		if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator->rloc) &&
+			FORKBITMAP_ISSET(rlocator->forks, BufTagGetForkNum(&bufHdr->tag)))
 			InvalidateBuffer(bufHdr);	/* releases spinlock */
 		else
 			UnlockBufHdr(bufHdr, buf_state);
diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c
index 6796756358..5125cfe640 100644
--- a/src/backend/storage/smgr/md.c
+++ b/src/backend/storage/smgr/md.c
@@ -1463,7 +1463,7 @@ DropRelationFiles(RelFileLocator *delrels, int ndelrels, bool isRedo)
 		srels[i] = srel;
 	}
 
-	smgrdounlinkall(srels, ndelrels, isRedo);
+	smgrdounlinkall(srels, NULL, ndelrels, isRedo);
 
 	for (i = 0; i < ndelrels; i++)
 		smgrclose(srels[i]);
diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c
index eb01040772..68139e3943 100644
--- a/src/backend/storage/smgr/smgr.c
+++ b/src/backend/storage/smgr/smgr.c
@@ -451,15 +451,19 @@ smgrdosyncall(SMgrRelation *rels, int nrels)
 /*
  * smgrdounlinkall() -- Immediately unlink all forks of all given relations
  *
- * All forks of all given relations are removed from the store.  This
- * should not be used during transactional operations, since it can't be
- * undone.
+ * Forks of all given relations are removed from the store.  This should not be
+ * used during transactional operations, since it can't be undone.
+ *
+ * If forks is NULL, all forks are removed for all relations. Otherwise, only
+ * the specified fork is removed for the relation at the corresponding position
+ * in the rels array. InvalidForkNumber means removing all forks for the
+ * corresponding relation.
  *
  * If isRedo is true, it is okay for the underlying file(s) to be gone
  * already.
  */
 void
-smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
+smgrdounlinkall(SMgrRelation *rels, ForkBitmap *forks, int nrels, bool isRedo)
 {
 	int			i = 0;
 	RelFileLocatorBackend *rlocators;
@@ -472,7 +476,7 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 	 * Get rid of any remaining buffers for the relations.  bufmgr will just
 	 * drop them without bothering to write the contents.
 	 */
-	DropRelationsAllBuffers(rels, nrels);
+	DropRelationsAllBuffers(rels, forks, nrels);
 
 	/*
 	 * create an array which contains all relations to be dropped, and close
@@ -486,9 +490,13 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 
 		rlocators[i] = rlocator;
 
-		/* Close the forks at smgr level */
+		/* Close the spacified forks at smgr level. */
 		for (forknum = 0; forknum <= MAX_FORKNUM; forknum++)
-			smgrsw[which].smgr_close(rels[i], forknum);
+		{
+			if (!forks || FORKBITMAP_ISSET(forks[i], forknum))
+				smgrsw[which].smgr_close(rels[i], forknum);
+			continue;
+		}
 	}
 
 	/*
@@ -515,7 +523,11 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
 		int			which = rels[i]->smgr_which;
 
 		for (forknum = 0; forknum <= MAX_FORKNUM; forknum++)
-			smgrsw[which].smgr_unlink(rlocators[i], forknum, isRedo);
+		{
+			if (!forks || FORKBITMAP_ISSET(forks[i], forknum))
+				smgrsw[which].smgr_unlink(rlocators[i], forknum, isRedo);
+			continue;
+		}
 	}
 
 	pfree(rlocators);
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index 8040250a0c..5c6d280499 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -3858,7 +3858,7 @@ RelationSetNewRelfilenumber(Relation relation, char persistence)
 		 * anyway.
 		 */
 		srel = smgropen(relation->rd_locator, relation->rd_backend);
-		smgrdounlinkall(&srel, 1, false);
+		smgrdounlinkall(&srel, NULL, 1, false);
 		smgrclose(srel);
 	}
 	else
diff --git a/src/include/common/relpath.h b/src/include/common/relpath.h
index 6f006d5a93..d0088903c5 100644
--- a/src/include/common/relpath.h
+++ b/src/include/common/relpath.h
@@ -61,6 +61,17 @@ typedef enum ForkNumber
 
 #define MAX_FORKNUM		INIT_FORKNUM
 
+/* ForkBitmap holds multiple forks as a bitmap */
+StaticAssertDecl(MAX_FORKNUM < 8, "MAX_FORKNUM too large for ForkBitmap");
+
+typedef uint8 ForkBitmap;
+#define FORKBITMAP_BIT(f)	  (1 << (f))
+#define FORKBITMAP_INIT(m, f) ((m) = FORKBITMAP_BIT((f)))
+#define FORKBITMAP_SET(m, f) ((m) |= FORKBITMAP_BIT((f)))
+#define FORKBITMAP_RESET(m, f) ((m) &= ~(FORKBITMAP_BIT(f)))
+#define FORKBITMAP_ISSET(m, f) ((m) & FORKBITMAP_BIT(f))
+#define FORKBITMAP_ALLFORKS() ((1 << (MAX_FORKNUM + 1)) - 1)
+
 #define FORKNAMECHARS	4		/* max chars for a fork name */
 
 extern PGDLLIMPORT const char *const forkNames[];
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index df881ea86c..b45c330e01 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -275,7 +275,7 @@ extern void DropRelationBuffers(struct SMgrRelationData *smgr_reln,
 								ForkNumber *forkNum,
 								int nforks, BlockNumber *firstDelBlock);
 extern void DropRelationsAllBuffers(struct SMgrRelationData **smgr_reln,
-									int nlocators);
+									ForkBitmap *forks, int nlocators);
 extern void DropDatabaseBuffers(Oid dbid);
 extern void SetRelationBuffersPersistence(struct SMgrRelationData *srel,
 										  bool permanent);
diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h
index e867ff92ab..222d631bf0 100644
--- a/src/include/storage/smgr.h
+++ b/src/include/storage/smgr.h
@@ -85,7 +85,8 @@ extern void smgrreleaseall(void);
 extern void smgrreleaserellocator(RelFileLocatorBackend rlocator);
 extern void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo);
 extern void smgrdosyncall(SMgrRelation *rels, int nrels);
-extern void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo);
+extern void smgrdounlinkall(SMgrRelation *rels, ForkBitmap *forks, int nrels,
+							bool isRedo);
 extern void smgrextend(SMgrRelation reln, ForkNumber forknum,
 					   BlockNumber blocknum, const void *buffer, bool skipFsync);
 extern void smgrzeroextend(SMgrRelation reln, ForkNumber forknum,
-- 
2.43.5

