From 05a75bff3a48007f393bf5f99e354ec0619d00c9 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Wed, 13 Feb 2019 14:34:46 +0900
Subject: [PATCH 5/5] Global LRU based cache pruning.

This adds a feature that removes leaset recently used cache entry
among all catcaches when the total memory amount goes avove
catalog_cache_max_size.
---
 doc/src/sgml/config.sgml           |  20 +++++++
 src/backend/utils/cache/catcache.c | 106 +++++++++++++++++++++++--------------
 src/backend/utils/misc/guc.c       |  21 +++-----
 src/include/utils/catcache.h       |   5 +-
 4 files changed, 94 insertions(+), 58 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bc2bef0878..daa6085693 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1701,6 +1701,26 @@ include_dir 'conf.d'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-catalog-cache-max-size" xreflabel="catalog_cache_max_size">
+      <term><varname>catalog_cache_max_size</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>catalog_cache_max_size</varname> configuration
+       parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies the maximum total amount of memory allowed for all system
+        catalog caches in kilobytes. The value defaults to 0, indicating that
+        pruning by this parameter is disabled at all. After the amount of
+        memory used by all catalog caches exceeds this size, a new cache entry
+        creation will remove one or more not-recently-used cache entries. This
+        means frequent creation of new cache entry may lead to a slight
+        slowdown of queries.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-max-stack-depth" xreflabel="max_stack_depth">
       <term><varname>max_stack_depth</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index fa0d19a9c3..3336ff6dc3 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -86,11 +86,9 @@ int catalog_cache_memory_target = 0;
 
 /*
  * GUC for limit by the number of entries. Entries are removed when the number
- * of them goes above catalog_cache_entry_limit and leaving newer entries by
- * the ratio specified by catalog_cache_prune_ratio.
+ * of them goes above catalog_cache_max_size in kilobytes
  */
-int catalog_cache_entry_limit = 0;
-double catalog_cache_prune_ratio = 0.8;
+int catalog_cache_max_size = 0;
 
 /*
  * Flag to keep track of whether catcache clock timer is active.
@@ -108,6 +106,8 @@ static CatCacheHeader *CacheHdr = NULL;
 
 /* Clock used to record the last accessed time of a catcache record. */
 TimestampTz	catcacheclock = 0;
+dlist_head	cc_lru_list = {0};
+Size		global_size = 0;
 
 /* age classes for pruning */
 static double ageclass[SYSCACHE_STATS_NAGECLASSES]
@@ -531,6 +531,8 @@ CatCacheRemoveCTup(CatCache *cache, CatCTup *ct)
 						 cache->cc_keyno, ct->keys);
 
 	cache->cc_memusage -= ct->size;
+	global_size -= ct->size;
+
 	pfree(ct);
 
 	--cache->cc_ntup;
@@ -887,8 +889,12 @@ InitCatCache(int id,
 	/* cc_head_alloc_size + consumed size for cc_bucket */
 	cp->cc_memusage =
 		MemoryContextGetConsumption(CacheMemoryContext) - base_size;
+	global_size += cp->cc_memusage;
+
+	/* initialize global LRU if not yet */
+	if (cc_lru_list.head.next == NULL)
+		dlist_init(&cc_lru_list);
 
-	dlist_init(&cp->cc_lru_list);
 	/*
 	 * new cache is initialized as far as we can go for now. print some
 	 * debugging information, if appropriate.
@@ -981,39 +987,27 @@ assign_catalog_cache_prune_min_age(int newval, void *extra)
 static bool
 CatCacheCleanupOldEntries(CatCache *cp)
 {
+	static TimestampTz prev_warn_emit = 0;
 	int			nremoved = 0;
 	int			nelems_before = cp->cc_ntup;
-	int			ndelelems = 0;
 	bool		prune_by_age = false;
-	bool		prune_by_number = false;
+	bool		prune_by_size = false;
 	dlist_mutable_iter	iter;
 
-	/* prune only if the size of the hash is above the target */
 	if (catalog_cache_prune_min_age >= 0 &&
 		cp->cc_memusage > (Size) catalog_cache_memory_target * 1024L)
 		prune_by_age = true;
 
-	if (catalog_cache_entry_limit > 0 &&
-		nelems_before >= catalog_cache_entry_limit)
-	{
-		ndelelems = nelems_before -
-			(int) (catalog_cache_entry_limit * catalog_cache_prune_ratio);
-
-		/* an arbitrary lower limit.. */
-		if (ndelelems < 256)
-			ndelelems = 256;
-		if (ndelelems > nelems_before)
-			ndelelems = nelems_before;
-
-		prune_by_number = true;
-	}
+	if (catalog_cache_max_size > 0 &&
+		global_size >= (Size) catalog_cache_max_size * 1024)
+		prune_by_size = true;
 
 	/* Return immediately if no pruning is wanted */
-	if (!prune_by_age && !prune_by_number)
+	if (!prune_by_age && !prune_by_size)
 		return false;
 
 	/* Scan over LRU to find entries to remove */
-	dlist_foreach_modify(iter, &cp->cc_lru_list)
+	dlist_foreach_modify(iter, &cc_lru_list)
 	{
 		CatCTup    *ct = dlist_container(CatCTup, lru_node, iter.cur);
 		bool		remove_this = false;
@@ -1023,8 +1017,8 @@ CatCacheCleanupOldEntries(CatCache *cp)
 			(ct->c_list && ct->c_list->refcount != 0))
 			continue;
 
-		/* check against age */
-		if (prune_by_age)
+		/* check against age. prune within this cache */
+		if (prune_by_age && ct->owner == cp)
 		{
 			long	entry_age;
 			int		us;
@@ -1056,31 +1050,58 @@ CatCacheCleanupOldEntries(CatCache *cp)
 				remove_this = true;
 		}
 
-		/* check against entry number */
-		if (prune_by_number)
+		/* check against global size. removes from all cache */
+		if (prune_by_size && !remove_this)
 		{
-			if (nremoved < ndelelems)
+			if (global_size >= (Size) catalog_cache_max_size * 1024)
 				remove_this = true;
 			else
-				prune_by_number = false; /* we're satisfied */
+				prune_by_size = false; /* we're satisfied */
 		}
 
+		if (!remove_this)
+			continue;
+
 		/* exit immediately if all finished */
-		if (!prune_by_age && !prune_by_number)
+		if (!prune_by_age && !prune_by_size)
 			break;
 
 		/* do the work */
-		if (remove_this)
-		{
-			CatCacheRemoveCTup(cp, ct);
-			nremoved++;
-		}
+		CatCacheRemoveCTup(ct->owner, ct);
+		nremoved++;
 	}
 
 	if (nremoved > 0)
 		elog(DEBUG1, "pruning catalog cache id=%d for %s: removed %d / %d",
 			 cp->id, cp->cc_relname, nremoved, nelems_before);
 
+	/*
+	 * Warn of too small setting of catalog_cache_max_size. Take 5 seconds
+	 * between messages, using statement start timestamp to avoid frequent
+	 * gettimeofday().
+	 */
+	if (prune_by_size &&
+		(prev_warn_emit == 0 ||
+		 GetCurrentStatementStartTimestamp() - prev_warn_emit > 5000000))
+	{
+		ErrorContextCallback *oldcb;
+
+		/* cancel error context callbacks  */
+		oldcb = error_context_stack;
+		error_context_stack = NULL;
+		
+		ereport(LOG, (
+					errmsg ("cannot reduce cache size to %d kilobytes, reduced to %d kilobytes",
+							catalog_cache_max_size,	(int)(global_size / 1024)),
+					errdetail ("Consider increasing the configuration parameter \"catalog_cache_max_size\"."),
+					errhidecontext(true),
+					errhidestmt(true)));
+
+		error_context_stack = oldcb;
+
+		prev_warn_emit = GetCurrentStatementStartTimestamp();
+	}
+
 	return nremoved > 0;
 }
 
@@ -1103,6 +1124,7 @@ RehashCatCache(CatCache *cp)
 	newbucket = (dlist_head *) MemoryContextAllocZero(CacheMemoryContext, newnbuckets * sizeof(dlist_head));
 
 	/* recalculate memory usage from the first */
+	global_size -= cp->cc_memusage;
 	cp->cc_memusage = cp->cc_head_alloc_size +
 		MemoryContextGetConsumption(CacheMemoryContext) - base_size;
 
@@ -1122,6 +1144,8 @@ RehashCatCache(CatCache *cp)
 		}
 	}
 
+	global_size += cp->cc_memusage;
+
 	/* Switch to the new array. */
 	pfree(cp->cc_bucket);
 	cp->cc_nbuckets = newnbuckets;
@@ -1513,7 +1537,7 @@ SearchCatCacheInternal(CatCache *cache,
 		if (catcacheclock - ct->lastaccess > MIN_LRU_UPDATE_INTERVAL)
 		{
 			ct->lastaccess = catcacheclock;
-			dlist_move_tail(&cache->cc_lru_list, &ct->lru_node);
+			dlist_move_tail(&cc_lru_list, &ct->lru_node);
 		}
 
 		/*
@@ -2138,7 +2162,8 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
 	ct->hash_value = hashValue;
 	ct->naccess = 0;
 	ct->lastaccess = catcacheclock;
-	dlist_push_tail(&cache->cc_lru_list, &ct->lru_node);
+	ct->owner = cache;
+	dlist_push_tail(&cc_lru_list, &ct->lru_node);
 
 	dlist_push_head(&cache->cc_bucket[hashIndex], &ct->cache_elem);
 
@@ -2147,6 +2172,7 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
 
 	ct->size = MemoryContextGetConsumption(CacheMemoryContext) - base_size;
 	cache->cc_memusage += ct->size;
+	global_size += ct->size;
 
 	/* increase refcount so that this survives pruning */
 	ct->refcount++;
@@ -2161,8 +2187,8 @@ CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, Datum *arguments,
 		!CatCacheCleanupOldEntries(cache))
 		RehashCatCache(cache);
 	/* we may still want to prune by entry number, check it */
-	else if (catalog_cache_entry_limit > 0 &&
-			 cache->cc_ntup > catalog_cache_entry_limit)
+	else if (catalog_cache_max_size > 0 &&
+			 global_size > catalog_cache_max_size * 1024)
 		CatCacheCleanupOldEntries(cache);
 
 	ct->refcount--;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 7f1670fa5b..7a52c70649 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2229,12 +2229,13 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
-		{"catalog_cache_entry_limit", PGC_USERSET, RESOURCES_MEM,
-			gettext_noop("Sets the maximum entries of catcache."),
-		 	NULL
+		{"catalog_cache_max_size", PGC_USERSET, RESOURCES_MEM,
+			gettext_noop("Sets the maximum size of catcache in kilobytes."),
+		 	NULL,
+		 	GUC_UNIT_KB
 		},
-		&catalog_cache_entry_limit,
-		0, 0, INT_MAX,
+		&catalog_cache_max_size,
+		0, 0, MAX_KILOBYTES,
 		NULL, NULL, NULL
 	},
 
@@ -3411,16 +3412,6 @@ static struct config_real ConfigureNamesReal[] =
 		NULL, NULL, NULL
 	},
 
-	{
-		{"catalog_cache_prune_ratio", PGC_USERSET, RESOURCES_MEM,
-			gettext_noop("Reduce ratio of pruning caused by catalog_cache_entry_limit."),
-		 	NULL
-		},
-		&catalog_cache_prune_ratio,
-		0.8, 0.0, 1.0,
-		NULL, NULL, NULL
-	},
-
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, 0.0, 0.0, 0.0, NULL, NULL, NULL
diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h
index 95cd885c16..1e2d6e7bd7 100644
--- a/src/include/utils/catcache.h
+++ b/src/include/utils/catcache.h
@@ -62,7 +62,6 @@ typedef struct catcache
 	slist_node	cc_next;		/* list link */
 	ScanKeyData cc_skey[CATCACHE_MAXKEYS];	/* precomputed key info for heap
 											 * scans */
-	dlist_head	cc_lru_list;
 	int			cc_head_alloc_size;/* consumed memory to allocate this struct */
 	int			cc_memusage;	/* memory usage of this catcache (excluding
 								 * header part) */
@@ -125,6 +124,7 @@ typedef struct catctup
 	int			naccess;		/* # of access to this entry, up to 2  */
 	TimestampTz	lastaccess;		/* approx. timestamp of the last usage */
 	dlist_node	lru_node;		/* LRU node */
+	CatCache   *owner;			/* owner catcache */
 	int			size;			/* palloc'ed size off this tuple */
 	/*
 	 * The tuple may also be a member of at most one CatCList.  (If a single
@@ -198,8 +198,7 @@ extern PGDLLIMPORT MemoryContext CacheMemoryContext;
 /* for guc.c, not PGDLLPMPORT'ed */
 extern int catalog_cache_prune_min_age;
 extern int catalog_cache_memory_target;
-extern int catalog_cache_entry_limit;
-extern double catalog_cache_prune_ratio;
+extern int catalog_cache_max_size;
 
 /* to use as access timestamp of catcache entries */
 extern TimestampTz catcacheclock;
-- 
2.16.3

