From b1de5cbacf06dd975cc2138a498c5d9897e14df7 Mon Sep 17 00:00:00 2001
From: John Naylor <john.naylor@postgresql.org>
Date: Fri, 23 Dec 2022 14:33:49 +0700
Subject: [PATCH v17 5/9] Template out inner and leaf nodes

Use a template for each insert, iteration, search, and
delete functions.

To optimize growing into node125, don't search for a
slot each time -- just copy into the first 32
slots and set the slot index at the same time.
Also set all the isset bits with a single store.

Remove node_*_125_update/insert/delete functions and
node_125_find_unused_slot, since they are now unused.
---
 src/backend/lib/radixtree.c             | 863 ++----------------------
 src/include/lib/radixtree_delete_impl.h | 100 +++
 src/include/lib/radixtree_insert_impl.h | 293 ++++++++
 src/include/lib/radixtree_iter_impl.h   | 129 ++++
 src/include/lib/radixtree_search_impl.h | 102 +++
 src/tools/pginclude/cpluspluscheck      |   6 +
 src/tools/pginclude/headerscheck        |   6 +
 7 files changed, 694 insertions(+), 805 deletions(-)
 create mode 100644 src/include/lib/radixtree_delete_impl.h
 create mode 100644 src/include/lib/radixtree_insert_impl.h
 create mode 100644 src/include/lib/radixtree_iter_impl.h
 create mode 100644 src/include/lib/radixtree_search_impl.h

diff --git a/src/backend/lib/radixtree.c b/src/backend/lib/radixtree.c
index 5203127f76..80cde09aaf 100644
--- a/src/backend/lib/radixtree.c
+++ b/src/backend/lib/radixtree.c
@@ -96,13 +96,6 @@
 #define BM_IDX(x)	((x) / BITS_PER_BITMAPWORD)
 #define BM_BIT(x)	((x) % BITS_PER_BITMAPWORD)
 
-/* Enum used rt_node_search() */
-typedef enum
-{
-	RT_ACTION_FIND = 0,			/* find the key-value */
-	RT_ACTION_DELETE,			/* delete the key-value */
-} rt_action;
-
 /*
  * Supported radix tree node kinds and size classes.
  *
@@ -422,10 +415,8 @@ static inline void rt_init_node(rt_node *node, uint8 kind, rt_size_class size_cl
 								bool inner);
 static void rt_free_node(radix_tree *tree, rt_node *node);
 static void rt_extend(radix_tree *tree, uint64 key);
-static inline bool rt_node_search_inner(rt_node *node, uint64 key, rt_action action,
-										rt_node **child_p);
-static inline bool rt_node_search_leaf(rt_node *node, uint64 key, rt_action action,
-									   uint64 *value_p);
+static inline bool rt_node_search_inner(rt_node *node, uint64 key, rt_node **child_p);
+static inline bool rt_node_search_leaf(rt_node *node, uint64 key, uint64 *value_p);
 static bool rt_node_insert_inner(radix_tree *tree, rt_node *parent, rt_node *node,
 								 uint64 key, rt_node *child);
 static bool rt_node_insert_leaf(radix_tree *tree, rt_node *parent, rt_node *node,
@@ -663,102 +654,6 @@ node_leaf_125_get_value(rt_node_leaf_125 *node, uint8 chunk)
 	return node->values[node->base.slot_idxs[chunk]];
 }
 
-static void
-node_inner_125_delete(rt_node_inner_125 *node, uint8 chunk)
-{
-	int			slotpos = node->base.slot_idxs[chunk];
-	int			idx = BM_IDX(slotpos);
-	int			bitnum = BM_BIT(slotpos);
-
-	Assert(!NODE_IS_LEAF(node));
-
-	node->base.isset[idx] &= ~((bitmapword) 1 << bitnum);
-	node->children[node->base.slot_idxs[chunk]] = NULL;
-	node->base.slot_idxs[chunk] = RT_NODE_125_INVALID_IDX;
-}
-
-static void
-node_leaf_125_delete(rt_node_leaf_125 *node, uint8 chunk)
-{
-	int			slotpos = node->base.slot_idxs[chunk];
-	int			idx = BM_IDX(slotpos);
-	int			bitnum = BM_BIT(slotpos);
-
-	Assert(NODE_IS_LEAF(node));
-	node->base.isset[idx] &= ~((bitmapword) 1 << bitnum);
-	node->base.slot_idxs[chunk] = RT_NODE_125_INVALID_IDX;
-}
-
-/* Return an unused slot in node-125 */
-static int
-node_125_find_unused_slot(bitmapword *isset)
-{
-	int			slotpos;
-	int			idx;
-	bitmapword	inverse;
-
-	/* get the first word with at least one bit not set */
-	for (idx = 0; idx < BM_IDX(128); idx++)
-	{
-		if (isset[idx] < ~((bitmapword) 0))
-			break;
-	}
-
-	/* To get the first unset bit in X, get the first set bit in ~X */
-	inverse = ~(isset[idx]);
-	slotpos = idx * BITS_PER_BITMAPWORD;
-	slotpos += bmw_rightmost_one_pos(inverse);
-
-	/* mark the slot used */
-	isset[idx] |= bmw_rightmost_one(inverse);
-
-	return slotpos;
- }
-
-static inline void
-node_inner_125_insert(rt_node_inner_125 *node, uint8 chunk, rt_node *child)
-{
-	int			slotpos;
-
-	Assert(!NODE_IS_LEAF(node));
-
-	slotpos = node_125_find_unused_slot(node->base.isset);
-	Assert(slotpos < node->base.n.fanout);
-
-	node->base.slot_idxs[chunk] = slotpos;
-	node->children[slotpos] = child;
-}
-
-/* Set the slot at the corresponding chunk */
-static inline void
-node_leaf_125_insert(rt_node_leaf_125 *node, uint8 chunk, uint64 value)
-{
-	int			slotpos;
-
-	Assert(NODE_IS_LEAF(node));
-
-	slotpos = node_125_find_unused_slot(node->base.isset);
-	Assert(slotpos < node->base.n.fanout);
-
-	node->base.slot_idxs[chunk] = slotpos;
-	node->values[slotpos] = value;
-}
-
-/* Update the child corresponding to 'chunk' to 'child' */
-static inline void
-node_inner_125_update(rt_node_inner_125 *node, uint8 chunk, rt_node *child)
-{
-	Assert(!NODE_IS_LEAF(node));
-	node->children[node->base.slot_idxs[chunk]] = child;
-}
-
-static inline void
-node_leaf_125_update(rt_node_leaf_125 *node, uint8 chunk, uint64 value)
-{
-	Assert(NODE_IS_LEAF(node));
-	node->values[node->base.slot_idxs[chunk]] = value;
-}
-
 /* Functions to manipulate inner and leaf node-256 */
 
 /* Return true if the slot corresponding to the given chunk is in use */
@@ -1075,189 +970,57 @@ rt_set_extend(radix_tree *tree, uint64 key, uint64 value, rt_node *parent,
 }
 
 /*
- * Search for the child pointer corresponding to 'key' in the given node, and
- * do the specified 'action'.
+ * Search for the child pointer corresponding to 'key' in the given node.
  *
  * Return true if the key is found, otherwise return false. On success, the child
  * pointer is set to child_p.
  */
 static inline bool
-rt_node_search_inner(rt_node *node, uint64 key, rt_action action, rt_node **child_p)
+rt_node_search_inner(rt_node *node, uint64 key, rt_node **child_p)
 {
-	uint8		chunk = RT_GET_KEY_CHUNK(key, node->shift);
-	bool		found = false;
-	rt_node    *child = NULL;
-
-	switch (node->kind)
-	{
-		case RT_NODE_KIND_4:
-			{
-				rt_node_inner_4 *n4 = (rt_node_inner_4 *) node;
-				int			idx = node_4_search_eq((rt_node_base_4 *) n4, chunk);
-
-				if (idx < 0)
-					break;
-
-				found = true;
-
-				if (action == RT_ACTION_FIND)
-					child = n4->children[idx];
-				else			/* RT_ACTION_DELETE */
-					chunk_children_array_delete(n4->base.chunks, n4->children,
-												n4->base.n.count, idx);
-
-				break;
-			}
-		case RT_NODE_KIND_32:
-			{
-				rt_node_inner_32 *n32 = (rt_node_inner_32 *) node;
-				int			idx = node_32_search_eq((rt_node_base_32 *) n32, chunk);
-
-				if (idx < 0)
-					break;
-
-				found = true;
-				if (action == RT_ACTION_FIND)
-					child = n32->children[idx];
-				else			/* RT_ACTION_DELETE */
-					chunk_children_array_delete(n32->base.chunks, n32->children,
-												n32->base.n.count, idx);
-				break;
-			}
-		case RT_NODE_KIND_125:
-			{
-				rt_node_inner_125 *n125 = (rt_node_inner_125 *) node;
-
-				if (!node_125_is_chunk_used((rt_node_base_125 *) n125, chunk))
-					break;
-
-				found = true;
-
-				if (action == RT_ACTION_FIND)
-					child = node_inner_125_get_child(n125, chunk);
-				else			/* RT_ACTION_DELETE */
-					node_inner_125_delete(n125, chunk);
-
-				break;
-			}
-		case RT_NODE_KIND_256:
-			{
-				rt_node_inner_256 *n256 = (rt_node_inner_256 *) node;
-
-				if (!node_inner_256_is_chunk_used(n256, chunk))
-					break;
-
-				found = true;
-				if (action == RT_ACTION_FIND)
-					child = node_inner_256_get_child(n256, chunk);
-				else			/* RT_ACTION_DELETE */
-					node_inner_256_delete(n256, chunk);
-
-				break;
-			}
-	}
-
-	/* update statistics */
-	if (action == RT_ACTION_DELETE && found)
-		node->count--;
-
-	if (found && child_p)
-		*child_p = child;
-
-	return found;
+#define RT_NODE_LEVEL_INNER
+#include "lib/radixtree_search_impl.h"
+#undef RT_NODE_LEVEL_INNER
 }
 
 /*
- * Search for the value corresponding to 'key' in the given node, and do the
- * specified 'action'.
+ * Search for the value corresponding to 'key' in the given node.
  *
  * Return true if the key is found, otherwise return false. On success, the pointer
  * to the value is set to value_p.
  */
 static inline bool
-rt_node_search_leaf(rt_node *node, uint64 key, rt_action action, uint64 *value_p)
+rt_node_search_leaf(rt_node *node, uint64 key, uint64 *value_p)
 {
-	uint8		chunk = RT_GET_KEY_CHUNK(key, node->shift);
-	bool		found = false;
-	uint64		value = 0;
-
-	switch (node->kind)
-	{
-		case RT_NODE_KIND_4:
-			{
-				rt_node_leaf_4 *n4 = (rt_node_leaf_4 *) node;
-				int			idx = node_4_search_eq((rt_node_base_4 *) n4, chunk);
-
-				if (idx < 0)
-					break;
-
-				found = true;
-
-				if (action == RT_ACTION_FIND)
-					value = n4->values[idx];
-				else			/* RT_ACTION_DELETE */
-					chunk_values_array_delete(n4->base.chunks, (uint64 *) n4->values,
-											  n4->base.n.count, idx);
-
-				break;
-			}
-		case RT_NODE_KIND_32:
-			{
-				rt_node_leaf_32 *n32 = (rt_node_leaf_32 *) node;
-				int			idx = node_32_search_eq((rt_node_base_32 *) n32, chunk);
-
-				if (idx < 0)
-					break;
-
-				found = true;
-				if (action == RT_ACTION_FIND)
-					value = n32->values[idx];
-				else			/* RT_ACTION_DELETE */
-					chunk_values_array_delete(n32->base.chunks, (uint64 *) n32->values,
-											  n32->base.n.count, idx);
-				break;
-			}
-		case RT_NODE_KIND_125:
-			{
-				rt_node_leaf_125 *n125 = (rt_node_leaf_125 *) node;
-
-				if (!node_125_is_chunk_used((rt_node_base_125 *) n125, chunk))
-					break;
-
-				found = true;
-
-				if (action == RT_ACTION_FIND)
-					value = node_leaf_125_get_value(n125, chunk);
-				else			/* RT_ACTION_DELETE */
-					node_leaf_125_delete(n125, chunk);
-
-				break;
-			}
-		case RT_NODE_KIND_256:
-			{
-				rt_node_leaf_256 *n256 = (rt_node_leaf_256 *) node;
-
-				if (!node_leaf_256_is_chunk_used(n256, chunk))
-					break;
-
-				found = true;
-				if (action == RT_ACTION_FIND)
-					value = node_leaf_256_get_value(n256, chunk);
-				else			/* RT_ACTION_DELETE */
-					node_leaf_256_delete(n256, chunk);
-
-				break;
-			}
-	}
-
-	/* update statistics */
-	if (action == RT_ACTION_DELETE && found)
-		node->count--;
+#define RT_NODE_LEVEL_LEAF
+#include "lib/radixtree_search_impl.h"
+#undef RT_NODE_LEVEL_LEAF
+}
 
-	if (found && value_p)
-		*value_p = value;
+/*
+ * Search for the child pointer corresponding to 'key' in the given node.
+ *
+ * Delete the node and return true if the key is found, otherwise return false.
+ */
+static inline bool
+rt_node_delete_inner(rt_node *node, uint64 key)
+{
+#define RT_NODE_LEVEL_INNER
+#include "lib/radixtree_delete_impl.h"
+#undef RT_NODE_LEVEL_INNER
+}
 
-	return found;
+/*
+ * Search for the value corresponding to 'key' in the given node.
+ *
+ * Delete the node and return true if the key is found, otherwise return false.
+ */
+static inline bool
+rt_node_delete_leaf(rt_node *node, uint64 key)
+{
+#define RT_NODE_LEVEL_LEAF
+#include "lib/radixtree_delete_impl.h"
+#undef RT_NODE_LEVEL_LEAF
 }
 
 /* Insert the child to the inner node */
@@ -1265,185 +1028,9 @@ static bool
 rt_node_insert_inner(radix_tree *tree, rt_node *parent, rt_node *node, uint64 key,
 					 rt_node *child)
 {
-	uint8		chunk = RT_GET_KEY_CHUNK(key, node->shift);
-	bool		chunk_exists = false;
-	rt_node		*newnode = NULL;
-
-	Assert(!NODE_IS_LEAF(node));
-
-	switch (node->kind)
-	{
-		case RT_NODE_KIND_4:
-			{
-				rt_node_inner_4 *n4 = (rt_node_inner_4 *) node;
-				int			idx;
-
-				idx = node_4_search_eq(&n4->base, chunk);
-				if (idx != -1)
-				{
-					/* found the existing chunk */
-					chunk_exists = true;
-					n4->children[idx] = child;
-					break;
-				}
-
-				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n4)))
-				{
-					rt_node_inner_32 *new32;
-
-					/* grow node from 4 to 32 */
-					newnode = rt_grow_node_kind(tree, node, RT_NODE_KIND_32);
-					new32 = (rt_node_inner_32 *) newnode;
-					chunk_children_array_copy(n4->base.chunks, n4->children,
-											  new32->base.chunks, new32->children);
-
-					Assert(parent != NULL);
-					rt_replace_node(tree, parent, node, newnode, key);
-					node = newnode;
-				}
-				else
-				{
-					int			insertpos = node_4_get_insertpos(&n4->base, chunk);
-					uint16		count = n4->base.n.count;
-
-					/* shift chunks and children */
-					if (count != 0 && insertpos < count)
-						chunk_children_array_shift(n4->base.chunks, n4->children,
-												   count, insertpos);
-
-					n4->base.chunks[insertpos] = chunk;
-					n4->children[insertpos] = child;
-					break;
-				}
-			}
-			/* FALLTHROUGH */
-		case RT_NODE_KIND_32:
-			{
-				const rt_size_class_elem minclass = rt_size_class_info[RT_CLASS_32_PARTIAL];
-				const rt_size_class_elem maxclass = rt_size_class_info[RT_CLASS_32_FULL];
-				rt_node_inner_32 *n32 = (rt_node_inner_32 *) node;
-				int			idx;
-
-				idx = node_32_search_eq(&n32->base, chunk);
-				if (idx != -1)
-				{
-					/* found the existing chunk */
-					chunk_exists = true;
-					n32->children[idx] = child;
-					break;
-				}
-
-				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n32)) &&
-					n32->base.n.count == minclass.fanout)
-				{
-					/* grow to the next size class of this kind */
-					newnode = rt_alloc_node(tree, RT_CLASS_32_FULL, true);
-					memcpy(newnode, node, minclass.inner_size);
-					newnode->fanout = maxclass.fanout;
-
-					Assert(parent != NULL);
-					rt_replace_node(tree, parent, node, newnode, key);
-					node = newnode;
-
-					/* also update pointer for this kind */
-					n32 = (rt_node_inner_32 *) newnode;
-				}
-
-				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n32)))
-				{
-					rt_node_inner_125 *new125;
-
-					/* grow node from 32 to 125 */
-					newnode = rt_grow_node_kind(tree, node, RT_NODE_KIND_125);
-					new125 = (rt_node_inner_125 *) newnode;
-					for (int i = 0; i < n32->base.n.count; i++)
-						node_inner_125_insert(new125, n32->base.chunks[i], n32->children[i]);
-
-					Assert(parent != NULL);
-					rt_replace_node(tree, parent, node, newnode, key);
-					node = newnode;
-				}
-				else
-				{
-					int	insertpos = node_32_get_insertpos(&n32->base, chunk);
-					int16 count = n32->base.n.count;
-
-					if (insertpos < count)
-					{
-						Assert(count > 0);
-						chunk_children_array_shift(n32->base.chunks, n32->children,
-												   count, insertpos);
-					}
-
-					n32->base.chunks[insertpos] = chunk;
-					n32->children[insertpos] = child;
-					break;
-				}
-			}
-			/* FALLTHROUGH */
-		case RT_NODE_KIND_125:
-			{
-				rt_node_inner_125 *n125 = (rt_node_inner_125 *) node;
-				int			cnt = 0;
-
-				if (node_125_is_chunk_used(&n125->base, chunk))
-				{
-					/* found the existing chunk */
-					chunk_exists = true;
-					node_inner_125_update(n125, chunk, child);
-					break;
-				}
-
-				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n125)))
-				{
-					rt_node_inner_256 *new256;
-					Assert(parent != NULL);
-
-					/* grow node from 125 to 256 */
-					newnode = rt_grow_node_kind(tree, node, RT_NODE_KIND_256);
-					new256 = (rt_node_inner_256 *) newnode;
-					for (int i = 0; i < RT_NODE_MAX_SLOTS && cnt < n125->base.n.count; i++)
-					{
-						if (!node_125_is_chunk_used(&n125->base, i))
-							continue;
-
-						node_inner_256_set(new256, i, node_inner_125_get_child(n125, i));
-						cnt++;
-					}
-
-					rt_replace_node(tree, parent, node, newnode, key);
-					node = newnode;
-				}
-				else
-				{
-					node_inner_125_insert(n125, chunk, child);
-					break;
-				}
-			}
-			/* FALLTHROUGH */
-		case RT_NODE_KIND_256:
-			{
-				rt_node_inner_256 *n256 = (rt_node_inner_256 *) node;
-
-				chunk_exists = node_inner_256_is_chunk_used(n256, chunk);
-				Assert(chunk_exists || FIXED_NODE_HAS_FREE_SLOT(n256, RT_CLASS_256));
-
-				node_inner_256_set(n256, chunk, child);
-				break;
-			}
-	}
-
-	/* Update statistics */
-	if (!chunk_exists)
-		node->count++;
-
-	/*
-	 * Done. Finally, verify the chunk and value is inserted or replaced
-	 * properly in the node.
-	 */
-	rt_verify_node(node);
-
-	return chunk_exists;
+#define RT_NODE_LEVEL_INNER
+#include "lib/radixtree_insert_impl.h"
+#undef RT_NODE_LEVEL_INNER
 }
 
 /* Insert the value to the leaf node */
@@ -1451,190 +1038,9 @@ static bool
 rt_node_insert_leaf(radix_tree *tree, rt_node *parent, rt_node *node,
 					uint64 key, uint64 value)
 {
-	uint8		chunk = RT_GET_KEY_CHUNK(key, node->shift);
-	bool		chunk_exists = false;
-
-	Assert(NODE_IS_LEAF(node));
-
-	switch (node->kind)
-	{
-		case RT_NODE_KIND_4:
-			{
-				rt_node_leaf_4 *n4 = (rt_node_leaf_4 *) node;
-				int			idx;
-
-				idx = node_4_search_eq((rt_node_base_4 *) n4, chunk);
-				if (idx != -1)
-				{
-					/* found the existing chunk */
-					chunk_exists = true;
-					n4->values[idx] = value;
-					break;
-				}
-
-				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n4)))
-				{
-					rt_node_leaf_32 *new32;
-					Assert(parent != NULL);
-
-					/* grow node from 4 to 32 */
-					new32 = (rt_node_leaf_32 *) rt_grow_node_kind(tree, (rt_node *) n4,
-																  RT_NODE_KIND_32);
-					chunk_values_array_copy(n4->base.chunks, n4->values,
-											new32->base.chunks, new32->values);
-					rt_replace_node(tree, parent, (rt_node *) n4, (rt_node *) new32, key);
-					node = (rt_node *) new32;
-				}
-				else
-				{
-					int			insertpos = node_4_get_insertpos((rt_node_base_4 *) n4, chunk);
-					int			count = n4->base.n.count;
-
-					/* shift chunks and values */
-					if (count != 0 && insertpos < count)
-						chunk_values_array_shift(n4->base.chunks, n4->values,
-												 count, insertpos);
-
-					n4->base.chunks[insertpos] = chunk;
-					n4->values[insertpos] = value;
-					break;
-				}
-			}
-			/* FALLTHROUGH */
-		case RT_NODE_KIND_32:
-			{
-				rt_node_leaf_32 *n32 = (rt_node_leaf_32 *) node;
-				int			idx;
-
-				idx = node_32_search_eq((rt_node_base_32 *) n32, chunk);
-				if (idx != -1)
-				{
-					/* found the existing chunk */
-					chunk_exists = true;
-					n32->values[idx] = value;
-					break;
-				}
-
-				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n32)))
-				{
-					Assert(parent != NULL);
-
-					if (n32->base.n.count == rt_size_class_info[RT_CLASS_32_PARTIAL].fanout)
-					{
-						/* use the same node kind, but expand to the next size class */
-						const Size size = rt_size_class_info[RT_CLASS_32_PARTIAL].leaf_size;
-						const int fanout = rt_size_class_info[RT_CLASS_32_FULL].fanout;
-						rt_node_leaf_32 *new32;
-
-						new32 = (rt_node_leaf_32 *) rt_alloc_node(tree, RT_CLASS_32_FULL, false);
-						memcpy(new32, n32, size);
-						new32->base.n.fanout = fanout;
-
-						rt_replace_node(tree, parent, (rt_node *) n32, (rt_node *) new32, key);
-
-						/* must update both pointers here */
-						node = (rt_node *) new32;
-						n32 = new32;
-
-						goto retry_insert_leaf_32;
-					}
-					else
-					{
-						rt_node_leaf_125 *new125;
-
-						/* grow node from 32 to 125 */
-						new125 = (rt_node_leaf_125 *) rt_grow_node_kind(tree, (rt_node *) n32,
-																		RT_NODE_KIND_125);
-						for (int i = 0; i < n32->base.n.count; i++)
-							node_leaf_125_insert(new125, n32->base.chunks[i], n32->values[i]);
-
-						rt_replace_node(tree, parent, (rt_node *) n32, (rt_node *) new125,
-										key);
-						node = (rt_node *) new125;
-					}
-				}
-				else
-				{
-				retry_insert_leaf_32:
-					{
-						int	insertpos = node_32_get_insertpos((rt_node_base_32 *) n32, chunk);
-						int	count = n32->base.n.count;
-
-						if (count != 0 && insertpos < count)
-							chunk_values_array_shift(n32->base.chunks, n32->values,
-													 count, insertpos);
-
-						n32->base.chunks[insertpos] = chunk;
-						n32->values[insertpos] = value;
-						break;
-					}
-				}
-			}
-			/* FALLTHROUGH */
-		case RT_NODE_KIND_125:
-			{
-				rt_node_leaf_125 *n125 = (rt_node_leaf_125 *) node;
-				int			cnt = 0;
-
-				if (node_125_is_chunk_used((rt_node_base_125 *) n125, chunk))
-				{
-					/* found the existing chunk */
-					chunk_exists = true;
-					node_leaf_125_update(n125, chunk, value);
-					break;
-				}
-
-				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n125)))
-				{
-					rt_node_leaf_256 *new256;
-					Assert(parent != NULL);
-
-					/* grow node from 125 to 256 */
-					new256 = (rt_node_leaf_256 *) rt_grow_node_kind(tree, (rt_node *) n125,
-																	RT_NODE_KIND_256);
-					for (int i = 0; i < RT_NODE_MAX_SLOTS && cnt < n125->base.n.count; i++)
-					{
-						if (!node_125_is_chunk_used((rt_node_base_125 *) n125, i))
-							continue;
-
-						node_leaf_256_set(new256, i, node_leaf_125_get_value(n125, i));
-						cnt++;
-					}
-
-					rt_replace_node(tree, parent, (rt_node *) n125, (rt_node *) new256,
-									key);
-					node = (rt_node *) new256;
-				}
-				else
-				{
-					node_leaf_125_insert(n125, chunk, value);
-					break;
-				}
-			}
-			/* FALLTHROUGH */
-		case RT_NODE_KIND_256:
-			{
-				rt_node_leaf_256 *n256 = (rt_node_leaf_256 *) node;
-
-				chunk_exists = node_leaf_256_is_chunk_used(n256, chunk);
-				Assert(chunk_exists || FIXED_NODE_HAS_FREE_SLOT(n256, RT_CLASS_256));
-
-				node_leaf_256_set(n256, chunk, value);
-				break;
-			}
-	}
-
-	/* Update statistics */
-	if (!chunk_exists)
-		node->count++;
-
-	/*
-	 * Done. Finally, verify the chunk and value is inserted or replaced
-	 * properly in the node.
-	 */
-	rt_verify_node(node);
-
-	return chunk_exists;
+#define RT_NODE_LEVEL_LEAF
+#include "lib/radixtree_insert_impl.h"
+#undef RT_NODE_LEVEL_LEAF
 }
 
 /*
@@ -1723,7 +1129,7 @@ rt_set(radix_tree *tree, uint64 key, uint64 value)
 		if (NODE_IS_LEAF(node))
 			break;
 
-		if (!rt_node_search_inner(node, key, RT_ACTION_FIND, &child))
+		if (!rt_node_search_inner(node, key, &child))
 		{
 			rt_set_extend(tree, key, value, parent, node);
 			return false;
@@ -1770,14 +1176,14 @@ rt_search(radix_tree *tree, uint64 key, uint64 *value_p)
 		if (NODE_IS_LEAF(node))
 			break;
 
-		if (!rt_node_search_inner(node, key, RT_ACTION_FIND, &child))
+		if (!rt_node_search_inner(node, key, &child))
 			return false;
 
 		node = child;
 		shift -= RT_NODE_SPAN;
 	}
 
-	return rt_node_search_leaf(node, key, RT_ACTION_FIND, value_p);
+	return rt_node_search_leaf(node, key, value_p);
 }
 
 /*
@@ -1810,7 +1216,7 @@ rt_delete(radix_tree *tree, uint64 key)
 		/* Push the current node to the stack */
 		stack[++level] = node;
 
-		if (!rt_node_search_inner(node, key, RT_ACTION_FIND, &child))
+		if (!rt_node_search_inner(node, key, &child))
 			return false;
 
 		node = child;
@@ -1819,7 +1225,7 @@ rt_delete(radix_tree *tree, uint64 key)
 
 	/* Delete the key from the leaf node if exists */
 	Assert(NODE_IS_LEAF(node));
-	deleted = rt_node_search_leaf(node, key, RT_ACTION_DELETE, NULL);
+	deleted = rt_node_delete_leaf(node, key);
 
 	if (!deleted)
 	{
@@ -1845,7 +1251,7 @@ rt_delete(radix_tree *tree, uint64 key)
 	{
 		node = stack[level--];
 
-		deleted = rt_node_search_inner(node, key, RT_ACTION_DELETE, NULL);
+		deleted = rt_node_delete_inner(node, key);
 		Assert(deleted);
 
 		/* If the node didn't become empty, we stop deleting the key */
@@ -1994,84 +1400,9 @@ rt_iter_update_key(rt_iter *iter, uint8 chunk, uint8 shift)
 static inline rt_node *
 rt_node_inner_iterate_next(rt_iter *iter, rt_node_iter *node_iter)
 {
-	rt_node    *child = NULL;
-	bool		found = false;
-	uint8		key_chunk;
-
-	switch (node_iter->node->kind)
-	{
-		case RT_NODE_KIND_4:
-			{
-				rt_node_inner_4 *n4 = (rt_node_inner_4 *) node_iter->node;
-
-				node_iter->current_idx++;
-				if (node_iter->current_idx >= n4->base.n.count)
-					break;
-
-				child = n4->children[node_iter->current_idx];
-				key_chunk = n4->base.chunks[node_iter->current_idx];
-				found = true;
-				break;
-			}
-		case RT_NODE_KIND_32:
-			{
-				rt_node_inner_32 *n32 = (rt_node_inner_32 *) node_iter->node;
-
-				node_iter->current_idx++;
-				if (node_iter->current_idx >= n32->base.n.count)
-					break;
-
-				child = n32->children[node_iter->current_idx];
-				key_chunk = n32->base.chunks[node_iter->current_idx];
-				found = true;
-				break;
-			}
-		case RT_NODE_KIND_125:
-			{
-				rt_node_inner_125 *n125 = (rt_node_inner_125 *) node_iter->node;
-				int			i;
-
-				for (i = node_iter->current_idx + 1; i < RT_NODE_MAX_SLOTS; i++)
-				{
-					if (node_125_is_chunk_used((rt_node_base_125 *) n125, i))
-						break;
-				}
-
-				if (i >= RT_NODE_MAX_SLOTS)
-					break;
-
-				node_iter->current_idx = i;
-				child = node_inner_125_get_child(n125, i);
-				key_chunk = i;
-				found = true;
-				break;
-			}
-		case RT_NODE_KIND_256:
-			{
-				rt_node_inner_256 *n256 = (rt_node_inner_256 *) node_iter->node;
-				int			i;
-
-				for (i = node_iter->current_idx + 1; i < RT_NODE_MAX_SLOTS; i++)
-				{
-					if (node_inner_256_is_chunk_used(n256, i))
-						break;
-				}
-
-				if (i >= RT_NODE_MAX_SLOTS)
-					break;
-
-				node_iter->current_idx = i;
-				child = node_inner_256_get_child(n256, i);
-				key_chunk = i;
-				found = true;
-				break;
-			}
-	}
-
-	if (found)
-		rt_iter_update_key(iter, key_chunk, node_iter->node->shift);
-
-	return child;
+#define RT_NODE_LEVEL_INNER
+#include "lib/radixtree_iter_impl.h"
+#undef RT_NODE_LEVEL_INNER
 }
 
 /*
@@ -2082,88 +1413,9 @@ static inline bool
 rt_node_leaf_iterate_next(rt_iter *iter, rt_node_iter *node_iter,
 						  uint64 *value_p)
 {
-	rt_node    *node = node_iter->node;
-	bool		found = false;
-	uint64		value;
-	uint8		key_chunk;
-
-	switch (node->kind)
-	{
-		case RT_NODE_KIND_4:
-			{
-				rt_node_leaf_4 *n4 = (rt_node_leaf_4 *) node_iter->node;
-
-				node_iter->current_idx++;
-				if (node_iter->current_idx >= n4->base.n.count)
-					break;
-
-				value = n4->values[node_iter->current_idx];
-				key_chunk = n4->base.chunks[node_iter->current_idx];
-				found = true;
-				break;
-			}
-		case RT_NODE_KIND_32:
-			{
-				rt_node_leaf_32 *n32 = (rt_node_leaf_32 *) node_iter->node;
-
-				node_iter->current_idx++;
-				if (node_iter->current_idx >= n32->base.n.count)
-					break;
-
-				value = n32->values[node_iter->current_idx];
-				key_chunk = n32->base.chunks[node_iter->current_idx];
-				found = true;
-				break;
-			}
-		case RT_NODE_KIND_125:
-			{
-				rt_node_leaf_125 *n125 = (rt_node_leaf_125 *) node_iter->node;
-				int			i;
-
-				for (i = node_iter->current_idx + 1; i < RT_NODE_MAX_SLOTS; i++)
-				{
-					if (node_125_is_chunk_used((rt_node_base_125 *) n125, i))
-						break;
-				}
-
-				if (i >= RT_NODE_MAX_SLOTS)
-					break;
-
-				node_iter->current_idx = i;
-				value = node_leaf_125_get_value(n125, i);
-				key_chunk = i;
-				found = true;
-				break;
-			}
-		case RT_NODE_KIND_256:
-			{
-				rt_node_leaf_256 *n256 = (rt_node_leaf_256 *) node_iter->node;
-				int			i;
-
-				for (i = node_iter->current_idx + 1; i < RT_NODE_MAX_SLOTS; i++)
-				{
-					if (node_leaf_256_is_chunk_used(n256, i))
-						break;
-				}
-
-				if (i >= RT_NODE_MAX_SLOTS)
-					break;
-
-				node_iter->current_idx = i;
-				value = node_leaf_256_get_value(n256, i);
-				key_chunk = i;
-				found = true;
-				break;
-			}
-	}
-
-	if (found)
-	{
-		rt_iter_update_key(iter, key_chunk, node_iter->node->shift);
-		*value_p = value;
-	}
-
-	return found;
+#define RT_NODE_LEVEL_LEAF
+#include "lib/radixtree_iter_impl.h"
+#undef RT_NODE_LEVEL_LEAF
 }
 
 /*
@@ -2229,6 +1481,7 @@ rt_verify_node(rt_node *node)
 				for (int i = 0; i < RT_NODE_MAX_SLOTS; i++)
 				{
 					uint8		slot = n125->slot_idxs[i];
+					int			idx = BM_IDX(slot);
 					int			bitnum = BM_BIT(slot);
 
 					if (!node_125_is_chunk_used(n125, i))
@@ -2236,7 +1489,7 @@ rt_verify_node(rt_node *node)
 
 					/* Check if the corresponding slot is used */
 					Assert(slot < node->fanout);
-					Assert((n125->isset[i] & ((bitmapword) 1 << bitnum)) != 0);
+					Assert((n125->isset[idx] & ((bitmapword) 1 << bitnum)) != 0);
 
 					cnt++;
 				}
@@ -2476,12 +1729,12 @@ rt_dump_search(radix_tree *tree, uint64 key)
 			uint64		dummy;
 
 			/* We reached at a leaf node, find the corresponding slot */
-			rt_node_search_leaf(node, key, RT_ACTION_FIND, &dummy);
+			rt_node_search_leaf(node, key, &dummy);
 
 			break;
 		}
 
-		if (!rt_node_search_inner(node, key, RT_ACTION_FIND, &child))
+		if (!rt_node_search_inner(node, key, &child))
 			break;
 
 		node = child;
diff --git a/src/include/lib/radixtree_delete_impl.h b/src/include/lib/radixtree_delete_impl.h
new file mode 100644
index 0000000000..24fd9cc02b
--- /dev/null
+++ b/src/include/lib/radixtree_delete_impl.h
@@ -0,0 +1,100 @@
+/* TODO: shrink nodes */
+
+#if defined(RT_NODE_LEVEL_INNER)
+#define RT_NODE4_TYPE rt_node_inner_4
+#define RT_NODE32_TYPE rt_node_inner_32
+#define RT_NODE125_TYPE rt_node_inner_125
+#define RT_NODE256_TYPE rt_node_inner_256
+#elif defined(RT_NODE_LEVEL_LEAF)
+#define RT_NODE4_TYPE rt_node_leaf_4
+#define RT_NODE32_TYPE rt_node_leaf_32
+#define RT_NODE125_TYPE rt_node_leaf_125
+#define RT_NODE256_TYPE rt_node_leaf_256
+#else
+#error node level must be either inner or leaf
+#endif
+
+	uint8		chunk = RT_GET_KEY_CHUNK(key, node->shift);
+
+	switch (node->kind)
+	{
+		case RT_NODE_KIND_4:
+			{
+				RT_NODE4_TYPE *n4 = (RT_NODE4_TYPE *) node;
+				int			idx = node_4_search_eq((rt_node_base_4 *) n4, chunk);
+
+				if (idx < 0)
+					return false;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				chunk_values_array_delete(n4->base.chunks, (uint64 *) n4->values,
+										  n4->base.n.count, idx);
+#else
+				chunk_children_array_delete(n4->base.chunks, n4->children,
+											n4->base.n.count, idx);
+#endif
+				break;
+			}
+		case RT_NODE_KIND_32:
+			{
+				RT_NODE32_TYPE *n32 = (RT_NODE32_TYPE *) node;
+				int			idx = node_32_search_eq((rt_node_base_32 *) n32, chunk);
+
+				if (idx < 0)
+					return false;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				chunk_values_array_delete(n32->base.chunks, (uint64 *) n32->values,
+										  n32->base.n.count, idx);
+#else
+				chunk_children_array_delete(n32->base.chunks, n32->children,
+											n32->base.n.count, idx);
+#endif
+				break;
+			}
+		case RT_NODE_KIND_125:
+			{
+				RT_NODE125_TYPE *n125 = (RT_NODE125_TYPE *) node;
+				int			slotpos = n125->base.slot_idxs[chunk];
+				int			idx;
+				int			bitnum;
+
+				if (slotpos == RT_NODE_125_INVALID_IDX)
+					return false;
+
+				idx = BM_IDX(slotpos);
+				bitnum = BM_BIT(slotpos);
+				n125->base.isset[idx] &= ~((bitmapword) 1 << bitnum);
+				n125->base.slot_idxs[chunk] = RT_NODE_125_INVALID_IDX;
+
+				break;
+			}
+		case RT_NODE_KIND_256:
+			{
+				RT_NODE256_TYPE *n256 = (RT_NODE256_TYPE *) node;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				if (!node_leaf_256_is_chunk_used(n256, chunk))
+#else
+				if (!node_inner_256_is_chunk_used(n256, chunk))
+#endif
+					return false;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				node_leaf_256_delete(n256, chunk);
+#else
+				node_inner_256_delete(n256, chunk);
+#endif
+				break;
+			}
+	}
+
+	/* update statistics */
+	node->count--;
+
+	return true;
+
+#undef RT_NODE4_TYPE
+#undef RT_NODE32_TYPE
+#undef RT_NODE125_TYPE
+#undef RT_NODE256_TYPE
diff --git a/src/include/lib/radixtree_insert_impl.h b/src/include/lib/radixtree_insert_impl.h
new file mode 100644
index 0000000000..c63fe9a3c0
--- /dev/null
+++ b/src/include/lib/radixtree_insert_impl.h
@@ -0,0 +1,293 @@
+#if defined(RT_NODE_LEVEL_INNER)
+#define RT_NODE4_TYPE rt_node_inner_4
+#define RT_NODE32_TYPE rt_node_inner_32
+#define RT_NODE125_TYPE rt_node_inner_125
+#define RT_NODE256_TYPE rt_node_inner_256
+#elif defined(RT_NODE_LEVEL_LEAF)
+#define RT_NODE4_TYPE rt_node_leaf_4
+#define RT_NODE32_TYPE rt_node_leaf_32
+#define RT_NODE125_TYPE rt_node_leaf_125
+#define RT_NODE256_TYPE rt_node_leaf_256
+#else
+#error node level must be either inner or leaf
+#endif
+
+	uint8		chunk = RT_GET_KEY_CHUNK(key, node->shift);
+	bool		chunk_exists = false;
+	rt_node		*newnode = NULL;
+
+#ifdef RT_NODE_LEVEL_LEAF
+	Assert(NODE_IS_LEAF(node));
+#else
+	Assert(!NODE_IS_LEAF(node));
+#endif
+
+	switch (node->kind)
+	{
+		case RT_NODE_KIND_4:
+			{
+				RT_NODE4_TYPE *n4 = (RT_NODE4_TYPE *) node;
+				int			idx;
+
+				idx = node_4_search_eq(&n4->base, chunk);
+				if (idx != -1)
+				{
+					/* found the existing chunk */
+					chunk_exists = true;
+#ifdef RT_NODE_LEVEL_LEAF
+					n4->values[idx] = value;
+#else
+					n4->children[idx] = child;
+#endif
+					break;
+				}
+
+				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n4)))
+				{
+					RT_NODE32_TYPE *new32;
+
+					/* grow node from 4 to 32 */
+					newnode = rt_grow_node_kind(tree, node, RT_NODE_KIND_32);
+					new32 = (RT_NODE32_TYPE *) newnode;
+#ifdef RT_NODE_LEVEL_LEAF
+					chunk_values_array_copy(n4->base.chunks, n4->values,
+											  new32->base.chunks, new32->values);
+#else
+					chunk_children_array_copy(n4->base.chunks, n4->children,
+											  new32->base.chunks, new32->children);
+#endif
+					Assert(parent != NULL);
+					rt_replace_node(tree, parent, node, newnode, key);
+					node = newnode;
+				}
+				else
+				{
+					int			insertpos = node_4_get_insertpos(&n4->base, chunk);
+					int			count = n4->base.n.count;
+
+					/* shift chunks and children */
+					if (insertpos < count)
+					{
+						Assert(count > 0);
+#ifdef RT_NODE_LEVEL_LEAF
+						chunk_values_array_shift(n4->base.chunks, n4->values,
+												   count, insertpos);
+#else
+						chunk_children_array_shift(n4->base.chunks, n4->children,
+												   count, insertpos);
+#endif
+					}
+
+					n4->base.chunks[insertpos] = chunk;
+#ifdef RT_NODE_LEVEL_LEAF
+					n4->values[insertpos] = value;
+#else
+					n4->children[insertpos] = child;
+#endif
+					break;
+				}
+			}
+			/* FALLTHROUGH */
+		case RT_NODE_KIND_32:
+			{
+				const rt_size_class_elem minclass = rt_size_class_info[RT_CLASS_32_PARTIAL];
+				const rt_size_class_elem maxclass = rt_size_class_info[RT_CLASS_32_FULL];
+				RT_NODE32_TYPE *n32 = (RT_NODE32_TYPE *) node;
+				int			idx;
+
+				idx = node_32_search_eq(&n32->base, chunk);
+				if (idx != -1)
+				{
+					/* found the existing chunk */
+					chunk_exists = true;
+#ifdef RT_NODE_LEVEL_LEAF
+					n32->values[idx] = value;
+#else
+					n32->children[idx] = child;
+#endif
+					break;
+				}
+
+				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n32)) &&
+					n32->base.n.fanout == minclass.fanout)
+				{
+					/* grow to the next size class of this kind */
+#ifdef RT_NODE_LEVEL_LEAF
+					newnode = rt_alloc_node(tree, RT_CLASS_32_FULL, false);
+					memcpy(newnode, node, minclass.leaf_size);
+#else
+					newnode = rt_alloc_node(tree, RT_CLASS_32_FULL, true);
+					memcpy(newnode, node, minclass.inner_size);
+#endif
+					newnode->fanout = maxclass.fanout;
+
+					Assert(parent != NULL);
+					rt_replace_node(tree, parent, node, newnode, key);
+					node = newnode;
+
+					/* also update pointer for this kind */
+					n32 = (RT_NODE32_TYPE *) newnode;
+				}
+
+				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n32)))
+				{
+					RT_NODE125_TYPE *new125;
+
+					Assert(n32->base.n.fanout == maxclass.fanout);
+
+					/* grow node from 32 to 125 */
+					newnode = rt_grow_node_kind(tree, node, RT_NODE_KIND_125);
+					new125 = (RT_NODE125_TYPE *) newnode;
+
+					for (int i = 0; i < maxclass.fanout; i++)
+					{
+						new125->base.slot_idxs[n32->base.chunks[i]] = i;
+#ifdef RT_NODE_LEVEL_LEAF
+						new125->values[i] = n32->values[i];
+#else
+						new125->children[i] = n32->children[i];
+#endif
+					}
+
+					Assert(maxclass.fanout <= sizeof(bitmapword) * BITS_PER_BYTE);
+					new125->base.isset[0] = (bitmapword) (((uint64) 1 << maxclass.fanout) - 1);
+
+					Assert(parent != NULL);
+					rt_replace_node(tree, parent, node, newnode, key);
+					node = newnode;
+				}
+				else
+				{
+					int	insertpos = node_32_get_insertpos(&n32->base, chunk);
+					int count = n32->base.n.count;
+
+					if (insertpos < count)
+					{
+						Assert(count > 0);
+#ifdef RT_NODE_LEVEL_LEAF
+						chunk_values_array_shift(n32->base.chunks, n32->values,
+												   count, insertpos);
+#else
+						chunk_children_array_shift(n32->base.chunks, n32->children,
+												   count, insertpos);
+#endif
+					}
+
+					n32->base.chunks[insertpos] = chunk;
+#ifdef RT_NODE_LEVEL_LEAF
+					n32->values[insertpos] = value;
+#else
+					n32->children[insertpos] = child;
+#endif
+					break;
+				}
+			}
+			/* FALLTHROUGH */
+		case RT_NODE_KIND_125:
+			{
+				RT_NODE125_TYPE *n125 = (RT_NODE125_TYPE *) node;
+				int			slotpos = n125->base.slot_idxs[chunk];
+				int			cnt = 0;
+
+				if (slotpos != RT_NODE_125_INVALID_IDX)
+				{
+					/* found the existing chunk */
+					chunk_exists = true;
+#ifdef RT_NODE_LEVEL_LEAF
+					n125->values[slotpos] = value;
+#else
+					n125->children[slotpos] = child;
+#endif
+					break;
+				}
+
+				if (unlikely(!VAR_NODE_HAS_FREE_SLOT(n125)))
+				{
+					RT_NODE256_TYPE *new256;
+
+					/* grow node from 125 to 256 */
+					newnode = rt_grow_node_kind(tree, node, RT_NODE_KIND_256);
+					new256 = (RT_NODE256_TYPE *) newnode;
+					for (int i = 0; i < RT_NODE_MAX_SLOTS && cnt < n125->base.n.count; i++)
+					{
+						if (!node_125_is_chunk_used(&n125->base, i))
+							continue;
+#ifdef RT_NODE_LEVEL_LEAF
+						node_leaf_256_set(new256, i, node_leaf_125_get_value(n125, i));
+#else
+						node_inner_256_set(new256, i, node_inner_125_get_child(n125, i));
+#endif
+						cnt++;
+					}
+
+					Assert(parent != NULL);
+					rt_replace_node(tree, parent, node, newnode, key);
+					node = newnode;
+				}
+				else
+				{
+					int			idx;
+					bitmapword	inverse;
+
+					/* get the first word with at least one bit not set */
+					for (idx = 0; idx < BM_IDX(128); idx++)
+					{
+						if (n125->base.isset[idx] < ~((bitmapword) 0))
+							break;
+					}
+
+					/* To get the first unset bit in X, get the first set bit in ~X */
+					inverse = ~(n125->base.isset[idx]);
+					slotpos = idx * BITS_PER_BITMAPWORD;
+					slotpos += bmw_rightmost_one_pos(inverse);
+					Assert(slotpos < node->fanout);
+
+					/* mark the slot used */
+					n125->base.isset[idx] |= bmw_rightmost_one(inverse);
+					n125->base.slot_idxs[chunk] = slotpos;
+
+#ifdef RT_NODE_LEVEL_LEAF
+					n125->values[slotpos] = value;
+#else
+					n125->children[slotpos] = child;
+#endif
+					break;
+				}
+			}
+			/* FALLTHROUGH */
+		case RT_NODE_KIND_256:
+			{
+				RT_NODE256_TYPE *n256 = (RT_NODE256_TYPE *) node;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				chunk_exists = node_leaf_256_is_chunk_used(n256, chunk);
+#else
+				chunk_exists = node_inner_256_is_chunk_used(n256, chunk);
+#endif
+				Assert(chunk_exists || FIXED_NODE_HAS_FREE_SLOT(n256, RT_CLASS_256));
+
+#ifdef RT_NODE_LEVEL_LEAF
+				node_leaf_256_set(n256, chunk, value);
+#else
+				node_inner_256_set(n256, chunk, child);
+#endif
+				break;
+			}
+	}
+
+	/* Update statistics */
+	if (!chunk_exists)
+		node->count++;
+
+	/*
+	 * Done. Finally, verify the chunk and value is inserted or replaced
+	 * properly in the node.
+	 */
+	rt_verify_node(node);
+
+	return chunk_exists;
+
+#undef RT_NODE4_TYPE
+#undef RT_NODE32_TYPE
+#undef RT_NODE125_TYPE
+#undef RT_NODE256_TYPE
diff --git a/src/include/lib/radixtree_iter_impl.h b/src/include/lib/radixtree_iter_impl.h
new file mode 100644
index 0000000000..bebf8e725a
--- /dev/null
+++ b/src/include/lib/radixtree_iter_impl.h
@@ -0,0 +1,129 @@
+#if defined(RT_NODE_LEVEL_INNER)
+#define RT_NODE4_TYPE rt_node_inner_4
+#define RT_NODE32_TYPE rt_node_inner_32
+#define RT_NODE125_TYPE rt_node_inner_125
+#define RT_NODE256_TYPE rt_node_inner_256
+#elif defined(RT_NODE_LEVEL_LEAF)
+#define RT_NODE4_TYPE rt_node_leaf_4
+#define RT_NODE32_TYPE rt_node_leaf_32
+#define RT_NODE125_TYPE rt_node_leaf_125
+#define RT_NODE256_TYPE rt_node_leaf_256
+#else
+#error node level must be either inner or leaf
+#endif
+
+#ifdef RT_NODE_LEVEL_LEAF
+	uint64		value;
+#else
+	rt_node    *child = NULL;
+#endif
+	bool		found = false;
+	uint8		key_chunk;
+
+	switch (node_iter->node->kind)
+	{
+		case RT_NODE_KIND_4:
+			{
+				RT_NODE4_TYPE *n4 = (RT_NODE4_TYPE *) node_iter->node;
+
+				node_iter->current_idx++;
+				if (node_iter->current_idx >= n4->base.n.count)
+					break;
+#ifdef RT_NODE_LEVEL_LEAF
+				value = n4->values[node_iter->current_idx];
+#else
+				child = n4->children[node_iter->current_idx];
+#endif
+				key_chunk = n4->base.chunks[node_iter->current_idx];
+				found = true;
+				break;
+			}
+		case RT_NODE_KIND_32:
+			{
+				RT_NODE32_TYPE *n32 = (RT_NODE32_TYPE *) node_iter->node;
+
+				node_iter->current_idx++;
+				if (node_iter->current_idx >= n32->base.n.count)
+					break;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				value = n32->values[node_iter->current_idx];
+#else
+				child = n32->children[node_iter->current_idx];
+#endif
+				key_chunk = n32->base.chunks[node_iter->current_idx];
+				found = true;
+				break;
+			}
+		case RT_NODE_KIND_125:
+			{
+				RT_NODE125_TYPE *n125 = (RT_NODE125_TYPE *) node_iter->node;
+				int			i;
+
+				for (i = node_iter->current_idx + 1; i < RT_NODE_MAX_SLOTS; i++)
+				{
+					if (node_125_is_chunk_used((rt_node_base_125 *) n125, i))
+						break;
+				}
+
+				if (i >= RT_NODE_MAX_SLOTS)
+					break;
+
+				node_iter->current_idx = i;
+#ifdef RT_NODE_LEVEL_LEAF
+				value = node_leaf_125_get_value(n125, i);
+#else
+				child = node_inner_125_get_child(n125, i);
+#endif
+				key_chunk = i;
+				found = true;
+				break;
+			}
+		case RT_NODE_KIND_256:
+			{
+				RT_NODE256_TYPE *n256 = (RT_NODE256_TYPE *) node_iter->node;
+				int			i;
+
+				for (i = node_iter->current_idx + 1; i < RT_NODE_MAX_SLOTS; i++)
+				{
+#ifdef RT_NODE_LEVEL_LEAF
+					if (node_leaf_256_is_chunk_used(n256, i))
+#else
+					if (node_inner_256_is_chunk_used(n256, i))
+#endif
+						break;
+				}
+
+				if (i >= RT_NODE_MAX_SLOTS)
+					break;
+
+				node_iter->current_idx = i;
+#ifdef RT_NODE_LEVEL_LEAF
+				value = node_leaf_256_get_value(n256, i);
+#else
+				child = node_inner_256_get_child(n256, i);
+#endif
+				key_chunk = i;
+				found = true;
+				break;
+			}
+	}
+
+	if (found)
+	{
+		rt_iter_update_key(iter, key_chunk, node_iter->node->shift);
+#ifdef RT_NODE_LEVEL_LEAF
+		*value_p = value;
+#endif
+	}
+
+#ifdef RT_NODE_LEVEL_LEAF
+	return found;
+#else
+	return child;
+#endif
+
+#undef RT_NODE4_TYPE
+#undef RT_NODE32_TYPE
+#undef RT_NODE125_TYPE
+#undef RT_NODE256_TYPE
diff --git a/src/include/lib/radixtree_search_impl.h b/src/include/lib/radixtree_search_impl.h
new file mode 100644
index 0000000000..d0366f9bb6
--- /dev/null
+++ b/src/include/lib/radixtree_search_impl.h
@@ -0,0 +1,102 @@
+#if defined(RT_NODE_LEVEL_INNER)
+#define RT_NODE4_TYPE rt_node_inner_4
+#define RT_NODE32_TYPE rt_node_inner_32
+#define RT_NODE125_TYPE rt_node_inner_125
+#define RT_NODE256_TYPE rt_node_inner_256
+#elif defined(RT_NODE_LEVEL_LEAF)
+#define RT_NODE4_TYPE rt_node_leaf_4
+#define RT_NODE32_TYPE rt_node_leaf_32
+#define RT_NODE125_TYPE rt_node_leaf_125
+#define RT_NODE256_TYPE rt_node_leaf_256
+#else
+#error node level must be either inner or leaf
+#endif
+
+	uint8		chunk = RT_GET_KEY_CHUNK(key, node->shift);
+
+#ifdef RT_NODE_LEVEL_LEAF
+	uint64		value = 0;
+#else
+	rt_node    *child = NULL;
+#endif
+
+	switch (node->kind)
+	{
+		case RT_NODE_KIND_4:
+			{
+				RT_NODE4_TYPE *n4 = (RT_NODE4_TYPE *) node;
+				int			idx = node_4_search_eq((rt_node_base_4 *) n4, chunk);
+
+				if (idx < 0)
+					return false;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				value = n4->values[idx];
+#else
+				child = n4->children[idx];
+#endif
+				break;
+			}
+		case RT_NODE_KIND_32:
+			{
+				RT_NODE32_TYPE *n32 = (RT_NODE32_TYPE *) node;
+				int			idx = node_32_search_eq((rt_node_base_32 *) n32, chunk);
+
+				if (idx < 0)
+					return false;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				value = n32->values[idx];
+#else
+				child = n32->children[idx];
+#endif
+				break;
+			}
+		case RT_NODE_KIND_125:
+			{
+				RT_NODE125_TYPE *n125 = (RT_NODE125_TYPE *) node;
+
+				if (!node_125_is_chunk_used((rt_node_base_125 *) n125, chunk))
+					return false;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				value = node_leaf_125_get_value(n125, chunk);
+#else
+				child = node_inner_125_get_child(n125, chunk);
+#endif
+				break;
+			}
+		case RT_NODE_KIND_256:
+			{
+				RT_NODE256_TYPE *n256 = (RT_NODE256_TYPE *) node;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				if (!node_leaf_256_is_chunk_used(n256, chunk))
+#else
+				if (!node_inner_256_is_chunk_used(n256, chunk))
+#endif
+					return false;
+
+#ifdef RT_NODE_LEVEL_LEAF
+				value = node_leaf_256_get_value(n256, chunk);
+#else
+				child = node_inner_256_get_child(n256, chunk);
+#endif
+				break;
+			}
+	}
+
+#ifdef RT_NODE_LEVEL_LEAF
+	Assert(value_p != NULL);
+	*value_p = value;
+#else
+	Assert(child_p != NULL);
+	*child_p = child;
+#endif
+
+	return true;
+
+#undef RT_NODE4_TYPE
+#undef RT_NODE32_TYPE
+#undef RT_NODE125_TYPE
+#undef RT_NODE256_TYPE
diff --git a/src/tools/pginclude/cpluspluscheck b/src/tools/pginclude/cpluspluscheck
index e52fe9f509..09fa6e7432 100755
--- a/src/tools/pginclude/cpluspluscheck
+++ b/src/tools/pginclude/cpluspluscheck
@@ -101,6 +101,12 @@ do
 	test "$f" = src/include/nodes/nodetags.h && continue
 	test "$f" = src/backend/nodes/nodetags.h && continue
 
+	# radixtree_*_impl.h cannot be included standalone: they are just code fragments.
+	test "$f" = src/include/lib/radixtree_delete_impl.h && continue
+	test "$f" = src/include/lib/radixtree_insert_impl.h && continue
+	test "$f" = src/include/lib/radixtree_iter_impl.h && continue
+	test "$f" = src/include/lib/radixtree_search_impl.h && continue
+
 	# These files are not meant to be included standalone, because
 	# they contain lists that might have multiple use-cases.
 	test "$f" = src/include/access/rmgrlist.h && continue
diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck
index abbba7aa63..d4d2f1da03 100755
--- a/src/tools/pginclude/headerscheck
+++ b/src/tools/pginclude/headerscheck
@@ -96,6 +96,12 @@ do
 	test "$f" = src/include/nodes/nodetags.h && continue
 	test "$f" = src/backend/nodes/nodetags.h && continue
 
+	# radixtree_*_impl.h cannot be included standalone: they are just code fragments.
+	test "$f" = src/include/lib/radixtree_delete_impl.h && continue
+	test "$f" = src/include/lib/radixtree_insert_impl.h && continue
+	test "$f" = src/include/lib/radixtree_iter_impl.h && continue
+	test "$f" = src/include/lib/radixtree_search_impl.h && continue
+
 	# These files are not meant to be included standalone, because
 	# they contain lists that might have multiple use-cases.
 	test "$f" = src/include/access/rmgrlist.h && continue
-- 
2.39.0

