From 7161a81d2792e0e91340398684fe211c415912d8 Mon Sep 17 00:00:00 2001
From: Peter Geoghegan <pg@bowt.ie>
Date: Fri, 15 Jul 2016 13:21:57 -0700
Subject: [PATCH] Add regression test cases for hash tuplesorts.

This includes a subtle behavioral change:  We now treat
maintenance_work_mem, or the number of usable buffers (whichever is
less) as the cut-off point at which hash index builds use tuplesort
rather than inserting tuples into the index directly, in scan order.
Previously, only shared_buffers was used (and never temp_buffers in the
case of temporary relations).

In addition to adding a test of the basic sanity of a hash index built
with a spool/tuplesort, assert that hash values are found in order as
they are retrieved from tuplesort.

These tests would have caught bugs fixed by commit 8cf739de, commit
2720e96a, and possibly others.
---
 src/backend/access/hash/hash.c             | 22 +++++++++++++---
 src/backend/access/hash/hashsort.c         | 22 +++++++++++++---
 src/test/regress/expected/create_index.out |  6 +++++
 src/test/regress/expected/hash_index.out   | 40 ++++++++++++++++++++++++++++++
 src/test/regress/sql/create_index.sql      |  5 ++++
 src/test/regress/sql/hash_index.sql        | 27 ++++++++++++++++++++
 6 files changed, 116 insertions(+), 6 deletions(-)

diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 19695ee..e68dc7c 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -22,6 +22,7 @@
 #include "access/relscan.h"
 #include "catalog/index.h"
 #include "commands/vacuum.h"
+#include "miscadmin.h"
 #include "optimizer/plancat.h"
 #include "utils/index_selfuncs.h"
 #include "utils/rel.h"
@@ -97,6 +98,7 @@ hashbuild(Relation heap, Relation index, IndexInfo *indexInfo)
 	double		reltuples;
 	double		allvisfrac;
 	uint32		num_buckets;
+	uint32		num_buckets_sort_threshold;
 	HashBuildState buildstate;
 
 	/*
@@ -120,12 +122,26 @@ hashbuild(Relation heap, Relation index, IndexInfo *indexInfo)
 	 * then we'll thrash horribly.  To prevent that scenario, we can sort the
 	 * tuples by (expected) bucket number.  However, such a sort is useless
 	 * overhead when the index does fit in RAM.  We choose to sort if the
-	 * initial index size exceeds NBuffers.
+	 * initial index size exceeds maintenance_work_mem, or the number of
+	 * buffers usable by the relation, whichever is less.  (It may be useful
+	 * to sort in cases where NLocBuffer is quite low and thrashing between
+	 * local buffers and kernel cache can be minimized by sorting.  Seems
+	 * unlikely that NBuffers would ever be less than maintenance_work_mem,
+	 * though, but be consistent.)
 	 *
 	 * NOTE: this test will need adjustment if a bucket is ever different from
-	 * one page.
+	 * one page.  Also, "initial index size" accounting does not include the
+	 * metapage, nor the first bitmap page.
 	 */
-	if (num_buckets >= (uint32) NBuffers)
+	if (heap->rd_rel->relpersistence != RELPERSISTENCE_TEMP)
+		num_buckets_sort_threshold = NBuffers;
+	else
+		num_buckets_sort_threshold = NLocBuffer;
+
+	num_buckets_sort_threshold =
+		Min((uint32) maintenance_work_mem / (BLCKSZ / 1024),
+			num_buckets_sort_threshold);
+	if (num_buckets >= num_buckets_sort_threshold)
 		buildstate.spool = _h_spoolinit(heap, index, num_buckets);
 	else
 		buildstate.spool = NULL;
diff --git a/src/backend/access/hash/hashsort.c b/src/backend/access/hash/hashsort.c
index 51b9f3f..c1d70c1 100644
--- a/src/backend/access/hash/hashsort.c
+++ b/src/backend/access/hash/hashsort.c
@@ -37,6 +37,7 @@ struct HSpool
 {
 	Tuplesortstate *sortstate;	/* state data for tuplesort.c */
 	Relation	index;
+	uint32		hash_mask;		/* bitmask for hash codes */
 };
 
 
@@ -47,7 +48,6 @@ HSpool *
 _h_spoolinit(Relation heap, Relation index, uint32 num_buckets)
 {
 	HSpool	   *hspool = (HSpool *) palloc0(sizeof(HSpool));
-	uint32		hash_mask;
 
 	hspool->index = index;
 
@@ -60,7 +60,7 @@ _h_spoolinit(Relation heap, Relation index, uint32 num_buckets)
 	 * we could just compute num_buckets - 1.  We prefer not to assume that
 	 * here, though.
 	 */
-	hash_mask = (((uint32) 1) << _hash_log2(num_buckets)) - 1;
+	hspool->hash_mask = (((uint32) 1) << _hash_log2(num_buckets)) - 1;
 
 	/*
 	 * We size the sort area as maintenance_work_mem rather than work_mem to
@@ -69,7 +69,7 @@ _h_spoolinit(Relation heap, Relation index, uint32 num_buckets)
 	 */
 	hspool->sortstate = tuplesort_begin_index_hash(heap,
 												   index,
-												   hash_mask,
+												   hspool->hash_mask,
 												   maintenance_work_mem,
 												   false);
 
@@ -105,12 +105,28 @@ _h_indexbuild(HSpool *hspool)
 {
 	IndexTuple	itup;
 	bool		should_free;
+#ifdef USE_ASSERT_CHECKING
+	uint32		lasthashkey;
+	uint32		hashkey = 0;
+#endif
 
 	tuplesort_performsort(hspool->sortstate);
 
 	while ((itup = tuplesort_getindextuple(hspool->sortstate,
 										   true, &should_free)) != NULL)
 	{
+		/*
+		 * Technically, it isn't critical that hash keys be found in sorted
+		 * order, since sorting is only ever used to increase locality of
+		 * access as a performance optimization.  It still seems like a good
+		 * idea to test tuplesort.c's handling of hash index tuple sorts
+		 * through an assertion, though.
+		 */
+#ifdef USE_ASSERT_CHECKING
+		lasthashkey = hashkey;
+		hashkey = _hash_get_indextuple_hashkey(itup) & hspool->hash_mask;
+		Assert(hashkey >= lasthashkey);
+#endif
 		_hash_doinsert(hspool->index, itup);
 		if (should_free)
 			pfree(itup);
diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out
index 97cc49e..a138c34 100644
--- a/src/test/regress/expected/create_index.out
+++ b/src/test/regress/expected/create_index.out
@@ -2346,6 +2346,12 @@ CREATE UNLOGGED TABLE unlogged_hash_table (id int4);
 CREATE INDEX unlogged_hash_index ON unlogged_hash_table USING hash (id int4_ops);
 DROP TABLE unlogged_hash_table;
 -- CREATE INDEX hash_ovfl_index ON hash_ovfl_heap USING hash (x int4_ops);
+-- Test hash index build tuplesorting.  Force hash tuplesort using low
+-- maintenance_work_mem setting and fillfactor:
+SET maintenance_work_mem = '1MB';
+CREATE INDEX hash_tuplesort_idx ON tenk1 USING hash (stringu1 name_ops) WITH (fillfactor = 10);
+WARNING:  hash indexes are not WAL-logged and their use is discouraged
+RESET maintenance_work_mem;
 --
 -- Test functional index
 --
diff --git a/src/test/regress/expected/hash_index.out b/src/test/regress/expected/hash_index.out
index 22835f8..fb8cc5b 100644
--- a/src/test/regress/expected/hash_index.out
+++ b/src/test/regress/expected/hash_index.out
@@ -187,6 +187,46 @@ SELECT h.seqno AS f20000
   20000
 (1 row)
 
+--
+-- Test index previously created with hash tuplesort
+--
+SET enable_seqscan = off;
+SET enable_bitmapscan = off;
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'ANAAAA';
+ sum | sum  
+-----+------
+ 750 | 6050
+(1 row)
+
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'UPAAAA';
+ sum | sum  
+-----+------
+ 830 | 7130
+(1 row)
+
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'OQAAAA';
+ sum | sum  
+-----+------
+ 730 | 7430
+(1 row)
+
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'NOT PRESENT';
+ sum | sum 
+-----+-----
+     |    
+(1 row)
+
+DROP INDEX hash_tuplesort_idx;
+RESET enable_seqscan;
+RESET enable_bitmapscan;
 -- UPDATE hash_ovfl_heap
 --    SET x = 1000
 --   WHERE x = 90;
diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql
index 7c582ea..31946e9 100644
--- a/src/test/regress/sql/create_index.sql
+++ b/src/test/regress/sql/create_index.sql
@@ -690,6 +690,11 @@ DROP TABLE unlogged_hash_table;
 
 -- CREATE INDEX hash_ovfl_index ON hash_ovfl_heap USING hash (x int4_ops);
 
+-- Test hash index build tuplesorting.  Force hash tuplesort using low
+-- maintenance_work_mem setting and fillfactor:
+SET maintenance_work_mem = '1MB';
+CREATE INDEX hash_tuplesort_idx ON tenk1 USING hash (stringu1 name_ops) WITH (fillfactor = 10);
+RESET maintenance_work_mem;
 
 --
 -- Test functional index
diff --git a/src/test/regress/sql/hash_index.sql b/src/test/regress/sql/hash_index.sql
index 411e8ae..062f354 100644
--- a/src/test/regress/sql/hash_index.sql
+++ b/src/test/regress/sql/hash_index.sql
@@ -139,6 +139,33 @@ SELECT h.seqno AS f20000
    FROM hash_f8_heap h
    WHERE h.random = '488912369'::float8;
 
+--
+-- Test index previously created with hash tuplesort
+--
+
+SET enable_seqscan = off;
+SET enable_bitmapscan = off;
+
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'ANAAAA';
+
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'UPAAAA';
+
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'OQAAAA';
+
+SELECT SUM(hundred), SUM(thousand)
+   FROM tenk1 h
+   WHERE h.stringu1 = 'NOT PRESENT';
+
+DROP INDEX hash_tuplesort_idx;
+RESET enable_seqscan;
+RESET enable_bitmapscan;
+
 -- UPDATE hash_ovfl_heap
 --    SET x = 1000
 --   WHERE x = 90;
-- 
2.7.4

