From 193ebcd603b321ec952f3ef43130282ab3c699bf Mon Sep 17 00:00:00 2001
From: James Coleman <jtc331@gmail.com>
Date: Fri, 27 Mar 2020 10:39:04 -0400
Subject: [PATCH v42 08/12] explain output munging and bug fixes

---
 src/backend/commands/explain.c                |  22 +-
 .../regress/expected/incremental_sort.out     | 320 ++++++++++--------
 src/test/regress/sql/incremental_sort.sql     | 118 ++++++-
 3 files changed, 311 insertions(+), 149 deletions(-)

diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 56f8e1fd21..39d51848b6 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -2784,26 +2784,38 @@ show_incremental_sort_group_info(IncrementalSortGroupInfo *groupInfo,
 		{
 			long		avgSpace = groupInfo->totalMemorySpaceUsed / groupInfo->groupCount;
 			const char *spaceTypeName;
+			StringInfoData memoryName;
+
+			spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_MEMORY);
+			initStringInfo(&memoryName);
+			appendStringInfo(&memoryName, "Sort Space %s", spaceTypeName);
+			ExplainOpenGroup("Sort Space", memoryName.data, true, es);
 
 			ExplainPropertyInteger("Average Sort Space Used", "kB", avgSpace, es);
 			ExplainPropertyInteger("Maximum Sort Space Used", "kB",
 								   groupInfo->maxMemorySpaceUsed, es);
-			spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_MEMORY);
-			ExplainPropertyText("Sort Space Type", spaceTypeName, es);
+
+			ExplainCloseGroup("Sort Spaces", memoryName.data, true, es);
 		}
 		if (groupInfo->maxDiskSpaceUsed > 0)
 		{
 			long		avgSpace = groupInfo->totalDiskSpaceUsed / groupInfo->groupCount;
 			const char *spaceTypeName;
+			StringInfoData diskName;
+
+			spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_DISK);
+			initStringInfo(&diskName);
+			appendStringInfo(&diskName, "Sort Space %s", spaceTypeName);
+			ExplainOpenGroup("Sort Space", diskName.data, true, es);
 
 			ExplainPropertyInteger("Average Sort Space Used", "kB", avgSpace, es);
 			ExplainPropertyInteger("Maximum Sort Space Used", "kB",
 								   groupInfo->maxDiskSpaceUsed, es);
-			spaceTypeName = tuplesort_space_type_name(SORT_SPACE_TYPE_MEMORY);
-			ExplainPropertyText("Sort Space Type", spaceTypeName, es);
+
+			ExplainCloseGroup("Sort Spaces", diskName.data, true, es);
 		}
 
-		ExplainCloseGroup("Incremental Sort Groups", "XXX Groups", true, es);
+		ExplainCloseGroup("Incremental Sort Groups", groupName.data, true, es);
 	}
 }
 
diff --git a/src/test/regress/expected/incremental_sort.out b/src/test/regress/expected/incremental_sort.out
index 65604b3429..ebb8412237 100644
--- a/src/test/regress/expected/incremental_sort.out
+++ b/src/test/regress/expected/incremental_sort.out
@@ -44,6 +44,102 @@ select * from (select * from tenk1 order by four) t order by four, ten;
 
 reset work_mem;
 create table t(a integer, b integer);
+create or replace function explain_analyze_without_memory(query text)
+returns table (out_line text) language plpgsql
+as
+$$
+declare
+  line text;
+begin
+  for line in
+    execute 'explain (analyze, costs off, summary off, timing off) ' || query
+  loop
+    out_line := regexp_replace(line, '\d+kB', 'NNkB', 'g');
+    return next;
+  end loop;
+end;
+$$;
+create or replace function explain_analyze_inc_sort_nodes(query text)
+returns jsonb language plpgsql
+as
+$$
+declare
+  elements jsonb;
+  element jsonb;
+  matching_nodes jsonb := '[]'::jsonb;
+begin
+  execute 'explain (analyze, costs off, summary off, timing off, format ''json'') ' || query into strict elements;
+  while jsonb_array_length(elements) > 0 loop
+    element := elements->0;
+    elements := elements - 0;
+    case jsonb_typeof(element)
+    when 'array' then
+      if jsonb_array_length(element) > 0 then
+        elements := elements || element;
+      end if;
+    when 'object' then
+      if element ? 'Plan' then
+        elements := elements || jsonb_build_array(element->'Plan');
+        element := element - 'Plan';
+      else
+        if element ? 'Plans' then
+          elements := elements || jsonb_build_array(element->'Plans');
+          element := element - 'Plans';
+        end if;
+        if (element->>'Node Type')::text = 'Incremental Sort' then
+          matching_nodes := matching_nodes || element;
+        end if;
+      end if;
+    end case;
+  end loop;
+  return matching_nodes;
+end;
+$$;
+create or replace function explain_analyze_inc_sort_nodes_without_memory(query text)
+returns jsonb language plpgsql
+as
+$$
+declare
+  nodes jsonb := '[]'::jsonb;
+  node jsonb;
+  group_key text;
+  space_key text;
+begin
+  for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop
+    for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop
+      for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop
+        node := jsonb_set(node, array[group_key, space_key, 'Average Sort Space Used'], '"NN"', false);
+        node := jsonb_set(node, array[group_key, space_key, 'Maximum Sort Space Used'], '"NN"', false);
+      end loop;
+    end loop;
+    nodes := nodes || node;
+  end loop;
+  return nodes;
+end;
+$$;
+create or replace function explain_analyze_inc_sort_nodes_verify_invariants(query text)
+returns bool language plpgsql
+as
+$$
+declare
+  node jsonb;
+  group_stats jsonb;
+  group_key text;
+  space_key text;
+begin
+  for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop
+    for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop
+      group_stats := node->group_key;
+      for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop
+        if (group_stats->space_key->'Maximum Sort Space Used')::bigint < (group_stats->space_key->'Maximum Sort Space Used')::bigint then
+          raise exception '% has invalid max space < average space', group_key;
+        end if;
+      end loop;
+    end loop;
+  end loop;
+  return true;
+end;
+$$;
 -- A single large group tested around each mode transition point.
 insert into t(a, b) select 1, i from generate_series(1, 100) n(i);
 explain (costs off) select * from (select * from t order by a) s order by a, b limit 31;
@@ -433,82 +529,59 @@ select * from (select * from t order by a) s order by a, b limit 55;
  2 | 55
 (55 rows)
 
--- Test EXPLAIN ANALYZE (text output) with only a fullsort group.
-explain (analyze, costs off, summary off, timing off)
-select * from (select * from t order by a) s order by a, b limit 55;
-                                           QUERY PLAN                                            
+-- Test EXPLAIN ANALYZE with only a fullsort group.
+select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 55');
+                                 explain_analyze_without_memory                                  
 -------------------------------------------------------------------------------------------------
  Limit (actual rows=55 loops=1)
    ->  Incremental Sort (actual rows=55 loops=1)
          Sort Key: t.a, t.b
          Presorted Key: t.a
-         Full-sort Groups: 2 (Methods: quicksort, top-N heapsort) Memory: 27kB (avg), 27kB (max)
+         Full-sort Groups: 2 (Methods: quicksort, top-N heapsort) Memory: NNkB (avg), NNkB (max)
          ->  Sort (actual rows=100 loops=1)
                Sort Key: t.a
-               Sort Method: quicksort  Memory: 30kB
+               Sort Method: quicksort  Memory: NNkB
                ->  Seq Scan on t (actual rows=100 loops=1)
 (9 rows)
 
-explain (analyze, costs off, summary off, timing off, format json)
-select * from (select * from t order by a) s order by a, b limit 55;
-                            QUERY PLAN                             
--------------------------------------------------------------------
- [                                                                +
-   {                                                              +
-     "Plan": {                                                    +
-       "Node Type": "Limit",                                      +
-       "Parallel Aware": false,                                   +
-       "Actual Rows": 55,                                         +
-       "Actual Loops": 1,                                         +
-       "Plans": [                                                 +
-         {                                                        +
-           "Node Type": "Incremental Sort",                       +
-           "Parent Relationship": "Outer",                        +
-           "Parallel Aware": false,                               +
-           "Actual Rows": 55,                                     +
-           "Actual Loops": 1,                                     +
-           "Sort Key": ["t.a", "t.b"],                            +
-           "Presorted Key": ["t.a"],                              +
-           "Full-sort Groups": {                                  +
-             "Group Count": 2,                                    +
-             "Sort Methods Used": ["quicksort", "top-N heapsort"],+
-             "Average Sort Space Used": 27,                       +
-             "Maximum Sort Space Used": 27,                       +
-             "Sort Space Type": "Memory"                          +
-           },                                                     +
-           "Plans": [                                             +
-             {                                                    +
-               "Node Type": "Sort",                               +
-               "Parent Relationship": "Outer",                    +
-               "Parallel Aware": false,                           +
-               "Actual Rows": 100,                                +
-               "Actual Loops": 1,                                 +
-               "Sort Key": ["t.a"],                               +
-               "Sort Method": "quicksort",                        +
-               "Sort Space Used": 30,                             +
-               "Sort Space Type": "Memory",                       +
-               "Plans": [                                         +
-                 {                                                +
-                   "Node Type": "Seq Scan",                       +
-                   "Parent Relationship": "Outer",                +
-                   "Parallel Aware": false,                       +
-                   "Relation Name": "t",                          +
-                   "Alias": "t",                                  +
-                   "Actual Rows": 100,                            +
-                   "Actual Loops": 1                              +
-                 }                                                +
-               ]                                                  +
-             }                                                    +
-           ]                                                      +
-         }                                                        +
-       ]                                                          +
-     },                                                           +
-     "Triggers": [                                                +
-     ]                                                            +
-   }                                                              +
+select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 55'));
+                   jsonb_pretty                   
+--------------------------------------------------
+ [                                               +
+     {                                           +
+         "Sort Key": [                           +
+             "t.a",                              +
+             "t.b"                               +
+         ],                                      +
+         "Node Type": "Incremental Sort",        +
+         "Actual Rows": 55,                      +
+         "Actual Loops": 1,                      +
+         "Presorted Key": [                      +
+             "t.a"                               +
+         ],                                      +
+         "Parallel Aware": false,                +
+         "Full-sort Groups": {                   +
+             "Group Count": 2,                   +
+             "Sort Methods Used": [              +
+                 "quicksort",                    +
+                 "top-N heapsort"                +
+             ],                                  +
+             "Sort Space Memory": {              +
+                 "Average Sort Space Used": "NN",+
+                 "Maximum Sort Space Used": "NN" +
+             }                                   +
+         },                                      +
+         "Parent Relationship": "Outer"          +
+     }                                           +
  ]
 (1 row)
 
+select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 55');
+ explain_analyze_inc_sort_nodes_verify_invariants 
+--------------------------------------------------
+ t
+(1 row)
+
 delete from t;
 -- An initial small group followed by a large group.
 insert into t(a, b) select (case when i < 5 then i else 9 end), i from generate_series(1, 100) n(i);
@@ -630,90 +703,69 @@ select * from t left join (select * from (select * from t order by a) v order by
 (2 rows)
 
 rollback;
--- Test EXPLAIN ANALYZE (text output) with both fullsort and presorted groups.
-explain (analyze, costs off, summary off, timing off)
-select * from (select * from t order by a) s order by a, b limit 70;
-                                   QUERY PLAN                                    
+-- Test EXPLAIN ANALYZE with both fullsort and presorted groups.
+select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 70');
+                         explain_analyze_without_memory                          
 ---------------------------------------------------------------------------------
  Limit (actual rows=70 loops=1)
    ->  Incremental Sort (actual rows=70 loops=1)
          Sort Key: t.a, t.b
          Presorted Key: t.a
-         Full-sort Groups: 1 (Methods: quicksort) Memory: 28kB (avg), 28kB (max)
-         Presorted Groups: 5 (Methods: quicksort) Memory: 25kB (avg), 25kB (max)
+         Full-sort Groups: 1 (Methods: quicksort) Memory: NNkB (avg), NNkB (max)
+         Presorted Groups: 5 (Methods: quicksort) Memory: NNkB (avg), NNkB (max)
          ->  Sort (actual rows=100 loops=1)
                Sort Key: t.a
-               Sort Method: quicksort  Memory: 30kB
+               Sort Method: quicksort  Memory: NNkB
                ->  Seq Scan on t (actual rows=100 loops=1)
 (10 rows)
 
-explain (analyze, costs off, summary off, timing off, format json)
-select * from (select * from t order by a) s order by a, b limit 70;
-                    QUERY PLAN                     
----------------------------------------------------
- [                                                +
-   {                                              +
-     "Plan": {                                    +
-       "Node Type": "Limit",                      +
-       "Parallel Aware": false,                   +
-       "Actual Rows": 70,                         +
-       "Actual Loops": 1,                         +
-       "Plans": [                                 +
-         {                                        +
-           "Node Type": "Incremental Sort",       +
-           "Parent Relationship": "Outer",        +
-           "Parallel Aware": false,               +
-           "Actual Rows": 70,                     +
-           "Actual Loops": 1,                     +
-           "Sort Key": ["t.a", "t.b"],            +
-           "Presorted Key": ["t.a"],              +
-           "Full-sort Groups": {                  +
-             "Group Count": 1,                    +
-             "Sort Methods Used": ["quicksort"],  +
-             "Average Sort Space Used": 28,       +
-             "Maximum Sort Space Used": 28,       +
-             "Sort Space Type": "Memory"          +
-           },                                     +
-           "Presorted Groups": {                  +
-             "Group Count": 5,                    +
-             "Sort Methods Used": ["quicksort"],  +
-             "Average Sort Space Used": 25,       +
-             "Maximum Sort Space Used": 25,       +
-             "Sort Space Type": "Memory"          +
-           },                                     +
-           "Plans": [                             +
-             {                                    +
-               "Node Type": "Sort",               +
-               "Parent Relationship": "Outer",    +
-               "Parallel Aware": false,           +
-               "Actual Rows": 100,                +
-               "Actual Loops": 1,                 +
-               "Sort Key": ["t.a"],               +
-               "Sort Method": "quicksort",        +
-               "Sort Space Used": 30,             +
-               "Sort Space Type": "Memory",       +
-               "Plans": [                         +
-                 {                                +
-                   "Node Type": "Seq Scan",       +
-                   "Parent Relationship": "Outer",+
-                   "Parallel Aware": false,       +
-                   "Relation Name": "t",          +
-                   "Alias": "t",                  +
-                   "Actual Rows": 100,            +
-                   "Actual Loops": 1              +
-                 }                                +
-               ]                                  +
-             }                                    +
-           ]                                      +
-         }                                        +
-       ]                                          +
-     },                                           +
-     "Triggers": [                                +
-     ]                                            +
-   }                                              +
+select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 70'));
+                   jsonb_pretty                   
+--------------------------------------------------
+ [                                               +
+     {                                           +
+         "Sort Key": [                           +
+             "t.a",                              +
+             "t.b"                               +
+         ],                                      +
+         "Node Type": "Incremental Sort",        +
+         "Actual Rows": 70,                      +
+         "Actual Loops": 1,                      +
+         "Presorted Key": [                      +
+             "t.a"                               +
+         ],                                      +
+         "Parallel Aware": false,                +
+         "Full-sort Groups": {                   +
+             "Group Count": 1,                   +
+             "Sort Methods Used": [              +
+                 "quicksort"                     +
+             ],                                  +
+             "Sort Space Memory": {              +
+                 "Average Sort Space Used": "NN",+
+                 "Maximum Sort Space Used": "NN" +
+             }                                   +
+         },                                      +
+         "Presorted Groups": {                   +
+             "Group Count": 5,                   +
+             "Sort Methods Used": [              +
+                 "quicksort"                     +
+             ],                                  +
+             "Sort Space Memory": {              +
+                 "Average Sort Space Used": "NN",+
+                 "Maximum Sort Space Used": "NN" +
+             }                                   +
+         },                                      +
+         "Parent Relationship": "Outer"          +
+     }                                           +
  ]
 (1 row)
 
+select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 70');
+ explain_analyze_inc_sort_nodes_verify_invariants 
+--------------------------------------------------
+ t
+(1 row)
+
 delete from t;
 -- Small groups of 10 tuples each tested around each mode transition point.
 insert into t(a, b) select i / 10, i from generate_series(1, 70) n(i);
diff --git a/src/test/regress/sql/incremental_sort.sql b/src/test/regress/sql/incremental_sort.sql
index 040324be32..b990b3b3de 100644
--- a/src/test/regress/sql/incremental_sort.sql
+++ b/src/test/regress/sql/incremental_sort.sql
@@ -18,6 +18,106 @@ reset work_mem;
 
 create table t(a integer, b integer);
 
+create or replace function explain_analyze_without_memory(query text)
+returns table (out_line text) language plpgsql
+as
+$$
+declare
+  line text;
+begin
+  for line in
+    execute 'explain (analyze, costs off, summary off, timing off) ' || query
+  loop
+    out_line := regexp_replace(line, '\d+kB', 'NNkB', 'g');
+    return next;
+  end loop;
+end;
+$$;
+
+create or replace function explain_analyze_inc_sort_nodes(query text)
+returns jsonb language plpgsql
+as
+$$
+declare
+  elements jsonb;
+  element jsonb;
+  matching_nodes jsonb := '[]'::jsonb;
+begin
+  execute 'explain (analyze, costs off, summary off, timing off, format ''json'') ' || query into strict elements;
+  while jsonb_array_length(elements) > 0 loop
+    element := elements->0;
+    elements := elements - 0;
+    case jsonb_typeof(element)
+    when 'array' then
+      if jsonb_array_length(element) > 0 then
+        elements := elements || element;
+      end if;
+    when 'object' then
+      if element ? 'Plan' then
+        elements := elements || jsonb_build_array(element->'Plan');
+        element := element - 'Plan';
+      else
+        if element ? 'Plans' then
+          elements := elements || jsonb_build_array(element->'Plans');
+          element := element - 'Plans';
+        end if;
+        if (element->>'Node Type')::text = 'Incremental Sort' then
+          matching_nodes := matching_nodes || element;
+        end if;
+      end if;
+    end case;
+  end loop;
+  return matching_nodes;
+end;
+$$;
+
+create or replace function explain_analyze_inc_sort_nodes_without_memory(query text)
+returns jsonb language plpgsql
+as
+$$
+declare
+  nodes jsonb := '[]'::jsonb;
+  node jsonb;
+  group_key text;
+  space_key text;
+begin
+  for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop
+    for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop
+      for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop
+        node := jsonb_set(node, array[group_key, space_key, 'Average Sort Space Used'], '"NN"', false);
+        node := jsonb_set(node, array[group_key, space_key, 'Maximum Sort Space Used'], '"NN"', false);
+      end loop;
+    end loop;
+    nodes := nodes || node;
+  end loop;
+  return nodes;
+end;
+$$;
+
+create or replace function explain_analyze_inc_sort_nodes_verify_invariants(query text)
+returns bool language plpgsql
+as
+$$
+declare
+  node jsonb;
+  group_stats jsonb;
+  group_key text;
+  space_key text;
+begin
+  for node in select * from jsonb_array_elements(explain_analyze_inc_sort_nodes(query)) t loop
+    for group_key in select unnest(array['Full-sort Groups', 'Presorted Groups']::text[]) t loop
+      group_stats := node->group_key;
+      for space_key in select unnest(array['Sort Space Memory', 'Sort Space Disk']::text[]) t loop
+        if (group_stats->space_key->'Maximum Sort Space Used')::bigint < (group_stats->space_key->'Maximum Sort Space Used')::bigint then
+          raise exception '% has invalid max space < average space', group_key;
+        end if;
+      end loop;
+    end loop;
+  end loop;
+  return true;
+end;
+$$;
+
 -- A single large group tested around each mode transition point.
 insert into t(a, b) select 1, i from generate_series(1, 100) n(i);
 explain (costs off) select * from (select * from t order by a) s order by a, b limit 31;
@@ -36,11 +136,10 @@ delete from t;
 insert into t(a, b) select (case when i < 50 then 1 else 2 end), i from generate_series(1, 100) n(i);
 explain (costs off) select * from (select * from t order by a) s order by a, b limit 55;
 select * from (select * from t order by a) s order by a, b limit 55;
--- Test EXPLAIN ANALYZE (text output) with only a fullsort group.
-explain (analyze, costs off, summary off, timing off)
-select * from (select * from t order by a) s order by a, b limit 55;
-explain (analyze, costs off, summary off, timing off, format json)
-select * from (select * from t order by a) s order by a, b limit 55;
+-- Test EXPLAIN ANALYZE with only a fullsort group.
+select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 55');
+select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 55'));
+select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 55');
 delete from t;
 
 -- An initial small group followed by a large group.
@@ -58,11 +157,10 @@ set local enable_sort = off;
 explain (costs off) select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2);
 select * from t left join (select * from (select * from t order by a) v order by a, b) s on s.a = t.a where t.a in (1, 2);
 rollback;
--- Test EXPLAIN ANALYZE (text output) with both fullsort and presorted groups.
-explain (analyze, costs off, summary off, timing off)
-select * from (select * from t order by a) s order by a, b limit 70;
-explain (analyze, costs off, summary off, timing off, format json)
-select * from (select * from t order by a) s order by a, b limit 70;
+-- Test EXPLAIN ANALYZE with both fullsort and presorted groups.
+select explain_analyze_without_memory('select * from (select * from t order by a) s order by a, b limit 70');
+select jsonb_pretty(explain_analyze_inc_sort_nodes_without_memory('select * from (select * from t order by a) s order by a, b limit 70'));
+select explain_analyze_inc_sort_nodes_verify_invariants('select * from (select * from t order by a) s order by a, b limit 70');
 delete from t;
 
 -- Small groups of 10 tuples each tested around each mode transition point.
-- 
2.17.1

