diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index f087ddb..863ab9d 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -2943,6 +2943,10 @@ remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel) /* * If subquery has regular DISTINCT (not DISTINCT ON), we're wasting our * time: all its output columns must be used in the distinctClause. + * (Note: the latter is not necessarily true anymore, because planner.c + * might have found some of the DISTINCT columns to be redundant and + * dropped them. But they'd still have sortgroupref markings, so unless + * we improve the heuristic below, we would not recognize them as unused.) */ if (subquery->distinctClause && !subquery->hasDistinctOn) return; @@ -2985,11 +2989,14 @@ remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel) /* * If it has a sortgroupref number, it's used in some sort/group - * clause so we'd better not remove it. Also, don't remove any - * resjunk columns, since their reason for being has nothing to do - * with anybody reading the subquery's output. (It's likely that - * resjunk columns in a sub-SELECT would always have ressortgroupref - * set, but even if they don't, it seems imprudent to remove them.) + * clause so we'd better not remove it. (This is a conservative + * heuristic, since it might not actually be used by any surviving + * sort/group clause; but we don't bother to expend the cycles needed + * for a more accurate test.) Also, don't remove any resjunk columns, + * since their reason for being has nothing to do with anybody reading + * the subquery's output. (It's likely that resjunk columns in a + * sub-SELECT would always have ressortgroupref set, but even if they + * don't, it seems imprudent to remove them.) */ if (tle->ressortgroupref || tle->resjunk) continue; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 6fde54f..650661f 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -122,7 +122,12 @@ static double preprocess_limit(PlannerInfo *root, double tuple_fraction, int64 *offset_est, int64 *count_est); static bool limit_needed(Query *parse); -static void remove_useless_groupby_columns(PlannerInfo *root); +static void remove_redundant_groupby_columns(PlannerInfo *root); +static void remove_redundant_distinct_columns(PlannerInfo *root); +static List *remove_functionally_dependent_clauses(PlannerInfo *root, + List *clauselist, + bool *clauses_were_removed, + bool *body_is_distinct); static List *preprocess_groupclause(PlannerInfo *root, List *force); static List *extract_rollup_sets(List *groupingSets); static List *reorder_grouping_sets(List *groupingSets, List *sortclause); @@ -825,7 +830,10 @@ subquery_planner(PlannerGlobal *glob, Query *parse, parse->havingQual = (Node *) newHaving; /* Remove any redundant GROUP BY columns */ - remove_useless_groupby_columns(root); + remove_redundant_groupby_columns(root); + + /* Likewise for redundant DISTINCT columns */ + remove_redundant_distinct_columns(root); /* * If we have any outer joins, try to reduce them to plain inner joins. @@ -2727,7 +2735,7 @@ limit_needed(Query *parse) /* - * remove_useless_groupby_columns + * remove_redundant_groupby_columns * Remove any columns in the GROUP BY clause that are redundant due to * being functionally dependent on other GROUP BY columns. * @@ -2739,35 +2747,103 @@ limit_needed(Query *parse) * parser's check_ungrouped_columns() and check_functional_grouping()). * * In principle, we could treat any NOT-NULL columns appearing in a UNIQUE - * index as the determining columns. But as with check_functional_grouping(), - * there's currently no way to represent dependency on a NOT NULL constraint, - * so we consider only the pkey for now. + * index as the determining columns. It is both impossible and unnecessary + * to install a dependency on a NOT NULL constraint, because the only way + * to change it is to do an ALTER TABLE, which would invalidate all plans + * that depend on the altered table anyway. TBD: add support for unique + * plus attnotnull. */ static void -remove_useless_groupby_columns(PlannerInfo *root) +remove_redundant_groupby_columns(PlannerInfo *root) { Query *parse = root->parse; - Bitmapset **groupbyattnos; - Bitmapset **surplusvars; - ListCell *lc; - int relid; - - /* No chance to do anything if there are less than two GROUP BY items */ - if (list_length(parse->groupClause) < 2) - return; + bool clauses_were_removed; + bool body_is_distinct; /* Don't fiddle with the GROUP BY clause if the query has grouping sets */ - if (parse->groupingSets) + if (NIL == parse->groupClause || parse->groupingSets) return; + parse->groupClause = remove_functionally_dependent_clauses(root, + parse->groupClause, + &clauses_were_removed, + &body_is_distinct); + + /* If the body is already distinct on groupClause and there are no + * aggregates then remove the GROUP BY. + */ + if (body_is_distinct && !parse->hasAggs) + parse->groupClause = NIL; +} + +/* + * remove_redundant_distinct_columns + * Similar to remove_redundant_groupby_columns but for the DISTINCT clause + */ +static void +remove_redundant_distinct_columns(PlannerInfo *root) +{ + Query *parse = root->parse; + bool clauses_were_removed; + bool body_is_distinct; + /* - * Scan the GROUP BY clause to find GROUP BY items that are simple Vars. - * Fill groupbyattnos[k] with a bitmapset of the column attnos of RTE k - * that are GROUP BY items. + * Don't bother trying to remove anything from a distinctOn clause. For + * this case, the distinctClauses are closely entwined with the ORDER BY + * clause, so we'd better not meddle with them. You might think that we + * can just apply the same optimizations to the ORDER BY too, but we can't + * since removing an item there could affect the order of the query + * results. */ - groupbyattnos = (Bitmapset **) palloc0(sizeof(Bitmapset *) * - (list_length(parse->rtable) + 1)); - foreach(lc, parse->groupClause) + if (NIL == parse->distinctClause || parse->hasDistinctOn) + return; + + parse->distinctClause = remove_functionally_dependent_clauses(root, + parse->distinctClause, + &clauses_were_removed, + &body_is_distinct); + root->hasModifiedDistinct = clauses_were_removed || body_is_distinct; + + /* If the body is already distinct on distinctClause then remove the DISTINCT + */ + if (body_is_distinct) + parse->distinctClause = NIL; +} + +/* + * remove_functionally_dependent_clauses + * Processes clauselist and removes any items which are deemed to be + * functionally dependent on other clauselist items. + * + * If any item from the list can be removed, then a new list is built which + * does not contain the removed items. If no item can be removed then the + * original list is returned. + */ +static List * +remove_functionally_dependent_clauses(PlannerInfo *root, + List *clauselist, + bool *clauses_were_removed, + bool *body_is_distinct) +{ + Query *parse = root->parse; + Bitmapset **clauseattnos; + Bitmapset **surplusvars; + ListCell *lc; + int relid; + int num_relids = 0; + bool all_simple_vars = true; + + *clauses_were_removed = false; + *body_is_distinct = false; + + /* + * Scan the clauselist to find items that are simple Vars. Fill + * clauseattnos[k] with a bitmapset of the column attnos of RTE k that are + * in the clauselist. + */ + clauseattnos = (Bitmapset **) palloc0(sizeof(Bitmapset *) * + (list_length(parse->rtable) + 1)); + foreach(lc, clauselist) { SortGroupClause *sgc = (SortGroupClause *) lfirst(lc); TargetEntry *tle = get_sortgroupclause_tle(sgc, parse->targetList); @@ -2777,26 +2853,34 @@ remove_useless_groupby_columns(PlannerInfo *root) * Ignore non-Vars and Vars from other query levels. * * XXX in principle, stable expressions containing Vars could also be - * removed, if all the Vars are functionally dependent on other GROUP - * BY items. But it's not clear that such cases occur often enough to - * be worth troubling over. + * removed, if all the Vars are functionally dependent on other items + * in the clauselist. But it's not clear that such cases occur often + * enough to be worth troubling over. */ if (!IsA(var, Var) || var->varlevelsup > 0) + { + all_simple_vars = false; continue; + } /* OK, remember we have this Var */ relid = var->varno; Assert(relid <= list_length(parse->rtable)); - groupbyattnos[relid] = bms_add_member(groupbyattnos[relid], - var->varattno - FirstLowInvalidHeapAttributeNumber); + + if (bms_is_empty(clauseattnos[relid])) + num_relids++; + + clauseattnos[relid] = bms_add_member(clauseattnos[relid], + var->varattno - FirstLowInvalidHeapAttributeNumber); } /* * Consider each relation and see if it is possible to remove some of its - * Vars from GROUP BY. For simplicity and speed, we do the actual removal - * in a separate pass. Here, we just fill surplusvars[k] with a bitmapset - * of the column attnos of RTE k that are removable GROUP BY items. + * Vars from the clauselist. For simplicity and speed, we do the actual + * removal in a separate pass. Here, we just fill surplusvars[k] with a + * bitmapset of the column attnos of RTE k that are removable clauselist + * items. */ surplusvars = NULL; /* don't allocate array unless required */ relid = 0; @@ -2813,8 +2897,9 @@ remove_useless_groupby_columns(PlannerInfo *root) if (rte->rtekind != RTE_RELATION) continue; - /* Nothing to do unless this rel has multiple Vars in GROUP BY */ - relattnos = groupbyattnos[relid]; + /* Nothing to do unless this rel has multiple Vars in clauselist */ + relattnos = clauseattnos[relid]; + if (bms_membership(relattnos) != BMS_MULTIPLE) continue; @@ -2824,11 +2909,30 @@ remove_useless_groupby_columns(PlannerInfo *root) */ pkattnos = get_primary_key_attnos(rte->relid, false, &constraintOid); if (pkattnos == NULL) - continue; + continue; // TBD: search for unique + not null + if (1 == num_relids && all_simple_vars && + list_length(parse->rtable) == 1 && + (bms_subset_compare(pkattnos, relattnos) == BMS_EQUAL || + bms_subset_compare(pkattnos, relattnos) == BMS_SUBSET1)) + { + /* If all clauses refer to the same relation and the pk is + * included in the clauses, then the output relation is + * distinct on the relattnos already. + */ + *body_is_distinct = true; + } + else if (list_length(clauselist) == 1) + { + /* If the body is not distinct and there's only one item in the + * clauselist that isn't the pk, then it can't be simplified. + */ + return clauselist; + } + /* * If the primary key is a proper subset of relattnos then we have - * some items in the GROUP BY that can be removed. + * some items in the clauselist that can be removed. */ if (bms_subset_compare(pkattnos, relattnos) == BMS_SUBSET1) { @@ -2839,26 +2943,25 @@ remove_useless_groupby_columns(PlannerInfo *root) if (surplusvars == NULL) surplusvars = (Bitmapset **) palloc0(sizeof(Bitmapset *) * (list_length(parse->rtable) + 1)); - - /* Remember the attnos of the removable columns */ + surplusvars[relid] = bms_difference(relattnos, pkattnos); - /* Also, mark the resulting plan as dependent on this constraint */ + /* Mark the resulting plan as dependent on this constraint */ parse->constraintDeps = lappend_oid(parse->constraintDeps, constraintOid); } } /* - * If we found any surplus Vars, build a new GROUP BY clause without them. + * If we found any surplus Vars, build a new clause list without them. * (Note: this may leave some TLEs with unreferenced ressortgroupref * markings, but that's harmless.) */ if (surplusvars != NULL) { - List *new_groupby = NIL; + List *new_clauselist = NIL; - foreach(lc, parse->groupClause) + foreach(lc, clauselist) { SortGroupClause *sgc = (SortGroupClause *) lfirst(lc); TargetEntry *tle = get_sortgroupclause_tle(sgc, parse->targetList); @@ -2872,11 +2975,15 @@ remove_useless_groupby_columns(PlannerInfo *root) var->varlevelsup > 0 || !bms_is_member(var->varattno - FirstLowInvalidHeapAttributeNumber, surplusvars[var->varno])) - new_groupby = lappend(new_groupby, sgc); + new_clauselist = lappend(new_clauselist, sgc); } - parse->groupClause = new_groupby; + *clauses_were_removed = true; + return new_clauselist; } + + /* nothing to change, just return the old list */ + return clauselist; } /* @@ -4743,10 +4850,10 @@ create_distinct_paths(PlannerInfo *root, * Unique node on those. Then consider doing an explicit sort of the * cheapest input path and Unique'ing that. * - * When we have DISTINCT ON, we must sort by the more rigorous of + * When we have DISTINCT ON, we must sort by the more restrictive of * DISTINCT and ORDER BY, else it won't have the desired behavior. * Also, if we do have to do an explicit sort, we might as well use - * the more rigorous ordering to avoid a second sort later. (Note + * the more restrictive ordering to avoid a second sort later. (Note * that the parser will have ensured that one clause is a prefix of * the other.) */ @@ -4773,13 +4880,14 @@ create_distinct_paths(PlannerInfo *root, } } - /* For explicit-sort case, always use the more rigorous clause */ + /* For explicit-sort case, always use the more restrictive clause */ if (list_length(root->distinct_pathkeys) < list_length(root->sort_pathkeys)) { needed_pathkeys = root->sort_pathkeys; /* Assert checks that parser didn't mess up... */ - Assert(pathkeys_contained_in(root->distinct_pathkeys, + Assert(root->hasModifiedDistinct || + pathkeys_contained_in(root->distinct_pathkeys, needed_pathkeys)); } else @@ -5654,7 +5762,7 @@ make_sort_input_target(PlannerInfo *root, * If the column has a sortgroupref, assume it has to be evaluated * before sorting. Generally such columns would be ORDER BY, GROUP * BY, etc targets. One exception is columns that were removed from - * GROUP BY by remove_useless_groupby_columns() ... but those would + * GROUP BY by remove_redundant_groupby_columns() ... but those would * only be Vars anyway. There don't seem to be any cases where it * would be worth the trouble to double-check. */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 5f2a4a7..eb00a0f 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -151,7 +151,8 @@ typedef struct Query List *windowClause; /* a list of WindowClause's */ - List *distinctClause; /* a list of SortGroupClause's */ + List *distinctClause; /* a list of SortGroupClause's. Not equivalent + * to non-junk targetList items. */ List *sortClause; /* a list of SortGroupClause's */ diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 85c5394..5c90e32 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -309,7 +309,8 @@ typedef struct PlannerInfo bool hasPseudoConstantQuals; /* true if any RestrictInfo has * pseudoconstant = true */ bool hasRecursion; /* true if planning a recursive WITH item */ - + bool hasModifiedDistinct; /* true if DISTINCT clause was modified */ + /* These fields are used only when hasRecursion is true: */ int wt_param_id; /* PARAM_EXEC ID for the work table */ struct Path *non_recursive_path; /* a path for non-recursive term */ diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index ce6b841..1cd6838 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -962,12 +962,10 @@ create temp table t2 (x int, y int, z int, primary key (x, y)); create temp table t3 (a int, b int, c int, primary key(a, b) deferrable); -- Non-primary-key columns can be removed from GROUP BY explain (costs off) select * from t1 group by a,b,c,d; - QUERY PLAN ----------------------- - HashAggregate - Group Key: a, b - -> Seq Scan on t1 -(3 rows) + QUERY PLAN +---------------- + Seq Scan on t1 +(1 row) -- No removal can happen if the complete PK is not present in GROUP BY explain (costs off) select a,c from t1 group by a,c,d; @@ -1017,6 +1015,90 @@ explain (costs off) select * from t3 group by a,b,c; -> Seq Scan on t3 (3 rows) +-- +-- Test removal of redundant DISTINCT columns +-- +-- Non-primary-key columns can be removed from DISTINCT clause +explain (costs off) select distinct a,b,c,d from t1; + QUERY PLAN +---------------- + Seq Scan on t1 +(1 row) + +-- No removal can happen if the complete PK is not present in DISTINCT clause +explain (costs off) select distinct a,c,d from t1; + QUERY PLAN +---------------------- + HashAggregate + Group Key: a, c, d + -> Seq Scan on t1 +(3 rows) + +-- Test removal across multiple relations +explain (costs off) select distinct t1.a,t1.b,t1.c,t1.d,t2.x,t2.y,t2.z +from t1 inner join t2 on t1.a = t2.x and t1.b = t2.y; + QUERY PLAN +------------------------------------------------------ + HashAggregate + Group Key: t1.a, t1.b, t2.x, t2.y + -> Hash Join + Hash Cond: ((t2.x = t1.a) AND (t2.y = t1.b)) + -> Seq Scan on t2 + -> Hash + -> Seq Scan on t1 +(7 rows) + +-- Test case where t1 can be optimized but not t2 +explain (costs off) select distinct t1.a,t1.b,t1.c,t1.d,t2.x,t2.z +from t1 inner join t2 on t1.a = t2.x and t1.b = t2.y; + QUERY PLAN +------------------------------------------------------ + HashAggregate + Group Key: t1.a, t1.b, t2.x, t2.z + -> Hash Join + Hash Cond: ((t2.x = t1.a) AND (t2.y = t1.b)) + -> Seq Scan on t2 + -> Hash + -> Seq Scan on t1 +(7 rows) + +-- Ensure we don't remove DISTINCT ON items +explain (costs off) select distinct on (a,b,c) d from t1 order by a,b,c,d; + QUERY PLAN +------------------------------ + Unique + -> Sort + Sort Key: a, b, c, d + -> Seq Scan on t1 +(4 rows) + +-- Sorted aggregation potentially interacts with redundant column removal. +-- Ensure that the distinct clause can be simplified when the order by +-- clause contains non-pk columns before the last pk column. +-- +explain verbose select distinct * from t1 order by a, c, b; + QUERY PLAN +----------------------------------------------------------------------- + Sort (cost=128.89..133.52 rows=1850 width=16) + Output: a, b, c, d + Sort Key: t1.a, t1.c, t1.b + -> Seq Scan on pg_temp_3.t1 (cost=0.00..28.50 rows=1850 width=16) + Output: a, b, c, d +(5 rows) + +-- Verify that removing unnecessary GROUP BY columns doesn't cause +-- a vector aggregate to become a scalar aggregate +-- +explain verbose select a, b, COUNT(*) from t1 group by a, b; + QUERY PLAN +---------------------------------------------------------------------- + HashAggregate (cost=42.38..44.38 rows=200 width=16) + Output: a, b, count(*) + Group Key: t1.a, t1.b + -> Seq Scan on pg_temp_3.t1 (cost=0.00..28.50 rows=1850 width=8) + Output: a, b, c, d +(5 rows) + drop table t1; drop table t2; drop table t3; diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index 9f4c88d..e68a549 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -4012,33 +4012,34 @@ select d.* from d left join (select distinct * from b) s explain (costs off) select d.* from d left join (select * from b group by b.id, b.c_id) s on d.a = s.id; - QUERY PLAN ------------------------------------------- - Merge Right Join - Merge Cond: (b.id = d.a) - -> Group - Group Key: b.id - -> Index Scan using b_pkey on b + QUERY PLAN +--------------------------------- + Merge Left Join + Merge Cond: (d.a = s.id) -> Sort Sort Key: d.a -> Seq Scan on d -(8 rows) + -> Sort + Sort Key: s.id + -> Subquery Scan on s + -> Seq Scan on b +(9 rows) -- similarly, but keying off a DISTINCT clause explain (costs off) select d.* from d left join (select distinct * from b) s on d.a = s.id; - QUERY PLAN --------------------------------------- - Merge Right Join - Merge Cond: (b.id = d.a) - -> Unique - -> Sort - Sort Key: b.id, b.c_id - -> Seq Scan on b + QUERY PLAN +--------------------------------- + Merge Left Join + Merge Cond: (d.a = s.id) -> Sort Sort Key: d.a -> Seq Scan on d + -> Sort + Sort Key: s.id + -> Subquery Scan on s + -> Seq Scan on b (9 rows) -- check join removal works when uniqueness of the join condition is enforced diff --git a/src/test/regress/sql/aggregates.sql b/src/test/regress/sql/aggregates.sql index 2eeb3ee..2fb9a10 100644 --- a/src/test/regress/sql/aggregates.sql +++ b/src/test/regress/sql/aggregates.sql @@ -362,6 +362,37 @@ group by t1.a,t1.b,t1.c,t1.d,t2.x,t2.z; -- Cannot optimize when PK is deferrable explain (costs off) select * from t3 group by a,b,c; +-- +-- Test removal of redundant DISTINCT columns +-- +-- Non-primary-key columns can be removed from DISTINCT clause +explain (costs off) select distinct a,b,c,d from t1; + +-- No removal can happen if the complete PK is not present in DISTINCT clause +explain (costs off) select distinct a,c,d from t1; + +-- Test removal across multiple relations +explain (costs off) select distinct t1.a,t1.b,t1.c,t1.d,t2.x,t2.y,t2.z +from t1 inner join t2 on t1.a = t2.x and t1.b = t2.y; + +-- Test case where t1 can be optimized but not t2 +explain (costs off) select distinct t1.a,t1.b,t1.c,t1.d,t2.x,t2.z +from t1 inner join t2 on t1.a = t2.x and t1.b = t2.y; + +-- Ensure we don't remove DISTINCT ON items +explain (costs off) select distinct on (a,b,c) d from t1 order by a,b,c,d; + +-- Sorted aggregation potentially interacts with redundant column removal. +-- Ensure that the distinct clause can be simplified when the order by +-- clause contains non-pk columns before the last pk column. +-- +explain verbose select distinct * from t1 order by a, c, b; + +-- Verify that removing unnecessary GROUP BY columns doesn't cause +-- a vector aggregate to become a scalar aggregate +-- +explain verbose select a, b, COUNT(*) from t1 group by a, b; + drop table t1; drop table t2; drop table t3;