From 011fb31ee82db6f893e140d77920bace81837ac4 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 19 Mar 2019 14:30:21 -0400 Subject: [PATCH] x --- doc/src/sgml/monitoring.sgml | 190 ++++++++++++++++++++++++++- src/backend/catalog/index.c | 9 ++ src/backend/catalog/system_views.sql | 26 ++++ src/backend/commands/cluster.c | 80 +++++++++++ src/backend/utils/adt/pgstatfuncs.c | 2 + src/include/commands/progress.h | 22 ++++ src/include/pgstat.h | 3 +- src/test/regress/expected/rules.out | 27 ++++ 8 files changed, 351 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index ac2721c8ad..4479b64efc 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -340,7 +340,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser pg_stat_progress_vacuumpg_stat_progress_vacuum One row for each backend (including autovacuum worker processes) running VACUUM, showing current progress. - See . + See . + + + + + pg_stat_progress_clusterpg_stat_progress_cluster + One row for each backend running + CLUSTER or VACUUM FULL, showing current progress. + See . @@ -3394,9 +3402,9 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, PostgreSQL has the ability to report the progress of - certain commands during command execution. Currently, the only command - which supports progress reporting is VACUUM. This may be - expanded in the future. + certain commands during command execution. Currently, the only commands + which support progress reporting are VACUUM and + CLUSTER. This may be expanded in the future. @@ -3408,9 +3416,11 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, one row for each backend (including autovacuum worker processes) that is currently vacuuming. The tables below describe the information that will be reported and provide information about how to interpret it. - Progress reporting is not currently supported for VACUUM FULL - and backends running VACUUM FULL will not be listed in this - view. + Progress for VACUUM FULL commands is reported via + pg_stat_progress_cluster + because both VACUUM FULL and CLUSTER + rewrite the table, while regular VACUUM only modifies it + in place. See . @@ -3587,6 +3597,172 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
+
+ + + CLUSTER Progress Reporting + + + Whenever CLUSTER or VACUUM FULL is + running, the pg_stat_progress_cluster view will + contain a row for each backend that is currently running either command. + The tables below describe the information that will be reported and + provide information about how to interpret it. + + + + <structname>pg_stat_progress_cluster</structname> View + + + + Column + Type + Description + + + + + + pid + integer + Process ID of backend. + + + datid + oid + OID of the database to which this backend is connected. + + + datname + name + Name of the database to which this backend is connected. + + + relid + oid + OID of the table being clustered. + + + command + text + + The command that is running. Either CLUSTER or VACUUM FULL. + + + + phase + text + + Current processing phase. See . + + + + cluster_index_relid + bigint + + If the table is being scanned using an index, this is the OID of the + index being used; otherwise, it is zero. + + + + heap_tuples_scanned + bigint + + Number of heap tuples scanned. + This counter only advances when the phase is + seq scanning heap, + index scanning heap + or writing new heap. + + + + heap_blks_total + bigint + + Total number of heap blocks in the table. This number is reported + as of the beginning of seq scanning heap. + + + + heap_blks_scanned + bigint + + Number of heap blocks scanned. This counter only advances when the + phase is seq scanning heap. + + + + index_rebuild_count + bigint + + Number of indexes rebuilt. This counter only advances when the phase + is rebuilding index. + + + + +
+ + + CLUSTER and VACUUM FULL phases + + + + Phase + Description + + + + + + initializing + + The command is preparing to begin scanning the heap. This phase is + expected to be very brief. + + + + seq scanning heap + + The command is currently scanning the table using a sequential scan. + + + + index scanning heap + + The command is currently scanning the table using an index scan. + + + + sorting tuples + + CLUSTER is currently sorting tuples. + + + + swapping relation files + + The command is currently swapping newly-built files into place. + + + + rebuilding index + + The command is currently rebuilding an index. + + + + performing final cleanup + + The command is performing final cleanup. When this phase is + completed, CLUSTER + or VACUUM FULL will end. + + + + +
+
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index c339a2bb77..8a634dd57e 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -52,6 +52,7 @@ #include "catalog/storage.h" #include "commands/tablecmds.h" #include "commands/event_trigger.h" +#include "commands/progress.h" #include "commands/trigger.h" #include "executor/executor.h" #include "miscadmin.h" @@ -59,6 +60,7 @@ #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" #include "parser/parser.h" +#include "pgstat.h" #include "rewrite/rewriteManip.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" @@ -3851,6 +3853,7 @@ reindex_relation(Oid relid, int flags, int options) List *indexIds; bool is_pg_class; bool result; + int i; /* * Open and lock the relation. ShareLock is sufficient since we only need @@ -3938,6 +3941,7 @@ reindex_relation(Oid relid, int flags, int options) /* Reindex all the indexes. */ doneIndexes = NIL; + i = 1; foreach(indexId, indexIds) { Oid indexOid = lfirst_oid(indexId); @@ -3955,6 +3959,11 @@ reindex_relation(Oid relid, int flags, int options) if (is_pg_class) doneIndexes = lappend_oid(doneIndexes, indexOid); + + /* Set index rebuild count */ + pgstat_progress_update_param(PROGRESS_CLUSTER_INDEX_REBUILD_COUNT, + i); + i++; } } PG_CATCH(); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index d962648bc5..87c0092787 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -907,6 +907,32 @@ CREATE VIEW pg_stat_progress_vacuum AS FROM pg_stat_get_progress_info('VACUUM') AS S LEFT JOIN pg_database D ON S.datid = D.oid; +CREATE VIEW pg_stat_progress_cluster AS + SELECT + S.pid AS pid, + S.datid AS datid, + D.datname AS datname, + S.relid AS relid, + CASE S.param1 WHEN 1 THEN 'CLUSTER' + WHEN 2 THEN 'VACUUM FULL' + END AS command, + CASE S.param2 WHEN 0 THEN 'initializing' + WHEN 1 THEN 'seq scanning heap' + WHEN 2 THEN 'index scanning heap' + WHEN 3 THEN 'sorting tuples' + WHEN 4 THEN 'writing new heap' + WHEN 5 THEN 'swapping relation files' + WHEN 6 THEN 'rebuilding index' + WHEN 7 THEN 'performing final cleanup' + END AS phase, + S.param3 AS cluster_index_relid, + S.param4 AS heap_tuples_scanned, + S.param5 AS heap_blks_total, + S.param6 AS heap_blks_scanned, + S.param7 AS index_rebuild_count + FROM pg_stat_get_progress_info('CLUSTER') AS S + LEFT JOIN pg_database D ON S.datid = D.oid; + CREATE VIEW pg_user_mappings AS SELECT U.oid AS umid, diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 3e2a807640..2e4fbd0663 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -36,10 +36,12 @@ #include "catalog/objectaccess.h" #include "catalog/toasting.h" #include "commands/cluster.h" +#include "commands/progress.h" #include "commands/tablecmds.h" #include "commands/vacuum.h" #include "miscadmin.h" #include "optimizer/optimizer.h" +#include "pgstat.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "storage/predicate.h" @@ -276,6 +278,14 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) /* Check for user-requested abort. */ CHECK_FOR_INTERRUPTS(); + pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid); + if (OidIsValid(indexOid)) + pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, + PROGRESS_CLUSTER_COMMAND_CLUSTER); + else + pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, + PROGRESS_CLUSTER_COMMAND_VACUUM_FULL); + /* * We grab exclusive access to the target rel and index for the duration * of the transaction. (This is redundant for the single-transaction @@ -286,7 +296,10 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) /* If the table has gone away, we can skip processing it */ if (!OldHeap) + { + pgstat_progress_end_command(); return; + } /* * Since we may open a new transaction for each relation, we have to check @@ -305,6 +318,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) if (!pg_class_ownercheck(tableOid, GetUserId())) { relation_close(OldHeap, AccessExclusiveLock); + pgstat_progress_end_command(); return; } @@ -319,6 +333,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) if (RELATION_IS_OTHER_TEMP(OldHeap)) { relation_close(OldHeap, AccessExclusiveLock); + pgstat_progress_end_command(); return; } @@ -330,6 +345,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid))) { relation_close(OldHeap, AccessExclusiveLock); + pgstat_progress_end_command(); return; } @@ -340,6 +356,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) if (!HeapTupleIsValid(tuple)) /* probably can't happen */ { relation_close(OldHeap, AccessExclusiveLock); + pgstat_progress_end_command(); return; } indexForm = (Form_pg_index) GETSTRUCT(tuple); @@ -347,6 +364,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) { ReleaseSysCache(tuple); relation_close(OldHeap, AccessExclusiveLock); + pgstat_progress_end_command(); return; } ReleaseSysCache(tuple); @@ -401,6 +419,7 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) !RelationIsPopulated(OldHeap)) { relation_close(OldHeap, AccessExclusiveLock); + pgstat_progress_end_command(); return; } @@ -416,6 +435,8 @@ cluster_rel(Oid tableOid, Oid indexOid, int options) rebuild_relation(OldHeap, indexOid, verbose); /* NB: rebuild_relation does table_close() on OldHeap */ + + pgstat_progress_end_command(); } /* @@ -928,6 +949,17 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, */ if (OldIndex != NULL && !use_sort) { + const int ci_index[] = { + PROGRESS_CLUSTER_PHASE, + PROGRESS_CLUSTER_INDEX_RELID + }; + int64 ci_val[2]; + + /* Set phase and OIDOldIndex to columns */ + ci_val[0] = PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP; + ci_val[1] = OIDOldIndex; + pgstat_progress_update_multi_param(2, ci_index, ci_val); + tableScan = NULL; heapScan = NULL; indexScan = index_beginscan(OldHeap, OldIndex, SnapshotAny, 0, 0); @@ -935,9 +967,17 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, } else { + /* Set phase */ + pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, + PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP); + tableScan = table_beginscan(OldHeap, SnapshotAny, 0, (ScanKey) NULL); heapScan = (HeapScanDesc) tableScan; indexScan = NULL; + + /* Set total heap blocks */ + pgstat_progress_update_param(PROGRESS_CLUSTER_TOTAL_HEAP_BLKS, + heapScan->rs_nblocks); } slot = table_slot_create(OldHeap, NULL); @@ -994,6 +1034,10 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, break; buf = heapScan->rs_cbuf; + + /* Set heap blocks scanned */ + pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_BLKS_SCANNED, + heapScan->rs_cblock); } LockBuffer(buf, BUFFER_LOCK_SHARE); @@ -1070,6 +1114,10 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, oldTupDesc, newTupDesc, values, isnull, rwstate); + + /* Report increase in number of tuples scanned */ + pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, + num_tuples); } if (indexScan != NULL) @@ -1085,8 +1133,24 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, */ if (tuplesort != NULL) { + double num_tuples = 0; + const int cp_index[] = { + PROGRESS_CLUSTER_PHASE, + PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, + }; + int64 cp_val[2]; + + /* Report that we are now sorting tuples */ + cp_val[0] = PROGRESS_CLUSTER_PHASE_SORT_TUPLES; + cp_val[1] = num_tuples; + pgstat_progress_update_multi_param(2, cp_index, cp_val); + tuplesort_performsort(tuplesort); + /* Report that we are now writing new heap */ + pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, + PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP); + for (;;) { HeapTuple tuple; @@ -1097,10 +1161,14 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, if (tuple == NULL) break; + num_tuples += 1; reform_and_rewrite_tuple(tuple, oldTupDesc, newTupDesc, values, isnull, rwstate); + /* Report num_tuples */ + pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, + num_tuples); } tuplesort_end(tuplesort); @@ -1539,6 +1607,10 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, int reindex_flags; int i; + /* Report that we are now swapping relation files */ + pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, + PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES); + /* Zero out possible results from swapped_relation_files */ memset(mapped_tables, 0, sizeof(mapped_tables)); @@ -1586,8 +1658,16 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, else if (newrelpersistence == RELPERSISTENCE_PERMANENT) reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; + /* Report that we are now reindexing relations */ + pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, + PROGRESS_CLUSTER_PHASE_REBUILD_INDEX); + reindex_relation(OIDOldHeap, reindex_flags, 0); + /* Report that we are now doing clean up */ + pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, + PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP); + /* * If the relation being rebuild is pg_class, swap_relation_files() * couldn't update pg_class's own pg_class entry (check comments in diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index da1d685c08..90a817a25c 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -468,6 +468,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) /* Translate command name into command type code. */ if (pg_strcasecmp(cmd, "VACUUM") == 0) cmdtype = PROGRESS_COMMAND_VACUUM; + else if (pg_strcasecmp(cmd, "CLUSTER") == 0) + cmdtype = PROGRESS_COMMAND_CLUSTER; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index 9858b36a38..0f637fe4e7 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -34,4 +34,26 @@ #define PROGRESS_VACUUM_PHASE_TRUNCATE 5 #define PROGRESS_VACUUM_PHASE_FINAL_CLEANUP 6 +/* Progress parameters for cluster */ +#define PROGRESS_CLUSTER_COMMAND 0 +#define PROGRESS_CLUSTER_PHASE 1 +#define PROGRESS_CLUSTER_INDEX_RELID 2 +#define PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED 3 +#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 4 +#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 5 +#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 6 + +/* Phases of cluster (as dvertised via PROGRESS_CLUSTER_PHASE) */ +#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1 +#define PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP 2 +#define PROGRESS_CLUSTER_PHASE_SORT_TUPLES 3 +#define PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP 4 +#define PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES 5 +#define PROGRESS_CLUSTER_PHASE_REBUILD_INDEX 6 +#define PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP 7 + +/* Commands of PROGRESS_CLUSTER */ +#define PROGRESS_CLUSTER_COMMAND_CLUSTER 1 +#define PROGRESS_CLUSTER_COMMAND_VACUUM_FULL 2 + #endif diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ea6cc8b560..c080fa6388 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -950,7 +950,8 @@ typedef enum typedef enum ProgressCommandType { PROGRESS_COMMAND_INVALID, - PROGRESS_COMMAND_VACUUM + PROGRESS_COMMAND_VACUUM, + PROGRESS_COMMAND_CLUSTER } ProgressCommandType; #define PGSTAT_NUM_PROGRESS_PARAM 10 diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index f104dc4a62..05cc9db019 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1830,6 +1830,33 @@ pg_stat_database_conflicts| SELECT d.oid AS datid, pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin, pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock FROM pg_database d; +pg_stat_progress_cluster| SELECT s.pid, + s.datid, + d.datname, + s.relid, + CASE s.param1 + WHEN 1 THEN 'CLUSTER'::text + WHEN 2 THEN 'VACUUM FULL'::text + ELSE NULL::text + END AS command, + CASE s.param2 + WHEN 0 THEN 'initializing'::text + WHEN 1 THEN 'seq scanning heap'::text + WHEN 2 THEN 'index scanning heap'::text + WHEN 3 THEN 'sorting tuples'::text + WHEN 4 THEN 'writing new heap'::text + WHEN 5 THEN 'swapping relation files'::text + WHEN 6 THEN 'rebuilding index'::text + WHEN 7 THEN 'performing final cleanup'::text + ELSE NULL::text + END AS phase, + s.param3 AS cluster_index_relid, + s.param4 AS heap_tuples_scanned, + s.param5 AS heap_blks_total, + s.param6 AS heap_blks_scanned, + s.param7 AS index_rebuild_count + FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10) + LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_vacuum| SELECT s.pid, s.datid, d.datname, -- 2.17.2 (Apple Git-113)