From af124a675637c44781ff84a979e6d9d0afb1e8d4 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horiguchi.kyotaro@lab.ntt.co.jp>
Date: Tue, 21 Nov 2017 10:47:52 +0900
Subject: [PATCH 4/4] Add truncation information to pg_stat_all_tables

This patch adds truncated and tried-but-not-truncated pages in the
last vacuum. This is intended to use to find uncertain failure of
truncation or unwanted aggressive trancation.
---
 doc/src/sgml/monitoring.sgml         | 10 ++++++++++
 src/backend/catalog/system_views.sql |  2 ++
 src/backend/commands/vacuum.c        | 14 ++++++++------
 src/backend/commands/vacuumlazy.c    | 13 ++++++++++++-
 src/backend/postmaster/pgstat.c      | 10 +++++++++-
 src/backend/utils/adt/pgstatfuncs.c  | 32 +++++++++++++++++++++++++++++++-
 src/include/catalog/pg_proc.h        |  4 ++++
 src/include/pgstat.h                 |  6 ++++++
 src/test/regress/expected/rules.out  |  6 ++++++
 9 files changed, 88 insertions(+), 9 deletions(-)

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index a0288cb..fd0507a 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -2585,6 +2585,16 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
      <entry>Oldest xmin used by the last vacuum on this table</entry>
     </row>
     <row>
+     <entry><structfield>last_vacuum_truncated</structfield></entry>
+     <entry><type>bigint</type></entry>
+     <entry>Number actually truncated pages during the last vacuum on this table</entry>
+    </row>
+    <row>
+     <entry><structfield>last_vacuum_untruncated</structfield></entry>
+     <entry><type>bigint</type></entry>
+     <entry>Number tried but actually not truncated pages during the last vacuum on this table</entry>
+    </row>
+    <row>
      <entry><structfield>last_vacuum_status</structfield></entry>
      <entry><type>text</type></entry>
      <entry>Status of the last vacuum or autovacuum.</entry>
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index c69fea9..528e9c5 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -529,6 +529,8 @@ CREATE VIEW pg_stat_all_tables AS
             pg_stat_get_last_analyze_time(C.oid) as last_analyze,
             pg_stat_get_last_autoanalyze_time(C.oid) as last_autoanalyze,
             pg_stat_get_last_vacuum_index_scans(C.oid) AS last_vacuum_index_scans,
+            pg_stat_get_last_vacuum_truncated(C.oid) AS last_vacuum_truncated,
+            pg_stat_get_last_vacuum_untruncated(C.oid) AS last_vacuum_untruncated,
             pg_stat_get_last_vacuum_oldest_xmin(C.oid) AS last_vacuum_oldest_xmin,
             pg_stat_get_last_vacuum_status(C.oid) AS last_vacuum_status,
             pg_stat_get_last_vacuum_status_detail(C.oid) AS last_vacuum_status_detail,
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index cf0bca7..cf754f9 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -1467,7 +1467,8 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params)
 
 	if (!onerel)
 	{
-		pgstat_report_vacuum(relid, false, 0, 0, 0, InvalidTransactionId,
+		pgstat_report_vacuum(relid, false,
+							 0, 0, 0, 0, 0,InvalidTransactionId,
 							 PGSTAT_VACUUM_SKIPPED, 0,
 							 PGSTAT_VACUUM_LOCK_FAILED);
 		PopActiveSnapshot();
@@ -1504,7 +1505,7 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params)
 
 		pgstat_report_vacuum(RelationGetRelid(onerel),
 							 onerel->rd_rel->relisshared,
-							 0, 0, 0, InvalidTransactionId,
+							 0, 0, 0, 0, 0, InvalidTransactionId,
 							 PGSTAT_VACUUM_FINISHED, 0,
 							 PGSTAT_VACUUM_NONTARGET);
 
@@ -1528,7 +1529,7 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params)
 
 		pgstat_report_vacuum(RelationGetRelid(onerel),
 							 onerel->rd_rel->relisshared,
-							 0, 0, 0, InvalidTransactionId,
+							 0, 0, 0, 0, 0, InvalidTransactionId,
 							 PGSTAT_VACUUM_FINISHED, 0,
 							 PGSTAT_VACUUM_NONTARGET);
 
@@ -1550,7 +1551,7 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params)
 
 		pgstat_report_vacuum(RelationGetRelid(onerel),
 							 onerel->rd_rel->relisshared,
-							 0, 0, 0, InvalidTransactionId,
+							 0, 0, 0, 0, 0, InvalidTransactionId,
 							 PGSTAT_VACUUM_FINISHED, 0,
 							 PGSTAT_VACUUM_NONTARGET);
 
@@ -1570,7 +1571,7 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params)
 
 		pgstat_report_vacuum(RelationGetRelid(onerel),
 							 onerel->rd_rel->relisshared,
-							 0, 0, 0, InvalidTransactionId,
+							 0, 0, 0, 0, 0, InvalidTransactionId,
 							 PGSTAT_VACUUM_FINISHED, 0,
 							 PGSTAT_VACUUM_NONTARGET);
 
@@ -1628,7 +1629,8 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params)
 		/* VACUUM FULL is now a variant of CLUSTER; see cluster.c */
 		cluster_rel(relid, InvalidOid, false,
 					(options & VACOPT_VERBOSE) != 0);
-		pgstat_report_vacuum(relid, isshared, 0, 0, 0, InvalidTransactionId,
+		pgstat_report_vacuum(relid, isshared, 0, 0, 0, 0, 0,
+							 InvalidTransactionId,
 							 PGSTAT_VACUUM_FINISHED, 0,
 							 PGSTAT_VACUUM_FULL);
 	}
diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c
index c53b4fa..53821f3 100644
--- a/src/backend/commands/vacuumlazy.c
+++ b/src/backend/commands/vacuumlazy.c
@@ -121,6 +121,7 @@ typedef struct LVRelStats
 	double		new_rel_tuples; /* new estimated total # of tuples */
 	double		new_dead_tuples;	/* new estimated total # of dead tuples */
 	BlockNumber pages_removed;
+	BlockNumber pages_not_removed;
 	double		tuples_deleted;
 	BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */
 	/* List of TIDs of tuples we intend to delete */
@@ -248,6 +249,7 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
 	vacrelstats->old_rel_tuples = onerel->rd_rel->reltuples;
 	vacrelstats->num_index_scans = 0;
 	vacrelstats->pages_removed = 0;
+	vacrelstats->pages_not_removed = 0;
 	vacrelstats->lock_waiter_detected = false;
 	vacrelstats->aggressive = aggressive;
 
@@ -290,8 +292,13 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
 	{
 		lazy_truncate_heap(onerel, vacrelstats);
 
+		/* just paranoia */
+		if (vacrelstats->rel_pages >= vacrelstats->nonempty_pages)
+			vacrelstats->pages_not_removed +=
+				vacrelstats->rel_pages - vacrelstats->nonempty_pages;
+
 		/* check if all empty pages are truncated */
-		if (vacrelstats->rel_pages > vacrelstats->nonempty_pages)
+		if (vacrelstats->pages_not_removed > 0)
 			vacuum_status_details |= PGSTAT_VACUUM_PARTIALLY_TRUNCATED;
 	}
 
@@ -363,6 +370,8 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
 						 onerel->rd_rel->relisshared,
 						 new_live_tuples,
 						 vacrelstats->new_dead_tuples,
+						 vacrelstats->pages_removed,
+						 vacrelstats->pages_not_removed,
 						 vacrelstats->num_index_scans,
 						 OldestXmin,
 						 PGSTAT_VACUUM_FINISHED, 0,
@@ -2283,6 +2292,8 @@ lazy_vacuum_cancel_handler(void)
 						 stats->shared,
 						 stats->new_rel_tuples,
 						 stats->new_dead_tuples,
+						 stats->pages_removed,
+						 stats->pages_not_removed,
 						 stats->num_index_scans,
 						 OldestXmin,
 						 status, phase, details);
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 3e1d051..a4a6169 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -1404,8 +1404,10 @@ pgstat_report_autovac(Oid dboid)
 void
 pgstat_report_vacuum(Oid tableoid, bool shared,
 					 PgStat_Counter livetuples, PgStat_Counter deadtuples,
+					 PgStat_Counter pages_removed,
+					 PgStat_Counter pages_not_removed,
 					 PgStat_Counter num_index_scans,
-					 TransactionId oldestxmin,
+					 TransactionId	oldestxmin,
 					 PgStat_Counter status, PgStat_Counter last_phase,
 					 PgStat_Counter details)
 {
@@ -1421,6 +1423,8 @@ pgstat_report_vacuum(Oid tableoid, bool shared,
 	msg.m_vacuumtime = GetCurrentTimestamp();
 	msg.m_live_tuples = livetuples;
 	msg.m_dead_tuples = deadtuples;
+	msg.m_pages_removed = pages_removed;
+	msg.m_pages_not_removed = pages_not_removed;
 	msg.m_num_index_scans = num_index_scans;
 	msg.m_oldest_xmin = oldestxmin;
 	msg.m_vacuum_status = status;
@@ -4594,6 +4598,8 @@ pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, Oid tableoid, bool create)
 		result->n_live_tuples = 0;
 		result->n_dead_tuples = 0;
 		result->changes_since_analyze = 0;
+		result->n_pages_removed = 0;
+		result->n_pages_not_removed = 0;
 		result->n_index_scans = 0;
 		result->oldest_xmin = InvalidTransactionId;
 		result->blocks_fetched = 0;
@@ -6009,6 +6015,8 @@ pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len)
 	case PGSTAT_VACUUM_FINISHED:
 		tabentry->n_live_tuples = msg->m_live_tuples;
 		tabentry->n_dead_tuples = msg->m_dead_tuples;
+		tabentry->n_pages_removed = msg->m_pages_removed;
+		tabentry->n_pages_not_removed = msg->m_pages_not_removed;
 		tabentry->n_index_scans = msg->m_num_index_scans;
 		tabentry->oldest_xmin = msg->m_oldest_xmin;
 		tabentry->vacuum_failcount = 0;
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 0fba265..b32bdf5 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -204,6 +204,36 @@ pg_stat_get_vacuum_necessity(PG_FUNCTION_ARGS)
 }
 
 Datum
+pg_stat_get_last_vacuum_truncated(PG_FUNCTION_ARGS)
+{
+	Oid			relid = PG_GETARG_OID(0);
+	int64		result;
+	PgStat_StatTabEntry *tabentry;
+
+	if ((tabentry = pgstat_fetch_stat_tabentry(relid)) == NULL)
+		result = 0;
+	else
+		result = (int64) (tabentry->n_pages_removed);
+
+	PG_RETURN_INT64(result);
+}
+
+Datum
+pg_stat_get_last_vacuum_untruncated(PG_FUNCTION_ARGS)
+{
+	Oid			relid = PG_GETARG_OID(0);
+	int64		result;
+	PgStat_StatTabEntry *tabentry;
+
+	if ((tabentry = pgstat_fetch_stat_tabentry(relid)) == NULL)
+		result = 0;
+	else
+		result = (int64) (tabentry->n_pages_not_removed);
+
+	PG_RETURN_INT64(result);
+}
+
+Datum
 pg_stat_get_last_vacuum_index_scans(PG_FUNCTION_ARGS)
 {
 	Oid			relid = PG_GETARG_OID(0);
@@ -349,7 +379,7 @@ pg_stat_get_last_vacuum_status_detail(PG_FUNCTION_ARGS)
 			break;
 
 		default:
-			result = "unknwon error";
+			result = "unknown status";
 			break;
 		}
 	}
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index 48e6942..da2e9b4 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -2889,6 +2889,10 @@ DATA(insert OID = 6118 (  pg_stat_get_subscription	PGNSP PGUID 12 1 0 0 0 f f f
 DESCR("statistics: information about subscription");
 DATA(insert OID = 2579 (  pg_stat_get_vacuum_necessity	PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_vacuum_necessity _null_ _null_ _null_ ));
 DESCR("statistics: true if needs vacuum");
+DATA(insert OID = 3422 (  pg_stat_get_last_vacuum_untruncated	PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_last_vacuum_untruncated _null_ _null_ _null_ ));
+DESCR("statistics: pages left untruncated in the last vacuum");
+DATA(insert OID = 3423 (  pg_stat_get_last_vacuum_truncated	PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_last_vacuum_truncated _null_ _null_ _null_ ));
+DESCR("statistics: pages truncated in the last vacuum");
 DATA(insert OID = 3424 (  pg_stat_get_last_vacuum_oldest_xmin	PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 28 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_last_vacuum_oldest_xmin _null_ _null_ _null_ ));
 DESCR("statistics: The oldest xmin used in the last vacuum");
 DATA(insert OID = 3281 (  pg_stat_get_last_vacuum_index_scans	PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 23 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_last_vacuum_index_scans _null_ _null_ _null_ ));
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index e18a630..6079661 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -389,6 +389,8 @@ typedef struct PgStat_MsgVacuum
 	TimestampTz m_vacuumtime;
 	PgStat_Counter m_live_tuples;
 	PgStat_Counter m_dead_tuples;
+	PgStat_Counter m_pages_removed;
+	PgStat_Counter m_pages_not_removed;
 	PgStat_Counter m_num_index_scans;
 	TransactionId  m_oldest_xmin;
 	PgStat_Counter m_vacuum_status;
@@ -654,6 +656,8 @@ typedef struct PgStat_StatTabEntry
 	PgStat_Counter n_live_tuples;
 	PgStat_Counter n_dead_tuples;
 	PgStat_Counter changes_since_analyze;
+	PgStat_Counter n_pages_removed;
+	PgStat_Counter n_pages_not_removed;
 	PgStat_Counter n_index_scans;
 	TransactionId  oldest_xmin;
 
@@ -1197,6 +1201,8 @@ extern void pgstat_reset_single_counter(Oid objectid, PgStat_Single_Reset_Type t
 extern void pgstat_report_autovac(Oid dboid);
 extern void pgstat_report_vacuum(Oid tableoid, bool shared,
 					 PgStat_Counter livetuples, PgStat_Counter deadtuples,
+					 PgStat_Counter pages_removed,
+					 PgStat_Counter pages_not_removed,
 					 PgStat_Counter num_index_scans,
 					 TransactionId oldextxmin,
 					 PgStat_Counter status, PgStat_Counter last_phase,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 18a122a..111d44f 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1765,6 +1765,8 @@ pg_stat_all_tables| SELECT c.oid AS relid,
     pg_stat_get_last_analyze_time(c.oid) AS last_analyze,
     pg_stat_get_last_autoanalyze_time(c.oid) AS last_autoanalyze,
     pg_stat_get_last_vacuum_index_scans(c.oid) AS last_vacuum_index_scans,
+    pg_stat_get_last_vacuum_truncated(c.oid) AS last_vacuum_truncated,
+    pg_stat_get_last_vacuum_untruncated(c.oid) AS last_vacuum_untruncated,
     pg_stat_get_last_vacuum_oldest_xmin(c.oid) AS last_vacuum_oldest_xmin,
     pg_stat_get_last_vacuum_status(c.oid) AS last_vacuum_status,
     pg_stat_get_last_vacuum_status_detail(c.oid) AS last_vacuum_status_detail,
@@ -1918,6 +1920,8 @@ pg_stat_sys_tables| SELECT pg_stat_all_tables.relid,
     pg_stat_all_tables.last_analyze,
     pg_stat_all_tables.last_autoanalyze,
     pg_stat_all_tables.last_vacuum_index_scans,
+    pg_stat_all_tables.last_vacuum_truncated,
+    pg_stat_all_tables.last_vacuum_untruncated,
     pg_stat_all_tables.last_vacuum_oldest_xmin,
     pg_stat_all_tables.last_vacuum_status,
     pg_stat_all_tables.last_vacuum_status_detail,
@@ -1967,6 +1971,8 @@ pg_stat_user_tables| SELECT pg_stat_all_tables.relid,
     pg_stat_all_tables.last_analyze,
     pg_stat_all_tables.last_autoanalyze,
     pg_stat_all_tables.last_vacuum_index_scans,
+    pg_stat_all_tables.last_vacuum_truncated,
+    pg_stat_all_tables.last_vacuum_untruncated,
     pg_stat_all_tables.last_vacuum_oldest_xmin,
     pg_stat_all_tables.last_vacuum_status,
     pg_stat_all_tables.last_vacuum_status_detail,
-- 
2.9.2

