From d49a4329630bd9d943cd770e180c3c70bb0c7817 Mon Sep 17 00:00:00 2001 From: Peter Geoghegan Date: Mon, 27 Mar 2023 19:09:05 -0700 Subject: [PATCH v8] Emit WAL record info via pg_get_wal_block_info --- .../pg_walinspect/expected/pg_walinspect.out | 4 +- .../pg_walinspect/pg_walinspect--1.0--1.1.sql | 21 +- contrib/pg_walinspect/pg_walinspect.c | 207 +++++++++++------- contrib/pg_walinspect/sql/pg_walinspect.sql | 4 +- doc/src/sgml/pgwalinspect.sgml | 99 ++++++--- 5 files changed, 211 insertions(+), 124 deletions(-) diff --git a/contrib/pg_walinspect/expected/pg_walinspect.out b/contrib/pg_walinspect/expected/pg_walinspect.out index b9e2bb994..950f0e921 100644 --- a/contrib/pg_walinspect/expected/pg_walinspect.out +++ b/contrib/pg_walinspect/expected/pg_walinspect.out @@ -121,7 +121,7 @@ UPDATE sample_tbl SET col1 = col1 + 1 WHERE col1 = 1; SELECT pg_current_wal_lsn() AS wal_lsn4 \gset -- Check if we get block data from WAL record. SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_block_info(:'wal_lsn3', :'wal_lsn4') - WHERE relfilenode = :'sample_tbl_oid' AND blockdata IS NOT NULL; + WHERE relfilenode = :'sample_tbl_oid' AND block_data IS NOT NULL; ok ---- t @@ -134,7 +134,7 @@ UPDATE sample_tbl SET col1 = col1 + 1 WHERE col1 = 2; SELECT pg_current_wal_lsn() AS wal_lsn6 \gset -- Check if we get FPI from WAL record. SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_block_info(:'wal_lsn5', :'wal_lsn6') - WHERE relfilenode = :'sample_tbl_oid' AND fpi IS NOT NULL; + WHERE relfilenode = :'sample_tbl_oid' AND block_fpi_data IS NOT NULL; ok ---- t diff --git a/contrib/pg_walinspect/pg_walinspect--1.0--1.1.sql b/contrib/pg_walinspect/pg_walinspect--1.0--1.1.sql index 586c3b446..8a314158e 100644 --- a/contrib/pg_walinspect/pg_walinspect--1.0--1.1.sql +++ b/contrib/pg_walinspect/pg_walinspect--1.0--1.1.sql @@ -12,17 +12,26 @@ DROP FUNCTION pg_get_wal_stats_till_end_of_wal(pg_lsn, boolean); -- CREATE FUNCTION pg_get_wal_block_info(IN start_lsn pg_lsn, IN end_lsn pg_lsn, - OUT lsn pg_lsn, + OUT start_lsn pg_lsn, + OUT end_lsn pg_lsn, + OUT prev_lsn pg_lsn, OUT blockid int2, OUT reltablespace oid, OUT reldatabase oid, OUT relfilenode oid, + OUT relforknumber int2, OUT relblocknumber int8, - OUT forkname text, - OUT blockdata bytea, - OUT fpi bytea, - OUT fpilen int4, - OUT fpiinfo text[] + OUT xid xid, + OUT resource_manager text, + OUT record_type text, + OUT record_length int4, + OUT main_data_length int4, + OUT block_data_length int4, + OUT block_fpi_length int4, + OUT block_fpi_info text[], + OUT description text, + OUT block_data bytea, + OUT block_fpi_data bytea ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_get_wal_block_info' diff --git a/contrib/pg_walinspect/pg_walinspect.c b/contrib/pg_walinspect/pg_walinspect.c index 062e90dbc..324e771f1 100644 --- a/contrib/pg_walinspect/pg_walinspect.c +++ b/contrib/pg_walinspect/pg_walinspect.c @@ -176,13 +176,18 @@ ReadNextXLogRecord(XLogReaderState *xlogreader) } /* - * Get a single WAL record info. + * Output values that make up a row describing caller's WAL record. + * + * This function leaks memory. Caller may need to use its own custom memory + * context. + * + * Keep this in sync with GetWALBlockInfo. */ static void GetWALRecordInfo(XLogReaderState *record, Datum *values, bool *nulls, uint32 ncols) { - const char *id; + const char *record_type; RmgrData desc; uint32 fpi_len = 0; StringInfoData rec_desc; @@ -190,10 +195,10 @@ GetWALRecordInfo(XLogReaderState *record, Datum *values, int i = 0; desc = GetRmgr(XLogRecGetRmid(record)); - id = desc.rm_identify(XLogRecGetInfo(record)); + record_type = desc.rm_identify(XLogRecGetInfo(record)); - if (id == NULL) - id = psprintf("UNKNOWN (%x)", XLogRecGetInfo(record) & ~XLR_INFO_MASK); + if (record_type == NULL) + record_type = psprintf("UNKNOWN (%x)", XLogRecGetInfo(record) & ~XLR_INFO_MASK); initStringInfo(&rec_desc); desc.rm_desc(&rec_desc, record); @@ -209,7 +214,7 @@ GetWALRecordInfo(XLogReaderState *record, Datum *values, values[i++] = LSNGetDatum(XLogRecGetPrev(record)); values[i++] = TransactionIdGetDatum(XLogRecGetXid(record)); values[i++] = CStringGetTextDatum(desc.rm_name); - values[i++] = CStringGetTextDatum(id); + values[i++] = CStringGetTextDatum(record_type); values[i++] = UInt32GetDatum(XLogRecGetTotalLen(record)); values[i++] = UInt32GetDatum(XLogRecGetDataLen(record)); values[i++] = UInt32GetDatum(fpi_len); @@ -229,101 +234,84 @@ GetWALRecordInfo(XLogReaderState *record, Datum *values, /* - * Store a set of block information from a single record (FPI and block - * information). + * Output one or more rows in rsinfo tuple store, each describing a single + * block reference from caller's WAL record. (Should only be called with + * records that have block references.) + * + * This function leaks memory. Caller may need to use its own custom memory + * context. + * + * Keep this in sync with GetWALRecordInfo. */ static void GetWALBlockInfo(FunctionCallInfo fcinfo, XLogReaderState *record) { -#define PG_GET_WAL_BLOCK_INFO_COLS 11 - int block_id; +#define PG_GET_WAL_BLOCK_INFO_COLS 20 + int blockid; ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + RmgrData desc; + const char *record_type; + StringInfoData rec_desc; - for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++) + Assert(XLogRecHasAnyBlockRefs(record)); + + desc = GetRmgr(XLogRecGetRmid(record)); + record_type = desc.rm_identify(XLogRecGetInfo(record)); + + if (record_type == NULL) + record_type = psprintf("UNKNOWN (%x)", + XLogRecGetInfo(record) & ~XLR_INFO_MASK); + + initStringInfo(&rec_desc); + desc.rm_desc(&rec_desc, record); + + for (blockid = 0; blockid <= XLogRecMaxBlockId(record); blockid++) { DecodedBkpBlock *blk; BlockNumber blkno; RelFileLocator rnode; - ForkNumber fork; + ForkNumber forknum; Datum values[PG_GET_WAL_BLOCK_INFO_COLS] = {0}; bool nulls[PG_GET_WAL_BLOCK_INFO_COLS] = {0}; + uint32 block_data_len = 0, + block_fpi_len = 0; + PGAlignedBlock buf; + Page page; + ArrayType *block_fpi_info = NULL; int i = 0; - if (!XLogRecHasBlockRef(record, block_id)) + if (!XLogRecHasBlockRef(record, blockid)) continue; - blk = XLogRecGetBlock(record, block_id); + blk = XLogRecGetBlock(record, blockid); - (void) XLogRecGetBlockTagExtended(record, block_id, - &rnode, &fork, &blkno, NULL); + (void) XLogRecGetBlockTagExtended(record, blockid, + &rnode, &forknum, &blkno, NULL); - values[i++] = LSNGetDatum(record->ReadRecPtr); - values[i++] = Int16GetDatum(block_id); - values[i++] = ObjectIdGetDatum(blk->rlocator.spcOid); - values[i++] = ObjectIdGetDatum(blk->rlocator.dbOid); - values[i++] = ObjectIdGetDatum(blk->rlocator.relNumber); - values[i++] = Int64GetDatum((int64) blkno); - - if (fork >= 0 && fork <= MAX_FORKNUM) - values[i++] = CStringGetTextDatum(forkNames[fork]); - else - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg_internal("invalid fork number: %u", fork))); - - /* Block data */ + /* Save block_data_len */ if (blk->has_data) - { - bytea *raw_data; - - /* Initialize bytea buffer to copy the data to */ - raw_data = (bytea *) palloc(blk->data_len + VARHDRSZ); - SET_VARSIZE(raw_data, blk->data_len + VARHDRSZ); - - /* Copy the data */ - memcpy(VARDATA(raw_data), blk->data, blk->data_len); - values[i++] = PointerGetDatum(raw_data); - } - else - { - /* No data, so set this field to NULL */ - nulls[i++] = true; - } + block_data_len = blk->data_len; if (blk->has_image) { - PGAlignedBlock buf; - Page page; - bytea *raw_page; + /* Block reference has an FPI, so prepare relevant output */ int bitcnt; int cnt = 0; Datum *flags; - ArrayType *a; page = (Page) buf.data; - - /* Full page image exists, so let's save it */ - if (!RestoreBlockImage(record, block_id, page)) + if (!RestoreBlockImage(record, blockid, page)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg_internal("%s", record->errormsg_buf))); - /* Initialize bytea buffer to copy the FPI to */ - raw_page = (bytea *) palloc(BLCKSZ + VARHDRSZ); - SET_VARSIZE(raw_page, BLCKSZ + VARHDRSZ); + /* Save block_fpi_len */ + block_fpi_len = blk->bimg_len; - /* Take a verbatim copy of the FPI */ - memcpy(VARDATA(raw_page), page, BLCKSZ); - - values[i++] = PointerGetDatum(raw_page); - values[i++] = UInt32GetDatum(blk->bimg_len); - - /* FPI flags */ + /* Construct and save block_fpi_info */ bitcnt = pg_popcount((const char *) &blk->bimg_info, sizeof(uint8)); - /* Build set of raw flags */ flags = (Datum *) palloc0(sizeof(Datum) * bitcnt); - if ((blk->bimg_info & BKPIMAGE_HAS_HOLE) != 0) flags[cnt++] = CStringGetTextDatum("HAS_HOLE"); if (blk->apply_image) @@ -336,18 +324,85 @@ GetWALBlockInfo(FunctionCallInfo fcinfo, XLogReaderState *record) flags[cnt++] = CStringGetTextDatum("COMPRESS_ZSTD"); Assert(cnt <= bitcnt); - a = construct_array_builtin(flags, cnt, TEXTOID); - values[i++] = PointerGetDatum(a); + block_fpi_info = construct_array_builtin(flags, cnt, TEXTOID); + } + + /* start_lsn, end_lsn, prev_lsn, and blockid outputs */ + values[i++] = LSNGetDatum(record->ReadRecPtr); + values[i++] = LSNGetDatum(record->EndRecPtr); + values[i++] = LSNGetDatum(XLogRecGetPrev(record)); + values[i++] = Int16GetDatum(blockid); + + /* relfile and block related outputs */ + values[i++] = ObjectIdGetDatum(blk->rlocator.spcOid); + values[i++] = ObjectIdGetDatum(blk->rlocator.dbOid); + values[i++] = ObjectIdGetDatum(blk->rlocator.relNumber); + values[i++] = Int16GetDatum(forknum); + values[i++] = Int64GetDatum((int64) blkno); + + /* xid, resource_manager, and record_type outputs */ + values[i++] = TransactionIdGetDatum(XLogRecGetXid(record)); + values[i++] = CStringGetTextDatum(desc.rm_name); + values[i++] = CStringGetTextDatum(record_type); + + /* + * record_length, main_data_length, block_data_len, and + * block_fpi_length outputs + */ + values[i++] = UInt32GetDatum(XLogRecGetTotalLen(record)); + values[i++] = UInt32GetDatum(XLogRecGetDataLen(record)); + values[i++] = UInt32GetDatum(block_data_len); + values[i++] = UInt32GetDatum(block_fpi_len); + + /* block_fpi_info (text array) output */ + if (block_fpi_info) + values[i++] = PointerGetDatum(block_fpi_info); + else + nulls[i++] = true; + + /* description output (describes WAL record) */ + if (rec_desc.len > 0) + values[i++] = CStringGetTextDatum(rec_desc.data); + else + nulls[i++] = true; + + /* block_data output */ + if (blk->has_data) + { + bytea *raw_data; + + /* Initialize bytea buffer to copy the data to */ + Assert(block_data_len > 0); + raw_data = (bytea *) palloc(block_data_len + VARHDRSZ); + SET_VARSIZE(raw_data, block_data_len + VARHDRSZ); + + /* Copy the data */ + memcpy(VARDATA(raw_data), blk->data, block_data_len); + values[i++] = PointerGetDatum(raw_data); } else + nulls[i++] = true; + + /* block_fpi_data output */ + if (blk->has_image) { - /* No full page image, so store NULLs for all its fields */ - memset(&nulls[i], true, 3 * sizeof(bool)); - i += 3; + bytea *raw_page; + + /* Initialize bytea buffer to copy the FPI to */ + raw_page = (bytea *) palloc(BLCKSZ + VARHDRSZ); + SET_VARSIZE(raw_page, BLCKSZ + VARHDRSZ); + + /* Take a verbatim copy of the FPI */ + memcpy(VARDATA(raw_page), page, BLCKSZ); + + values[i++] = PointerGetDatum(raw_page); } + else + nulls[i++] = true; Assert(i == PG_GET_WAL_BLOCK_INFO_COLS); + /* Store a tuple for this block reference */ tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); } @@ -356,11 +411,7 @@ GetWALBlockInfo(FunctionCallInfo fcinfo, XLogReaderState *record) } /* - * Get information about all the blocks saved in WAL records between start - * and end LSNs. This produces information about the full page images with - * their relation information, and the data saved in each block associated - * to a record. Decompression is applied to the full page images, if - * necessary. + * Get WAL record info, disaggregated on block reference */ Datum pg_get_wal_block_info(PG_FUNCTION_ARGS) @@ -484,7 +535,7 @@ ValidateInputLSNs(XLogRecPtr start_lsn, XLogRecPtr *end_lsn) } /* - * Get info and data of all WAL records between start LSN and end LSN. + * Get info of all WAL records between start LSN and end LSN. */ static void GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, @@ -536,7 +587,7 @@ GetWALRecordsInfo(FunctionCallInfo fcinfo, XLogRecPtr start_lsn, } /* - * Get info and data of all WAL records between start LSN and end LSN. + * Get info of all WAL records between start LSN and end LSN. */ Datum pg_get_wal_records_info(PG_FUNCTION_ARGS) diff --git a/contrib/pg_walinspect/sql/pg_walinspect.sql b/contrib/pg_walinspect/sql/pg_walinspect.sql index 2f0a909c1..0541e5fbf 100644 --- a/contrib/pg_walinspect/sql/pg_walinspect.sql +++ b/contrib/pg_walinspect/sql/pg_walinspect.sql @@ -78,7 +78,7 @@ UPDATE sample_tbl SET col1 = col1 + 1 WHERE col1 = 1; SELECT pg_current_wal_lsn() AS wal_lsn4 \gset -- Check if we get block data from WAL record. SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_block_info(:'wal_lsn3', :'wal_lsn4') - WHERE relfilenode = :'sample_tbl_oid' AND blockdata IS NOT NULL; + WHERE relfilenode = :'sample_tbl_oid' AND block_data IS NOT NULL; -- Force full-page image on the next update. SELECT pg_current_wal_lsn() AS wal_lsn5 \gset @@ -87,7 +87,7 @@ UPDATE sample_tbl SET col1 = col1 + 1 WHERE col1 = 2; SELECT pg_current_wal_lsn() AS wal_lsn6 \gset -- Check if we get FPI from WAL record. SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_block_info(:'wal_lsn5', :'wal_lsn6') - WHERE relfilenode = :'sample_tbl_oid' AND fpi IS NOT NULL; + WHERE relfilenode = :'sample_tbl_oid' AND block_fpi_data IS NOT NULL; -- =================================================================== -- Tests for permissions diff --git a/doc/src/sgml/pgwalinspect.sgml b/doc/src/sgml/pgwalinspect.sgml index 9a0241a8d..ffcb39bb2 100644 --- a/doc/src/sgml/pgwalinspect.sgml +++ b/doc/src/sgml/pgwalinspect.sgml @@ -94,9 +94,10 @@ block_ref | blkref #0: rel 1663/5/60221 fork main blk 2 Gets information of all the valid WAL records between start_lsn and end_lsn. - Returns one row per WAL record. If a future - end_lsn (i.e. ahead of the current LSN of - the server) is specified, it returns information until the end of WAL. + Returns one row per WAL record. If end_lsn + specified is in future (i.e. ahead of the current LSN of the server) or + FFFFFFFF/FFFFFFFF (i.e. highest possible value for + pg_lsn type), it returns information until the end of WAL. The function raises an error if start_lsn is not available. For example, usage of the function is as follows: @@ -133,11 +134,13 @@ block_ref | end_lsn. By default, it returns one row per resource_manager type. When per_record is set to true, - it returns one row per record_type. If a - future end_lsn (i.e. ahead of the current - LSN of the server) is specified, it returns statistics until the end - of WAL. An error is raised if start_lsn is - not available. For example, usage of the function is as follows: + it returns one row per record_type. + If end_lsn specified is in future (i.e. ahead + of the current LSN of the server) or FFFFFFFF/FFFFFFFF + (i.e. highest possible value for pg_lsn type), it returns + statistics until the end of WAL. An error is raised if + start_lsn is not available. For example, usage + of the function is as follows: postgres=# SELECT * FROM pg_get_wal_stats('0/1E847D00', '0/1E84F500') WHERE count > 0 LIMIT 1 AND @@ -164,38 +167,62 @@ combined_size_percentage | 2.8634072910530795 - Gets a copy of the block information stored in WAL records. This includes - copies of the block data (NULL if none) and full page - images as bytea values (after - applying decompression when necessary, or NULL if none) - and their information associated with all the valid WAL records between - start_lsn and - end_lsn. Returns one row per block registered - in a WAL record. If a future end_lsn (i.e. - ahead of the current LSN of the server) is specified, it returns - statistics until the end of WAL. An error is raised if - start_lsn is not available. For example, - usage of the function is as follows: + Gets both WAL record and block information of all the valid WAL records + between start_lsn and + end_lsn. It omits WAL records that contain no + block references. The block information includes copies of the block data + (NULL if none) and full page images as + bytea values (after applying decompression when necessary, + or NULL if none). Returns one row per block registered + in a WAL record. If end_lsn specified is in + future (i.e. ahead of the current LSN of the server) or + FFFFFFFF/FFFFFFFF (i.e. highest possible value for + pg_lsn type), it returns information until the end of WAL. + An error is raised if start_lsn is not + available. For example, usage of the function is as follows: -postgres=# SELECT lsn, blockid, reltablespace, reldatabase, relfilenode, - relblocknumber, forkname, - substring(blockdata for 24) as block_trimmed, +postgres=# SELECT start_lsn, end_lsn, prev_lsn, reltablespace, + reldatabase, relfilenode, relblocknumber, + blockid, xid, resource_manager, record_type, + record_length, main_data_length, description, + forkname, substring(blockdata for 24) as block_trimmed, substring(fpi for 24) as fpi_trimmed, fpilen, fpiinfo - FROM pg_get_wal_block_info('0/1871080', '0/1871440'); --[ RECORD 1 ]--+--------------------------------------------------- -lsn | 0/18712F8 -blockid | 0 -reltablespace | 1663 -reldatabase | 16384 -relfilenode | 16392 -relblocknumber | 0 -forkname | main -block_trimmed | \x02800128180164000000 -fpi_trimmed | \x0000000050108701000000002c00601f00200420e0020000 -fpilen | 204 -fpiinfo | {HAS_HOLE,APPLY} + FROM pg_get_wal_block_info('0/14BF938', '0/14BFBA0'); +-[ RECORD 1 ]----+--------------------------------------------------- +start_lsn | 0/14BFA20 +end_lsn | 0/14BFB38 +prev_lsn | 0/14BF9A8 +reltablespace | 1663 +reldatabase | 5 +relfilenode | 16391 +relblocknumber | 9 +blockid | 0 +xid | 735 +resource_manager | Heap +record_type | HOT_UPDATE +record_length | 279 +main_data_length | 14 +description | off 1 xmax 735 flags 0x10 ; new off 5 xmax 0 +forkname | main +block_trimmed | \x02800128180102000000 +fpi_trimmed | \x00000000d0f84b01000000002c00601f00200420df020000 +fpilen | 204 +fpiinfo | {HAS_HOLE,APPLY} + + + Note that although pg_get_wal_records_info + and pg_get_wal_block_info functions return + information of WAL records in common, the + pg_get_wal_records_info outputs block information + (without block data and FPI data) as a text column + (NULL if none), and + pg_get_wal_block_info outputs one row for each block + (including block data and FPI data) associated with each WAL record, omitting + WAL records that contain no block references. + + -- 2.39.2