From a65c1d160d4a97a021e4d946f8fe14fe04819a16 Mon Sep 17 00:00:00 2001 From: Osumi Takamichi Date: Tue, 12 Jan 2021 02:35:16 +0000 Subject: [PATCH v06] new wal_level to disable WAL logging In order to speed up the performance, especially for bulk data loading or pg_dumpall, this feature generates only limited types of WALs. This means we gain this speed-up even at the cost of crash recovery. During the operation of this new wal_level, an unexpected stoppage or shutdown of the server makes the whole cluster corrupted and unrecoverable. In other words, any kind of accidents make the server never start up again. Therefore, taking a full backup before and after the change of this wal_level is a must. Author: Takamichi Osumi Reviewed-by: Tsunakawa Takayuki Reviewed-by: Fujii Masao Reviewed-by: Laurenz Albe Reviewed-by: Ashutosh Bapat Reviewed-by: Kyotaro Horiguchi Reviewed-by: Masahiko Sawada Reviewed-by: Robert Haas Discussion: https://www.postgresql.org/message-id/TYAPR01MB29901EBE5A3ACCE55BA99186FE320%40TYAPR01MB2990.jpnprd01.prod.outlook.com --- doc/src/sgml/config.sgml | 20 ++++++++++++++++++-- doc/src/sgml/perform.sgml | 13 ++++++++++--- src/backend/access/rmgrdesc/xlogdesc.c | 1 + src/backend/access/transam/varsup.c | 2 +- src/backend/access/transam/xlog.c | 15 ++++++++++++--- src/backend/access/transam/xloginsert.c | 12 ++++++++++++ src/backend/postmaster/postmaster.c | 6 +++--- src/backend/utils/misc/postgresql.conf.sample | 6 ++++-- src/bin/pg_controldata/pg_controldata.c | 2 ++ src/include/access/xlog.h | 3 ++- src/include/access/xlogdefs.h | 2 +- src/include/utils/rel.h | 3 ++- 12 files changed, 68 insertions(+), 17 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7c0a673..65a4a24 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2622,7 +2622,16 @@ include_dir 'conf.d' data to support WAL archiving and replication, including running read-only queries on a standby server. minimal removes all logging except the information required to recover from a crash or - immediate shutdown. Finally, + immediate shutdown. none generates strictly limited types of WAL. + Those are WAL for transaction, transaction resources, GIST and generic WAL record type. + This means that the amount of WAL during none + is much less than that of minimal for ordinal operation. + Intrinsically, the purpose of none + is to accelerate data bulk loading at the expense of recovery. Accordingly, note that + any kind of accidental server stoppage during none makes the whole cluster + corrupted and never start up again. Therefore, never use this mode unless the operation + during this mode is repeatable and the cluster can be backed up before and after the operation. + Thus, this mode requires careful planning and preparation. Finally, logical adds information necessary to support logical decoding. Each level includes the information logged at all lower levels. This parameter can only be set at server start. @@ -2646,6 +2655,12 @@ include_dir 'conf.d' data from a base backup and the WAL logs, so replica or higher must be used to enable WAL archiving () and streaming replication. + In the same way, none does not create almost all types of WAL logs in principle. + Therefore, this wal_level can be used to maximize the speed of data loading. + For example, bulk data loading or version upgrade using pg_dumpall. + On the other hand, an unexpected crash of the server makes the database cluster + inconsistent and never able to restart. For that reason, before and after utilizing this level, + get a full backup of the whole cluster. In logical level, the same information is logged as @@ -3403,7 +3418,8 @@ include_dir 'conf.d' changed without leaving archiving mode. This parameter can only be set at server start. archive_mode cannot be enabled when - wal_level is set to minimal. + wal_level is set to none or + minimal. diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 117a1f7..9b5202b 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1741,10 +1741,17 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; new base backup after the load has completed than to process a large amount of incremental WAL data. To prevent incremental WAL logging while loading, disable archiving and streaming replication, by setting - to minimal, + to either none + or minimal, to off, and to zero. - But note that changing these settings requires a server restart. + Changing wal_level to none + is extremely performance-oriented feature. Therefore, paying + a careful attention that a crash or server stoppage during the data loading causes + corruption of the whole cluster is needed. When it happens, + the server will not restart again any more. Thus, the administrator + needs to set up the cluster from the full backup taken just before + the operation. Also, note that changing these settings requires a server restart. @@ -1810,7 +1817,7 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; If using WAL archiving or streaming replication, consider disabling them during the restore. To do that, set archive_mode to off, - wal_level to minimal, and + wal_level to minimal or none, and max_wal_senders to zero before loading the dump. Afterwards, set them back to the right values and take a fresh base backup. diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index 92cc7ea..8b6b640 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -25,6 +25,7 @@ * GUC support */ const struct config_enum_entry wal_level_options[] = { + {"none", WAL_LEVEL_NONE, false}, {"minimal", WAL_LEVEL_MINIMAL, false}, {"replica", WAL_LEVEL_REPLICA, false}, {"archive", WAL_LEVEL_REPLICA, true}, /* deprecated */ diff --git a/src/backend/access/transam/varsup.c b/src/backend/access/transam/varsup.c index 2264c2c..f2d2e7e 100644 --- a/src/backend/access/transam/varsup.c +++ b/src/backend/access/transam/varsup.c @@ -368,7 +368,7 @@ SetTransactionIdLimit(TransactionId oldest_datfrozenxid, Oid oldest_datoid) * within 3M transactions of data loss. This leaves lots of room for the * DBA to fool around fixing things in a standalone backend, while not * being significant compared to total XID space. (VACUUM requires an XID - * if it truncates at wal_level!=minimal. "VACUUM (ANALYZE)", which a DBA + * if it truncates at wal_level<=minimal. "VACUUM (ANALYZE)", which a DBA * might do by reflex, assigns an XID. Hence, we had better be sure * there's lots of XIDs left...) Also, at default BLCKSZ, this leaves two * completely-idle segments. In the event of edge-case bugs involving diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index ede93ad..b04f815 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -6267,11 +6267,11 @@ CheckRequiredParameterValues(void) * For archive recovery, the WAL must be generated with at least 'replica' * wal_level. */ - if (ArchiveRecoveryRequested && ControlFile->wal_level == WAL_LEVEL_MINIMAL) + if (ArchiveRecoveryRequested && ControlFile->wal_level <= WAL_LEVEL_MINIMAL) { ereport(WARNING, - (errmsg("WAL was generated with wal_level=minimal, data may be missing"), - errhint("This happens if you temporarily set wal_level=minimal without taking a new base backup."))); + (errmsg("WAL was generated with wal_level<=minimal, data may be missing"), + errhint("This happens if you temporarily set wal_level<=minimal without taking a new base backup."))); } /* @@ -6399,6 +6399,15 @@ StartupXLOG(void) (errmsg("control file contains invalid database cluster state"))); } + /* + * Detect if the server previously crashed under wal_level='none' or not. + */ + if (ControlFile->wal_level == WAL_LEVEL_NONE && + (ControlFile->state != DB_SHUTDOWNED && ControlFile->state != DB_SHUTDOWNED_IN_RECOVERY)) + ereport(ERROR, + (errmsg("detected an unexpected server shutdown when WAL logging was disabled"), + errhint("It looks like you need to deploy a new cluster from your full backup again."))); + /* This is just to allow attaching to startup process with a debugger */ #ifdef XLOG_REPLAY_DELAY if (ControlFile->state != DB_SHUTDOWNED) diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 7052dc2..de0a92d 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -449,6 +449,18 @@ XLogInsert(RmgrId rmid, uint8 info) return EndPos; } + /* Issues only limited types of WAL such as XLOG resources and transaction */ + if (wal_level == WAL_LEVEL_NONE && + !((rmid == RM_XLOG_ID && info == XLOG_CHECKPOINT_SHUTDOWN) || + (rmid == RM_XLOG_ID && info == XLOG_PARAMETER_CHANGE) || + (rmid == RM_XACT_ID && info == XLOG_XACT_PREPARE) || + (rmid == RM_GIST_ID && info == RM_GIST_ID) || + (rmid == RM_GENERIC_ID))) + { + XLogResetInsertion(); + return GetXLogInsertRecPtr(); + } + do { XLogRecPtr RedoRecPtr; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 7de27ee..437d649 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -904,10 +904,10 @@ PostmasterMain(int argc, char *argv[]) ReservedBackends, MaxConnections); ExitPostmaster(1); } - if (XLogArchiveMode > ARCHIVE_MODE_OFF && wal_level == WAL_LEVEL_MINIMAL) + if (XLogArchiveMode > ARCHIVE_MODE_OFF && wal_level <= WAL_LEVEL_MINIMAL) ereport(ERROR, - (errmsg("WAL archival cannot be enabled when wal_level is \"minimal\""))); - if (max_wal_senders > 0 && wal_level == WAL_LEVEL_MINIMAL) + (errmsg("WAL archival cannot be enabled when wal_level is \"none\" or \"\"minimal"))); + if (max_wal_senders > 0 && wal_level <= WAL_LEVEL_MINIMAL) ereport(ERROR, (errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\""))); diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 8930a94..0b73316 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -196,8 +196,10 @@ # - Settings - -#wal_level = replica # minimal, replica, or logical - # (change requires restart) +#wal_level = replica # none, minimal, replica, or logical + # (change requires restart. + # choosing the none wal_level + # can cause unrecoverable data corruption) #fsync = on # flush data to disk for crash safety # (turning this off can cause # unrecoverable data corruption) diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 3e00ac0..90ec0dc 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -74,6 +74,8 @@ wal_level_str(WalLevel wal_level) { switch (wal_level) { + case WAL_LEVEL_NONE: + return "none"; case WAL_LEVEL_MINIMAL: return "minimal"; case WAL_LEVEL_REPLICA: diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 75ec107..8f96c4e 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -161,7 +161,8 @@ extern int XLogArchiveMode; /* WAL levels */ typedef enum WalLevel { - WAL_LEVEL_MINIMAL = 0, + WAL_LEVEL_NONE = 0, + WAL_LEVEL_MINIMAL, WAL_LEVEL_REPLICA, WAL_LEVEL_LOGICAL } WalLevel; diff --git a/src/include/access/xlogdefs.h b/src/include/access/xlogdefs.h index 65836d5..c2e6954 100644 --- a/src/include/access/xlogdefs.h +++ b/src/include/access/xlogdefs.h @@ -59,7 +59,7 @@ typedef uint16 RepOriginId; /* * Because O_DIRECT bypasses the kernel buffers, and because we never - * read those buffers except during crash recovery or if wal_level != minimal, + * read those buffers except during crash recovery or if wal_level <= minimal, * it is a win to use it in all cases where we sync on each write(). We could * allow O_DIRECT with fsync(), but it is unclear if fsync() could process * writes not buffered in the kernel. Also, O_DIRECT is never enough to force diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 10b6398..b7a06e9 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -561,7 +561,8 @@ typedef struct ViewOptions * RelFileNode" in src/backend/access/transam/README. */ #define RelationNeedsWAL(relation) \ - ((relation)->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT && \ + (wal_level != WAL_LEVEL_NONE && \ + (relation)->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT && \ (XLogIsNeeded() || \ (relation->rd_createSubid == InvalidSubTransactionId && \ relation->rd_firstRelfilenodeSubid == InvalidSubTransactionId))) -- 2.2.0