diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index cb95aa3..05ac0fa 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -722,7 +722,7 @@ WriteTruncateXlogRec(int pageno) rdata.buffer = InvalidBuffer; rdata.next = NULL; recptr = XLogInsert(RM_CLOG_ID, CLOG_TRUNCATE, &rdata); - XLogFlush(recptr); + XLogFlush(recptr, true, true); } /* diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index 5a8f654..452af68 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -696,9 +696,11 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) * XLogFlush were to fail, we must PANIC. This isn't much of a * restriction because XLogFlush is just about all critical * section anyway, but let's make sure. + * Also wait for the synchronous standby to receive WAL upto + * max_lsn. */ START_CRIT_SECTION(); - XLogFlush(max_lsn); + XLogFlush(max_lsn, true, true); END_CRIT_SECTION(); } } diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index e975f8d..0c235c9 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1049,7 +1049,14 @@ EndPrepare(GlobalTransaction gxact) gxact->prepare_lsn = XLogInsert(RM_XACT_ID, XLOG_XACT_PREPARE, records.head); - XLogFlush(gxact->prepare_lsn); + + /* + * Wait for synchronous replication, if required. + * + * Note that at this stage we have marked the prepare, but still show as + * running in the procarray (twice!) and continue to hold locks. + */ + XLogFlush(gxact->prepare_lsn, false, true); /* If we crash now, we have prepared: WAL replay will fix things */ @@ -1090,14 +1097,6 @@ EndPrepare(GlobalTransaction gxact) END_CRIT_SECTION(); - /* - * Wait for synchronous replication, if required. - * - * Note that at this stage we have marked the prepare, but still show as - * running in the procarray (twice!) and continue to hold locks. - */ - SyncRepWaitForLSN(gxact->prepare_lsn); - records.tail = records.head = NULL; } @@ -2046,8 +2045,14 @@ RecordTransactionCommitPrepared(TransactionId xid, * a contradiction) */ - /* Flush XLOG to disk */ - XLogFlush(recptr); + /* + * Flush XLOG to disk, + * Wait for synchronous replication, if required. + * + * Note that at this stage we have marked clog, but still show as running + * in the procarray and continue to hold locks. + */ + XLogFlush(recptr, false, true); /* Mark the transaction committed in pg_clog */ TransactionIdCommitTree(xid, nchildren, children); @@ -2056,14 +2061,6 @@ RecordTransactionCommitPrepared(TransactionId xid, MyPgXact->delayChkpt = false; END_CRIT_SECTION(); - - /* - * Wait for synchronous replication, if required. - * - * Note that at this stage we have marked clog, but still show as running - * in the procarray and continue to hold locks. - */ - SyncRepWaitForLSN(recptr); } /* @@ -2126,8 +2123,14 @@ RecordTransactionAbortPrepared(TransactionId xid, recptr = XLogInsert(RM_XACT_ID, XLOG_XACT_ABORT_PREPARED, rdata); - /* Always flush, since we're about to remove the 2PC state file */ - XLogFlush(recptr); + /* + * Always flush, since we're about to remove the 2PC state file. + * Wait for synchronous replication, if required. + * + * Note that at this stage we have marked clog, but still show as running + * in the procarray and continue to hold locks. + */ + XLogFlush(recptr, false, true); /* * Mark the transaction aborted in clog. This is not absolutely necessary @@ -2136,12 +2139,4 @@ RecordTransactionAbortPrepared(TransactionId xid, TransactionIdAbortTree(xid, nchildren, children); END_CRIT_SECTION(); - - /* - * Wait for synchronous replication, if required. - * - * Note that at this stage we have marked clog, but still show as running - * in the procarray and continue to hold locks. - */ - SyncRepWaitForLSN(recptr); } diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 31e868d..9331742 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -965,6 +965,7 @@ RecordTransactionCommit(void) SharedInvalidationMessage *invalMessages = NULL; bool RelcacheInitFileInval = false; bool wrote_xlog; + bool ret = false; /* Get data needed for commit record */ nrels = smgrGetPendingDeletes(true, &rels); @@ -1143,8 +1144,8 @@ RecordTransactionCommit(void) if ((wrote_xlog && synchronous_commit > SYNCHRONOUS_COMMIT_OFF) || forceSyncCommit || nrels > 0) { - XLogFlush(XactLastRecEnd); - + XLogFlush(XactLastRecEnd, false, true); + ret = true; /* * Now we may update the CLOG, if we wrote a COMMIT record above */ @@ -1195,7 +1196,7 @@ RecordTransactionCommit(void) * in the procarray and continue to hold locks. */ if (wrote_xlog) - SyncRepWaitForLSN(XactLastRecEnd); + SyncRepWaitForLSN(XactLastRecEnd, false, !ret); /* Reset XactLastRecEnd until the next transaction writes something */ XactLastRecEnd = 0; @@ -4663,7 +4664,7 @@ xact_redo_commit_internal(TransactionId xid, XLogRecPtr lsn, * after deletion, but that would leave a small window where the * WAL-first rule would be violated. */ - XLogFlush(lsn); + XLogFlush(lsn, true, false); for (i = 0; i < nrels; i++) { @@ -4690,7 +4691,7 @@ xact_redo_commit_internal(TransactionId xid, XLogRecPtr lsn, * for any user that requested ForceSyncCommit(). */ if (XactCompletionForceSyncCommit(xinfo)) - XLogFlush(lsn); + XLogFlush(lsn, true, false); } diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c9e3a7a..5f6b9ba 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1224,7 +1224,7 @@ begin:; if (isLogSwitch) { TRACE_POSTGRESQL_XLOG_SWITCH(); - XLogFlush(EndPos); + XLogFlush(EndPos, true, true); /* * Even though we reserved the rest of the segment for us, which is * reflected in EndPos, we return a pointer to just the end of the @@ -2996,7 +2996,7 @@ UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force) * already held, and we try to avoid acquiring it if possible. */ void -XLogFlush(XLogRecPtr record) +XLogFlush(XLogRecPtr record, bool ForDataFlush, bool Wait) { XLogRecPtr WriteRqstPtr; XLogwrtRqst WriteRqst; @@ -3134,6 +3134,8 @@ XLogFlush(XLogRecPtr record) /* wake up walsenders now that we've released heavily contended locks */ WalSndWakeupProcessRequests(); + SyncRepWaitForLSN(WriteRqstPtr, ForDataFlush, Wait); + /* * If we still haven't flushed to the request point then we have a * problem; most likely, the requested flush point is past end of XLOG. @@ -8230,7 +8232,12 @@ CreateCheckPoint(int flags) XLOG_CHECKPOINT_ONLINE, &rdata); - XLogFlush(recptr); + /* + * At this point, ensure that the synchronous standby has received the + * checkpoint WAL. Otherwise failure after the control file update will + * cause the master to start from a location not known to the standby + */ + XLogFlush(recptr, true, !shutdown); /* * We mustn't write any new WAL after a shutdown checkpoint, or it will be @@ -8387,7 +8394,7 @@ CreateEndOfRecoveryRecord(void) recptr = XLogInsert(RM_XLOG_ID, XLOG_END_OF_RECOVERY, &rdata); - XLogFlush(recptr); + XLogFlush(recptr, true, true); /* * Update the control file so that crash recovery can follow the timeline diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index 971a149..a77471b 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -285,9 +285,13 @@ RelationTruncate(Relation rel, BlockNumber nblocks) * or visibility map. If we crashed during that window, we'd be left * with a truncated heap, but the FSM or visibility map would still * contain entries for the non-existent heap pages. + * + * Also ensure that the WAL is received by the synchronous standby. + * Otherwise, we may have a situation where the heap is truncated, but + * the action never replayed on the standby */ if (fsm || vm) - XLogFlush(lsn); + XLogFlush(lsn, true, true); } /* Do the real work */ @@ -519,7 +523,7 @@ smgr_redo(XLogRecPtr lsn, XLogRecord *record) * after truncation, but that would leave a small window where the * WAL-first rule could be violated. */ - XLogFlush(lsn); + XLogFlush(lsn, true, false); smgrtruncate(reln, MAIN_FORKNUM, xlrec->blkno); diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index 5424281..6a58350 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -65,6 +65,8 @@ char *SyncRepStandbyNames; static bool announce_next_takeover = true; static int SyncRepWaitMode = SYNC_REP_NO_WAIT; +static int SyncTransferMode = SYNC_REP_NO_WAIT; +int synchronous_transfer = SYNCHRONOUS_TRANSFER_COMMIT; static void SyncRepQueueInsert(int mode); static void SyncRepCancelWait(void); @@ -85,25 +87,55 @@ static bool SyncRepQueueIsOrderedByLSN(int mode); * Wait for synchronous replication, if requested by user. * * Initially backends start in state SYNC_REP_NOT_WAITING and then - * change that state to SYNC_REP_WAITING before adding ourselves - * to the wait queue. During SyncRepWakeQueue() a WALSender changes - * the state to SYNC_REP_WAIT_COMPLETE once replication is confirmed. - * This backend then resets its state to SYNC_REP_NOT_WAITING. + * change that state to SYNC_REP_WAITING/SYNC_REP_WAITING_FOR_DATA_FLUSH + * before adding ourselves to the wait queue. During SyncRepWakeQueue() a + * WALSender changes the state to SYNC_REP_WAIT_COMPLETE once replication is + * confirmed. This backend then resets its state to SYNC_REP_NOT_WAITING. + * + * ForDataFlush - if TRUE, we wait for the flushing data page. + * Otherwise wait for the sync standby + * + * Wait - if FALSE, we don't actually wait, but tell the caller whether or not + * the standby has already made progressed upto the given XactCommitLSN + * + * Return TRUE if either the sync standby is not + * configured/turned off OR the standby has made enough progress */ -void -SyncRepWaitForLSN(XLogRecPtr XactCommitLSN) +bool +SyncRepWaitForLSN(XLogRecPtr XactCommitLSN, bool ForDataFlush, bool Wait) { char *new_status = NULL; const char *old_status; - int mode = SyncRepWaitMode; + int mode = !ForDataFlush ? SyncRepWaitMode : SyncTransferMode; + bool ret; /* * Fast exit if user has not requested sync replication, or there are no * sync replication standby names defined. Note that those standbys don't * need to be connected. */ - if (!SyncRepRequested() || !SyncStandbysDefined()) - return; + if ((!SyncRepRequested() || !SyncStandbysDefined()) && + !SyncTransRequested() && !ForDataFlush) + return true; + + /* + * If the caller has specified ForDataFlush, but synchronous transfer + * is not specified or its turned off, exit. + * + * We would like to allow the failback safe mechanism even for cascaded + * standbys as well. But we can't really wait for the standby to catch + * up until we reach a consistent state since the standbys won't be + * even able to connect without us reaching in that state (XXX Confirm) + */ + if ((!SyncTransRequested()) && ForDataFlush) + return true; + + /* + * If the caller has not specified ForDataFlush, but synchronous commit + * is skipped by values of synchronous_transfer, exit. + */ + if (IsSyncRepSkipped() && !ForDataFlush) + return true; Assert(SHMQueueIsDetached(&(MyProc->syncRepLinks))); Assert(WalSndCtl != NULL); @@ -118,12 +150,14 @@ SyncRepWaitForLSN(XLogRecPtr XactCommitLSN) * Also check that the standby hasn't already replied. Unlikely race * condition but we'll be fetching that cache line anyway so its likely to * be a low cost check. + * And if we are told not to block on the standby, exit */ - if (!WalSndCtl->sync_standbys_defined || - XactCommitLSN <= WalSndCtl->lsn[mode]) + if ((!ForDataFlush && !WalSndCtl->sync_standbys_defined) || + XactCommitLSN <= WalSndCtl->lsn[mode] || + !Wait) { LWLockRelease(SyncRepLock); - return; + return true; } /* @@ -150,6 +184,8 @@ SyncRepWaitForLSN(XLogRecPtr XactCommitLSN) new_status[len] = '\0'; /* truncate off " waiting ..." */ } + ret = false; + /* * Wait for specified LSN to be confirmed. * @@ -186,7 +222,10 @@ SyncRepWaitForLSN(XLogRecPtr XactCommitLSN) LWLockRelease(SyncRepLock); } if (syncRepState == SYNC_REP_WAIT_COMPLETE) + { + ret = true; break; + } /* * If a wait for synchronous replication is pending, we can neither @@ -263,6 +302,8 @@ SyncRepWaitForLSN(XLogRecPtr XactCommitLSN) set_ps_display(new_status, false); pfree(new_status); } + + return ret; } /* @@ -370,6 +411,7 @@ SyncRepReleaseWaiters(void) volatile WalSnd *syncWalSnd = NULL; int numwrite = 0; int numflush = 0; + int numdataflush = 0; int priority = 0; int i; @@ -379,11 +421,10 @@ SyncRepReleaseWaiters(void) * up, still running base backup or the current flush position is still * invalid, then leave quickly also. */ - if (MyWalSnd->sync_standby_priority == 0 || - MyWalSnd->state < WALSNDSTATE_STREAMING || + + if (MyWalSnd->state < WALSNDSTATE_STREAMING || XLogRecPtrIsInvalid(MyWalSnd->flush)) return; - /* * We're a potential sync standby. Release waiters if we are the highest * priority standby. If there are multiple standbys with same priorities @@ -399,7 +440,6 @@ SyncRepReleaseWaiters(void) if (walsnd->pid != 0 && walsnd->state == WALSNDSTATE_STREAMING && - walsnd->sync_standby_priority > 0 && (priority == 0 || priority > walsnd->sync_standby_priority) && !XLogRecPtrIsInvalid(walsnd->flush)) @@ -438,12 +478,17 @@ SyncRepReleaseWaiters(void) walsndctl->lsn[SYNC_REP_WAIT_FLUSH] = MyWalSnd->flush; numflush = SyncRepWakeQueue(false, SYNC_REP_WAIT_FLUSH); } + if (walsndctl->lsn[SYNC_REP_WAIT_DATA_FLUSH] < MyWalSnd->flush) + { + walsndctl->lsn[SYNC_REP_WAIT_DATA_FLUSH] = MyWalSnd->flush; + numdataflush = SyncRepWakeQueue(false, SYNC_REP_WAIT_DATA_FLUSH); + } LWLockRelease(SyncRepLock); elog(DEBUG3, "released %d procs up to write %X/%X, %d procs up to flush %X/%X", numwrite, (uint32) (MyWalSnd->write >> 32), (uint32) MyWalSnd->write, - numflush, (uint32) (MyWalSnd->flush >> 32), (uint32) MyWalSnd->flush); + numflush, (uint32) (MyWalSnd->flush >> 32), (uint32) MyWalSnd->flush); /* * If we are managing the highest priority standby, though we weren't @@ -709,3 +754,18 @@ assign_synchronous_commit(int newval, void *extra) break; } } + +void +assign_synchronous_transfer(int newval, void *extra) +{ + switch (newval) + { + case SYNCHRONOUS_TRANSFER_ALL: + case SYNCHRONOUS_TRANSFER_DATA_FLUSH: + SyncTransferMode = SYNC_REP_WAIT_DATA_FLUSH; + break; + default: + SyncTransferMode = SYNC_REP_NO_WAIT; + break; + } +} diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 8079226..fc603d7 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -41,6 +41,7 @@ #include "pg_trace.h" #include "pgstat.h" #include "postmaster/bgwriter.h" +#include "replication/syncrep.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" #include "storage/ipc.h" @@ -1975,7 +1976,7 @@ FlushBuffer(volatile BufferDesc *buf, SMgrRelation reln) * skip the flush if the buffer isn't permanent. */ if (buf->flags & BM_PERMANENT) - XLogFlush(recptr); + XLogFlush(recptr, true, false); /* * Now it's safe to write buffer to disk. Note that no one else should diff --git a/src/backend/utils/cache/relmapper.c b/src/backend/utils/cache/relmapper.c index 2c7d9f3..13eb2df 100644 --- a/src/backend/utils/cache/relmapper.c +++ b/src/backend/utils/cache/relmapper.c @@ -721,7 +721,12 @@ write_relmap_file(bool shared, RelMapFile *newmap, lsn = XLogInsert(RM_RELMAP_ID, XLOG_RELMAP_UPDATE, rdata); /* As always, WAL must hit the disk before the data update does */ - XLogFlush(lsn); + XLogFlush(lsn, true, true); + + /* + * XXX Should we also wait for the failback safe standby to receive the + * WAL ? + */ } errno = 0; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 5aefd1b..69568be 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -381,6 +381,18 @@ static const struct config_enum_entry synchronous_commit_options[] = { }; /* + * Although only "all", "data_flush", and "commit" are documented, we + * accept all the likely variants of "off". + */ +static const struct config_enum_entry synchronous_transfer_options[] = { + {"all", SYNCHRONOUS_TRANSFER_ALL, false}, + {"data_flush", SYNCHRONOUS_TRANSFER_DATA_FLUSH, false}, + {"commit", SYNCHRONOUS_TRANSFER_COMMIT, true}, + {"0", SYNCHRONOUS_TRANSFER_COMMIT, true}, + {NULL, 0, false} +}; + +/* * Options for enum values stored in other modules */ extern const struct config_enum_entry wal_level_options[]; @@ -3277,6 +3289,16 @@ static struct config_enum ConfigureNamesEnum[] = }, { + {"synchronous_transfer", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Sets the data flush synchronization level"), + NULL + }, + &synchronous_transfer, + SYNCHRONOUS_TRANSFER_COMMIT, synchronous_transfer_options, + NULL, assign_synchronous_transfer, NULL + }, + + { {"trace_recovery_messages", PGC_SIGHUP, DEVELOPER_OPTIONS, gettext_noop("Enables logging of recovery-related debugging information."), gettext_noop("Each level includes all the levels that follow it. The later" diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index d69a02b..d6603c2 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -220,6 +220,8 @@ #synchronous_standby_names = '' # standby servers that provide sync rep # comma-separated list of application_name # from standby(s); '*' = all +#synchronous_transfer = commit # data page synchronization level + # commit, data_flush or all #vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed # - Standby Servers - diff --git a/src/backend/utils/time/tqual.c b/src/backend/utils/time/tqual.c index 55563ea..c081ee0 100644 --- a/src/backend/utils/time/tqual.c +++ b/src/backend/utils/time/tqual.c @@ -62,6 +62,7 @@ #include "access/subtrans.h" #include "access/transam.h" #include "access/xact.h" +#include "replication/syncrep.h" #include "storage/bufmgr.h" #include "storage/procarray.h" #include "utils/tqual.h" @@ -118,6 +119,15 @@ SetHintBits(HeapTupleHeader tuple, Buffer buffer, if (XLogNeedsFlush(commitLSN) && BufferIsPermanent(buffer)) return; /* not flushed yet, so don't set hint */ + + /* + * If synchronous_transfer is configured to data_flush or all, we should + * also check if the commit WAL record has made to the standby before + * allowing hint bit updates. We should not wait for the standby to receive + * the WAL since its OK to delay hint bit updates + */ + if (!SyncRepWaitForLSN(commitLSN, true, false)) + return; } tuple->t_infomask |= infomask; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 002862c..68072d2 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -262,7 +262,7 @@ typedef struct CheckpointStatsData extern CheckpointStatsData CheckpointStats; extern XLogRecPtr XLogInsert(RmgrId rmid, uint8 info, XLogRecData *rdata); -extern void XLogFlush(XLogRecPtr RecPtr); +extern void XLogFlush(XLogRecPtr RecPtr, bool ForDataFush, bool Wait); extern bool XLogBackgroundFlush(void); extern bool XLogNeedsFlush(XLogRecPtr RecPtr); extern int XLogFileInit(XLogSegNo segno, bool *use_existent, bool use_lock); diff --git a/src/include/replication/syncrep.h b/src/include/replication/syncrep.h index ac23ea6..4540625 100644 --- a/src/include/replication/syncrep.h +++ b/src/include/replication/syncrep.h @@ -19,23 +19,42 @@ #define SyncRepRequested() \ (max_wal_senders > 0 && synchronous_commit > SYNCHRONOUS_COMMIT_LOCAL_FLUSH) +#define SyncTransRequested() \ + (max_wal_senders > 0 && synchronous_transfer > SYNCHRONOUS_TRANSFER_COMMIT) + +#define IsSyncRepSkipped() \ + (max_wal_senders > 0 && synchronous_transfer == SYNCHRONOUS_TRANSFER_DATA_FLUSH) + /* SyncRepWaitMode */ -#define SYNC_REP_NO_WAIT -1 -#define SYNC_REP_WAIT_WRITE 0 -#define SYNC_REP_WAIT_FLUSH 1 +#define SYNC_REP_NO_WAIT -1 +#define SYNC_REP_WAIT_WRITE 0 +#define SYNC_REP_WAIT_FLUSH 1 +#define SYNC_REP_WAIT_DATA_FLUSH 2 -#define NUM_SYNC_REP_WAIT_MODE 2 +#define NUM_SYNC_REP_WAIT_MODE 3 /* syncRepState */ -#define SYNC_REP_NOT_WAITING 0 -#define SYNC_REP_WAITING 1 -#define SYNC_REP_WAIT_COMPLETE 2 +#define SYNC_REP_NOT_WAITING 0 +#define SYNC_REP_WAITING 1 +#define SYNC_REP_WAIT_COMPLETE 2 + +typedef enum +{ + SYNCHRONOUS_TRANSFER_COMMIT, /* no wait for flush data page */ + SYNCHRONOUS_TRANSFER_DATA_FLUSH, /* wait for data page flush only + * no wait for WAL */ + SYNCHRONOUS_TRANSFER_ALL /* wait for data page flush */ +} SynchronousTransferLevel; /* user-settable parameters for synchronous replication */ extern char *SyncRepStandbyNames; +/* user-settable parameters for failback safe replication */ +extern int synchronous_transfer; + /* called by user backend */ -extern void SyncRepWaitForLSN(XLogRecPtr XactCommitLSN); +extern bool SyncRepWaitForLSN(XLogRecPtr XactCommitLSN, + bool ForDataFlush, bool Wait); /* called at backend exit */ extern void SyncRepCleanupAtProcExit(void); @@ -52,5 +71,6 @@ extern int SyncRepWakeQueue(bool all, int mode); extern bool check_synchronous_standby_names(char **newval, void **extra, GucSource source); extern void assign_synchronous_commit(int newval, void *extra); +extern void assign_synchronous_transfer(int newval, void *extra); #endif /* _SYNCREP_H */