diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c new file mode 100644 index 3ae2848..3e70792 *** a/src/backend/storage/buffer/buf_init.c --- b/src/backend/storage/buffer/buf_init.c *************** InitBufferPool(void) *** 95,106 **** BufferDesc *buf = GetBufferDescriptor(i); CLEAR_BUFFERTAG(buf->tag); - buf->flags = 0; - buf->usage_count = 0; - buf->refcount = 0; - buf->wait_backend_pid = 0; ! SpinLockInit(&buf->buf_hdr_lock); buf->buf_id = i; --- 95,103 ---- BufferDesc *buf = GetBufferDescriptor(i); CLEAR_BUFFERTAG(buf->tag); ! pg_atomic_init_u32(&buf->state, 0); ! buf->wait_backend_pid = 0; buf->buf_id = i; diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c new file mode 100644 index 8c0358e..ec96bc3 *** a/src/backend/storage/buffer/bufmgr.c --- b/src/backend/storage/buffer/bufmgr.c *************** *** 51,56 **** --- 51,58 ---- #include "utils/resowner_private.h" #include "utils/timestamp.h" + #define likely(x) __builtin_expect((x),1) + #define unlikely(x) __builtin_expect((x),0) /* Note: these two macros only work on shared buffers, not local ones! */ #define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ)) *************** ReadBuffer_common(SMgrRelation smgr, cha *** 774,782 **** */ if (isLocalBuf) { /* Only need to adjust flags */ ! Assert(bufHdr->flags & BM_VALID); ! bufHdr->flags &= ~BM_VALID; } else { --- 776,788 ---- */ if (isLocalBuf) { + uint32 state; + + state = pg_atomic_read_u32(&bufHdr->state); /* Only need to adjust flags */ ! Assert(state & BM_VALID); ! state &= ~BM_VALID; ! pg_atomic_write_u32(&bufHdr->state, state); } else { *************** ReadBuffer_common(SMgrRelation smgr, cha *** 788,795 **** do { LockBufHdr(bufHdr); ! Assert(bufHdr->flags & BM_VALID); ! bufHdr->flags &= ~BM_VALID; UnlockBufHdr(bufHdr); } while (!StartBufferIO(bufHdr, true)); } --- 794,801 ---- do { LockBufHdr(bufHdr); ! Assert(pg_atomic_read_u32(&bufHdr->state) & BM_VALID); ! pg_atomic_fetch_and_u32(&bufHdr->state, ~BM_VALID); UnlockBufHdr(bufHdr); } while (!StartBufferIO(bufHdr, true)); } *************** ReadBuffer_common(SMgrRelation smgr, cha *** 807,813 **** * it's not been recycled) but come right back here to try smgrextend * again. */ ! Assert(!(bufHdr->flags & BM_VALID)); /* spinlock not needed */ bufBlock = isLocalBuf ? LocalBufHdrGetBlock(bufHdr) : BufHdrGetBlock(bufHdr); --- 813,819 ---- * it's not been recycled) but come right back here to try smgrextend * again. */ ! Assert(!(pg_atomic_read_u32(&bufHdr->state) & BM_VALID)); /* spinlock not needed */ bufBlock = isLocalBuf ? LocalBufHdrGetBlock(bufHdr) : BufHdrGetBlock(bufHdr); *************** ReadBuffer_common(SMgrRelation smgr, cha *** 885,891 **** if (isLocalBuf) { /* Only need to adjust flags */ ! bufHdr->flags |= BM_VALID; } else { --- 891,897 ---- if (isLocalBuf) { /* Only need to adjust flags */ ! pg_atomic_fetch_or_u32(&bufHdr->state, BM_VALID); } else { *************** BufferAlloc(SMgrRelation smgr, char relp *** 939,945 **** BufferTag oldTag; /* previous identity of selected buffer */ uint32 oldHash; /* hash value for oldTag */ LWLock *oldPartitionLock; /* buffer partition lock for it */ ! BufFlags oldFlags; int buf_id; volatile BufferDesc *buf; bool valid; --- 945,951 ---- BufferTag oldTag; /* previous identity of selected buffer */ uint32 oldHash; /* hash value for oldTag */ LWLock *oldPartitionLock; /* buffer partition lock for it */ ! uint32 oldFlags; int buf_id; volatile BufferDesc *buf; bool valid; *************** BufferAlloc(SMgrRelation smgr, char relp *** 1013,1022 **** */ buf = StrategyGetBuffer(strategy); ! Assert(buf->refcount == 0); /* Must copy buffer flags while we still hold the spinlock */ ! oldFlags = buf->flags; /* Pin the buffer and then release the buffer spinlock */ PinBuffer_Locked(buf); --- 1019,1028 ---- */ buf = StrategyGetBuffer(strategy); ! Assert((pg_atomic_read_u32(&buf->state) & BUF_REFCOUNT_MASK) == 0); /* Must copy buffer flags while we still hold the spinlock */ ! oldFlags = pg_atomic_read_u32(&buf->state) & BUF_FLAG_MASK; /* Pin the buffer and then release the buffer spinlock */ PinBuffer_Locked(buf); *************** BufferAlloc(SMgrRelation smgr, char relp *** 1210,1217 **** * recycle this buffer; we must undo everything we've done and start * over with a new victim buffer. */ ! oldFlags = buf->flags; ! if (buf->refcount == 1 && !(oldFlags & BM_DIRTY)) break; UnlockBufHdr(buf); --- 1216,1224 ---- * recycle this buffer; we must undo everything we've done and start * over with a new victim buffer. */ ! oldFlags = pg_atomic_read_u32(&buf->state) & BUF_FLAG_MASK; ! if ((pg_atomic_read_u32(&buf->state) & BUF_REFCOUNT_MASK) == 1 && ! !(oldFlags & BM_DIRTY)) break; UnlockBufHdr(buf); *************** BufferAlloc(SMgrRelation smgr, char relp *** 1232,1243 **** * 1 so that the buffer can survive one clock-sweep pass.) */ buf->tag = newTag; ! buf->flags &= ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR | BM_PERMANENT); if (relpersistence == RELPERSISTENCE_PERMANENT) ! buf->flags |= BM_TAG_VALID | BM_PERMANENT; else ! buf->flags |= BM_TAG_VALID; ! buf->usage_count = 1; UnlockBufHdr(buf); --- 1239,1257 ---- * 1 so that the buffer can survive one clock-sweep pass.) */ buf->tag = newTag; ! pg_atomic_fetch_and_u32(&buf->state, ! ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED | ! BM_CHECKPOINT_NEEDED | BM_IO_ERROR | ! BM_PERMANENT | ! BUF_USAGECOUNT_MASK)); if (relpersistence == RELPERSISTENCE_PERMANENT) ! pg_atomic_fetch_or_u32(&buf->state, ! BM_TAG_VALID | BM_PERMANENT | ! BUF_USAGECOUNT_ONE); else ! pg_atomic_fetch_or_u32(&buf->state, ! BM_TAG_VALID | ! BUF_USAGECOUNT_ONE); UnlockBufHdr(buf); *************** InvalidateBuffer(volatile BufferDesc *bu *** 1286,1292 **** BufferTag oldTag; uint32 oldHash; /* hash value for oldTag */ LWLock *oldPartitionLock; /* buffer partition lock for it */ ! BufFlags oldFlags; /* Save the original buffer tag before dropping the spinlock */ oldTag = buf->tag; --- 1300,1306 ---- BufferTag oldTag; uint32 oldHash; /* hash value for oldTag */ LWLock *oldPartitionLock; /* buffer partition lock for it */ ! uint32 oldFlags; /* Save the original buffer tag before dropping the spinlock */ oldTag = buf->tag; *************** retry: *** 1329,1335 **** * yet done StartBufferIO, WaitIO will fall through and we'll effectively * be busy-looping here.) */ ! if (buf->refcount != 0) { UnlockBufHdr(buf); LWLockRelease(oldPartitionLock); --- 1343,1349 ---- * yet done StartBufferIO, WaitIO will fall through and we'll effectively * be busy-looping here.) */ ! if ((pg_atomic_read_u32(&buf->state) & BUF_REFCOUNT_MASK) != 0) { UnlockBufHdr(buf); LWLockRelease(oldPartitionLock); *************** retry: *** 1344,1353 **** * Clear out the buffer's tag and flags. We must do this to ensure that * linear scans of the buffer array don't think the buffer is valid. */ ! oldFlags = buf->flags; CLEAR_BUFFERTAG(buf->tag); ! buf->flags = 0; ! buf->usage_count = 0; UnlockBufHdr(buf); --- 1358,1366 ---- * Clear out the buffer's tag and flags. We must do this to ensure that * linear scans of the buffer array don't think the buffer is valid. */ ! oldFlags = pg_atomic_read_u32(&buf->state) & BUF_FLAG_MASK; CLEAR_BUFFERTAG(buf->tag); ! pg_atomic_fetch_and_u32(&buf->state, BM_LOCKED | ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK)); UnlockBufHdr(buf); *************** MarkBufferDirty(Buffer buffer) *** 1399,1410 **** LockBufHdr(bufHdr); ! Assert(bufHdr->refcount > 0); /* * If the buffer was not dirty already, do vacuum accounting. */ ! if (!(bufHdr->flags & BM_DIRTY)) { VacuumPageDirty++; pgBufferUsage.shared_blks_dirtied++; --- 1412,1423 ---- LockBufHdr(bufHdr); ! Assert((pg_atomic_read_u32(&bufHdr->state) & BUF_REFCOUNT_MASK) > 0); /* * If the buffer was not dirty already, do vacuum accounting. */ ! if (!(pg_atomic_read_u32(&bufHdr->state) & BM_DIRTY)) { VacuumPageDirty++; pgBufferUsage.shared_blks_dirtied++; *************** MarkBufferDirty(Buffer buffer) *** 1412,1418 **** VacuumCostBalance += VacuumCostPageDirty; } ! bufHdr->flags |= (BM_DIRTY | BM_JUST_DIRTIED); UnlockBufHdr(bufHdr); } --- 1425,1432 ---- VacuumCostBalance += VacuumCostPageDirty; } ! pg_atomic_fetch_or_u32(&bufHdr->state, ! BM_DIRTY | BM_JUST_DIRTIED); UnlockBufHdr(bufHdr); } *************** PinBuffer(volatile BufferDesc *buf, Buff *** 1495,1517 **** if (ref == NULL) { ReservePrivateRefCountEntry(); ref = NewPrivateRefCountEntry(b); ! LockBufHdr(buf); ! buf->refcount++; ! if (strategy == NULL) ! { ! if (buf->usage_count < BM_MAX_USAGE_COUNT) ! buf->usage_count++; ! } ! else { ! if (buf->usage_count == 0) ! buf->usage_count = 1; } - result = (buf->flags & BM_VALID) != 0; - UnlockBufHdr(buf); } else { --- 1509,1548 ---- if (ref == NULL) { + uint32 state; + uint32 oldstate; + ReservePrivateRefCountEntry(); ref = NewPrivateRefCountEntry(b); ! state = pg_atomic_read_u32(&buf->state); ! oldstate = state; ! ! while (true) { ! /* spin-wait till lock is free */ ! while (unlikely(state & BM_LOCKED)) ! { ! pg_spin_delay(); ! state = pg_atomic_read_u32(&buf->state); ! oldstate = state; ! } ! ! /* increase refcount */ ! state += 1; ! ! /* increase usagecount unless already max */ ! if (((state & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) != BM_MAX_USAGE_COUNT) ! state += BUF_USAGECOUNT_ONE; ! ! result = (state & BM_VALID) != 0; ! ! if (likely(pg_atomic_compare_exchange_u32(&buf->state, &oldstate, state))) ! break; ! ! /* get ready for next loop, oldstate has been updated by cas */ ! state = oldstate; } } else { *************** PinBuffer_Locked(volatile BufferDesc *bu *** 1558,1564 **** */ Assert(GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf), false) == NULL); ! buf->refcount++; UnlockBufHdr(buf); b = BufferDescriptorGetBuffer(buf); --- 1589,1595 ---- */ Assert(GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf), false) == NULL); ! pg_atomic_fetch_add_u32(&buf->state, 1); UnlockBufHdr(buf); b = BufferDescriptorGetBuffer(buf); *************** UnpinBuffer(volatile BufferDesc *buf, bo *** 1594,1623 **** ref->refcount--; if (ref->refcount == 0) { /* I'd better not still hold any locks on the buffer */ Assert(!LWLockHeldByMe(buf->content_lock)); Assert(!LWLockHeldByMe(buf->io_in_progress_lock)); ! LockBufHdr(buf); ! ! /* Decrement the shared reference count */ ! Assert(buf->refcount > 0); ! buf->refcount--; /* Support LockBufferForCleanup() */ ! if ((buf->flags & BM_PIN_COUNT_WAITER) && ! buf->refcount == 1) { ! /* we just released the last pin other than the waiter's */ ! int wait_backend_pid = buf->wait_backend_pid; ! buf->flags &= ~BM_PIN_COUNT_WAITER; ! UnlockBufHdr(buf); ! ProcSendSignal(wait_backend_pid); ! } ! else ! UnlockBufHdr(buf); ForgetPrivateRefCountEntry(ref); } } --- 1625,1665 ---- ref->refcount--; if (ref->refcount == 0) { + uint32 state; + /* I'd better not still hold any locks on the buffer */ Assert(!LWLockHeldByMe(buf->content_lock)); Assert(!LWLockHeldByMe(buf->io_in_progress_lock)); ! /* ! * Decrement the shared reference count. ! * ! * Arguably it'd be more robust if we checked for BM_LOCKED here, but ! * currently all manipulation of ->state for shared buffers is through ! * atomics. ! */ ! state = pg_atomic_fetch_sub_u32(&buf->state, 1); ! Assert((state & BUF_REFCOUNT_MASK) > 0); /* Support LockBufferForCleanup() */ ! if (state & BM_PIN_COUNT_WAITER) { ! LockBufHdr(buf); ! if (pg_atomic_read_u32(&buf->state) & BM_PIN_COUNT_WAITER && ! (pg_atomic_read_u32(&buf->state) & BUF_REFCOUNT_MASK) == 1) ! { ! /* we just released the last pin other than the waiter's */ ! int wait_backend_pid = buf->wait_backend_pid; + pg_atomic_fetch_and_u32(&buf->state, + ~BM_PIN_COUNT_WAITER); + UnlockBufHdr(buf); + ProcSendSignal(wait_backend_pid); + } + else + UnlockBufHdr(buf); + } ForgetPrivateRefCountEntry(ref); } } *************** BufferSync(int flags) *** 1680,1688 **** */ LockBufHdr(bufHdr); ! if ((bufHdr->flags & mask) == mask) { ! bufHdr->flags |= BM_CHECKPOINT_NEEDED; num_to_write++; } --- 1722,1731 ---- */ LockBufHdr(bufHdr); ! if ((pg_atomic_read_u32(&bufHdr->state) & mask) == mask) { ! pg_atomic_fetch_or_u32(&bufHdr->state, ! BM_CHECKPOINT_NEEDED); num_to_write++; } *************** BufferSync(int flags) *** 1721,1727 **** * write the buffer though we didn't need to. It doesn't seem worth * guarding against this, though. */ ! if (bufHdr->flags & BM_CHECKPOINT_NEEDED) { if (SyncOneBuffer(buf_id, false) & BUF_WRITTEN) { --- 1764,1770 ---- * write the buffer though we didn't need to. It doesn't seem worth * guarding against this, though. */ ! if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED) { if (SyncOneBuffer(buf_id, false) & BUF_WRITTEN) { *************** SyncOneBuffer(int buf_id, bool skip_rece *** 2081,2086 **** --- 2124,2130 ---- { volatile BufferDesc *bufHdr = GetBufferDescriptor(buf_id); int result = 0; + uint32 state; ReservePrivateRefCountEntry(); *************** SyncOneBuffer(int buf_id, bool skip_rece *** 2095,2101 **** */ LockBufHdr(bufHdr); ! if (bufHdr->refcount == 0 && bufHdr->usage_count == 0) result |= BUF_REUSABLE; else if (skip_recently_used) { --- 2139,2148 ---- */ LockBufHdr(bufHdr); ! state = pg_atomic_read_u32(&bufHdr->state); ! ! if ((state & BUF_REFCOUNT_MASK) == 0 && ! (state & BUF_USAGECOUNT_MASK) == 0) result |= BUF_REUSABLE; else if (skip_recently_used) { *************** SyncOneBuffer(int buf_id, bool skip_rece *** 2104,2110 **** return result; } ! if (!(bufHdr->flags & BM_VALID) || !(bufHdr->flags & BM_DIRTY)) { /* It's clean, so nothing to do */ UnlockBufHdr(bufHdr); --- 2151,2157 ---- return result; } ! if (!(state & BM_VALID) || !(state & BM_DIRTY)) { /* It's clean, so nothing to do */ UnlockBufHdr(bufHdr); *************** PrintBufferLeakWarning(Buffer buffer) *** 2256,2261 **** --- 2303,2309 ---- int32 loccount; char *path; BackendId backend; + uint32 state; Assert(BufferIsValid(buffer)); if (BufferIsLocal(buffer)) *************** PrintBufferLeakWarning(Buffer buffer) *** 2273,2284 **** /* theoretically we should lock the bufhdr here */ path = relpathbackend(buf->tag.rnode, backend, buf->tag.forkNum); elog(WARNING, "buffer refcount leak: [%03d] " "(rel=%s, blockNum=%u, flags=0x%x, refcount=%u %d)", buffer, path, ! buf->tag.blockNum, buf->flags, ! buf->refcount, loccount); pfree(path); } --- 2321,2333 ---- /* theoretically we should lock the bufhdr here */ path = relpathbackend(buf->tag.rnode, backend, buf->tag.forkNum); + state = pg_atomic_read_u32(&buf->state); elog(WARNING, "buffer refcount leak: [%03d] " "(rel=%s, blockNum=%u, flags=0x%x, refcount=%u %d)", buffer, path, ! buf->tag.blockNum, state & BUF_FLAG_MASK, ! state & BUF_REFCOUNT_MASK, loccount); pfree(path); } *************** FlushBuffer(volatile BufferDesc *buf, SM *** 2424,2430 **** recptr = BufferGetLSN(buf); /* To check if block content changes while flushing. - vadim 01/17/97 */ ! buf->flags &= ~BM_JUST_DIRTIED; UnlockBufHdr(buf); /* --- 2473,2479 ---- recptr = BufferGetLSN(buf); /* To check if block content changes while flushing. - vadim 01/17/97 */ ! pg_atomic_fetch_and_u32(&buf->state, ~BM_JUST_DIRTIED); UnlockBufHdr(buf); /* *************** FlushBuffer(volatile BufferDesc *buf, SM *** 2444,2450 **** * disastrous system-wide consequences. To make sure that can't happen, * skip the flush if the buffer isn't permanent. */ ! if (buf->flags & BM_PERMANENT) XLogFlush(recptr); /* --- 2493,2499 ---- * disastrous system-wide consequences. To make sure that can't happen, * skip the flush if the buffer isn't permanent. */ ! if (pg_atomic_read_u32(&buf->state) & BM_PERMANENT) XLogFlush(recptr); /* *************** BufferIsPermanent(Buffer buffer) *** 2538,2544 **** * old value or the new value, but not random garbage. */ bufHdr = GetBufferDescriptor(buffer - 1); ! return (bufHdr->flags & BM_PERMANENT) != 0; } /* --- 2587,2593 ---- * old value or the new value, but not random garbage. */ bufHdr = GetBufferDescriptor(buffer - 1); ! return (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT) != 0; } /* *************** FlushRelationBuffers(Relation rel) *** 2874,2880 **** { bufHdr = GetLocalBufferDescriptor(i); if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) && ! (bufHdr->flags & BM_VALID) && (bufHdr->flags & BM_DIRTY)) { ErrorContextCallback errcallback; Page localpage; --- 2923,2930 ---- { bufHdr = GetLocalBufferDescriptor(i); if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) && ! (pg_atomic_read_u32(&bufHdr->state) & (BM_VALID | BM_DIRTY)) ! == (BM_VALID | BM_DIRTY)) { ErrorContextCallback errcallback; Page localpage; *************** FlushRelationBuffers(Relation rel) *** 2895,2901 **** localpage, false); ! bufHdr->flags &= ~(BM_DIRTY | BM_JUST_DIRTIED); /* Pop the error context stack */ error_context_stack = errcallback.previous; --- 2945,2951 ---- localpage, false); ! pg_atomic_fetch_and_u32(&bufHdr->state, ~(BM_DIRTY | BM_JUST_DIRTIED)); /* Pop the error context stack */ error_context_stack = errcallback.previous; *************** FlushRelationBuffers(Relation rel) *** 2923,2929 **** LockBufHdr(bufHdr); if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) && ! (bufHdr->flags & BM_VALID) && (bufHdr->flags & BM_DIRTY)) { PinBuffer_Locked(bufHdr); LWLockAcquire(bufHdr->content_lock, LW_SHARED); --- 2973,2980 ---- LockBufHdr(bufHdr); if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) && ! (pg_atomic_read_u32(&bufHdr->state) & (BM_VALID | BM_DIRTY)) ! == (BM_VALID | BM_DIRTY)) { PinBuffer_Locked(bufHdr); LWLockAcquire(bufHdr->content_lock, LW_SHARED); *************** FlushDatabaseBuffers(Oid dbid) *** 2975,2981 **** LockBufHdr(bufHdr); if (bufHdr->tag.rnode.dbNode == dbid && ! (bufHdr->flags & BM_VALID) && (bufHdr->flags & BM_DIRTY)) { PinBuffer_Locked(bufHdr); LWLockAcquire(bufHdr->content_lock, LW_SHARED); --- 3026,3033 ---- LockBufHdr(bufHdr); if (bufHdr->tag.rnode.dbNode == dbid && ! (pg_atomic_read_u32(&bufHdr->state) & (BM_VALID | BM_DIRTY)) ! == (BM_VALID | BM_DIRTY)) { PinBuffer_Locked(bufHdr); LWLockAcquire(bufHdr->content_lock, LW_SHARED); *************** MarkBufferDirtyHint(Buffer buffer, bool *** 3093,3104 **** * is only intended to be used in cases where failing to write out the * data would be harmless anyway, it doesn't really matter. */ ! if ((bufHdr->flags & (BM_DIRTY | BM_JUST_DIRTIED)) != (BM_DIRTY | BM_JUST_DIRTIED)) { XLogRecPtr lsn = InvalidXLogRecPtr; bool dirtied = false; bool delayChkpt = false; /* * If we need to protect hint bit updates from torn writes, WAL-log a --- 3145,3157 ---- * is only intended to be used in cases where failing to write out the * data would be harmless anyway, it doesn't really matter. */ ! if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) != (BM_DIRTY | BM_JUST_DIRTIED)) { XLogRecPtr lsn = InvalidXLogRecPtr; bool dirtied = false; bool delayChkpt = false; + uint32 state; /* * If we need to protect hint bit updates from torn writes, WAL-log a *************** MarkBufferDirtyHint(Buffer buffer, bool *** 3109,3115 **** * We don't check full_page_writes here because that logic is included * when we call XLogInsert() since the value changes dynamically. */ ! if (XLogHintBitIsNeeded() && (bufHdr->flags & BM_PERMANENT)) { /* * If we're in recovery we cannot dirty a page because of a hint. --- 3162,3168 ---- * We don't check full_page_writes here because that logic is included * when we call XLogInsert() since the value changes dynamically. */ ! if (XLogHintBitIsNeeded() && (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT)) { /* * If we're in recovery we cannot dirty a page because of a hint. *************** MarkBufferDirtyHint(Buffer buffer, bool *** 3149,3156 **** } LockBufHdr(bufHdr); ! Assert(bufHdr->refcount > 0); ! if (!(bufHdr->flags & BM_DIRTY)) { dirtied = true; /* Means "will be dirtied by this action" */ --- 3202,3213 ---- } LockBufHdr(bufHdr); ! ! state = pg_atomic_read_u32(&bufHdr->state); ! ! Assert((state & BUF_REFCOUNT_MASK) > 0); ! ! if (!(state & BM_DIRTY)) { dirtied = true; /* Means "will be dirtied by this action" */ *************** MarkBufferDirtyHint(Buffer buffer, bool *** 3170,3176 **** if (!XLogRecPtrIsInvalid(lsn)) PageSetLSN(page, lsn); } ! bufHdr->flags |= (BM_DIRTY | BM_JUST_DIRTIED); UnlockBufHdr(bufHdr); if (delayChkpt) --- 3227,3235 ---- if (!XLogRecPtrIsInvalid(lsn)) PageSetLSN(page, lsn); } ! ! pg_atomic_fetch_or_u32(&bufHdr->state, BM_DIRTY | BM_JUST_DIRTIED); ! UnlockBufHdr(bufHdr); if (delayChkpt) *************** UnlockBuffers(void) *** 3208,3216 **** * Don't complain if flag bit not set; it could have been reset but we * got a cancel/die interrupt before getting the signal. */ ! if ((buf->flags & BM_PIN_COUNT_WAITER) != 0 && buf->wait_backend_pid == MyProcPid) ! buf->flags &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(buf); --- 3267,3275 ---- * Don't complain if flag bit not set; it could have been reset but we * got a cancel/die interrupt before getting the signal. */ ! if ((pg_atomic_read_u32(&buf->state) & BM_PIN_COUNT_WAITER) != 0 && buf->wait_backend_pid == MyProcPid) ! pg_atomic_fetch_and_u32(&buf->state, ~BM_PIN_COUNT_WAITER); UnlockBufHdr(buf); *************** LockBufferForCleanup(Buffer buffer) *** 3304,3328 **** for (;;) { /* Try to acquire lock */ LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); LockBufHdr(bufHdr); ! Assert(bufHdr->refcount > 0); ! if (bufHdr->refcount == 1) { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr); return; } /* Failed, so mark myself as waiting for pincount 1 */ ! if (bufHdr->flags & BM_PIN_COUNT_WAITER) { UnlockBufHdr(bufHdr); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); elog(ERROR, "multiple backends attempting to wait for pincount 1"); } bufHdr->wait_backend_pid = MyProcPid; ! bufHdr->flags |= BM_PIN_COUNT_WAITER; PinCountWaitBuf = bufHdr; UnlockBufHdr(bufHdr); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); --- 3363,3392 ---- for (;;) { + int state; + /* Try to acquire lock */ LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); LockBufHdr(bufHdr); ! ! state = pg_atomic_read_u32(&bufHdr->state); ! ! Assert((state & BUF_REFCOUNT_MASK) > 0); ! if ((state & BUF_REFCOUNT_MASK) == 1) { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr); return; } /* Failed, so mark myself as waiting for pincount 1 */ ! if (state & BM_PIN_COUNT_WAITER) { UnlockBufHdr(bufHdr); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); elog(ERROR, "multiple backends attempting to wait for pincount 1"); } bufHdr->wait_backend_pid = MyProcPid; ! pg_atomic_fetch_or_u32(&bufHdr->state, BM_PIN_COUNT_WAITER); PinCountWaitBuf = bufHdr; UnlockBufHdr(bufHdr); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); *************** LockBufferForCleanup(Buffer buffer) *** 3349,3357 **** * better be safe. */ LockBufHdr(bufHdr); ! if ((bufHdr->flags & BM_PIN_COUNT_WAITER) != 0 && bufHdr->wait_backend_pid == MyProcPid) ! bufHdr->flags &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(bufHdr); PinCountWaitBuf = NULL; --- 3413,3421 ---- * better be safe. */ LockBufHdr(bufHdr); ! if ((pg_atomic_read_u32(&bufHdr->state) & BM_PIN_COUNT_WAITER) != 0 && bufHdr->wait_backend_pid == MyProcPid) ! pg_atomic_fetch_and_u32(&bufHdr->state, ~BM_PIN_COUNT_WAITER); UnlockBufHdr(bufHdr); PinCountWaitBuf = NULL; *************** bool *** 3393,3414 **** ConditionalLockBufferForCleanup(Buffer buffer) { volatile BufferDesc *bufHdr; Assert(BufferIsValid(buffer)); if (BufferIsLocal(buffer)) { /* There should be exactly one pin */ ! Assert(LocalRefCount[-buffer - 1] > 0); ! if (LocalRefCount[-buffer - 1] != 1) return false; /* Nobody else to wait for */ return true; } /* There should be exactly one local pin */ ! Assert(GetPrivateRefCount(buffer) > 0); ! if (GetPrivateRefCount(buffer) != 1) return false; /* Try to acquire lock */ --- 3457,3481 ---- ConditionalLockBufferForCleanup(Buffer buffer) { volatile BufferDesc *bufHdr; + uint32 refcount; Assert(BufferIsValid(buffer)); if (BufferIsLocal(buffer)) { + refcount = LocalRefCount[-buffer - 1]; /* There should be exactly one pin */ ! Assert(refcount > 0); ! if (refcount != 1) return false; /* Nobody else to wait for */ return true; } /* There should be exactly one local pin */ ! refcount = GetPrivateRefCount(buffer); ! Assert(refcount); ! if (refcount != 1) return false; /* Try to acquire lock */ *************** ConditionalLockBufferForCleanup(Buffer b *** 3417,3424 **** bufHdr = GetBufferDescriptor(buffer - 1); LockBufHdr(bufHdr); ! Assert(bufHdr->refcount > 0); ! if (bufHdr->refcount == 1) { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr); --- 3484,3493 ---- bufHdr = GetBufferDescriptor(buffer - 1); LockBufHdr(bufHdr); ! ! refcount = pg_atomic_read_u32(&bufHdr->state) & BUF_REFCOUNT_MASK; ! Assert(refcount > 0); ! if (refcount == 1) { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr); *************** WaitIO(volatile BufferDesc *buf) *** 3456,3462 **** */ for (;;) { ! BufFlags sv_flags; /* * It may not be necessary to acquire the spinlock to check the flag --- 3525,3531 ---- */ for (;;) { ! uint32 state; /* * It may not be necessary to acquire the spinlock to check the flag *************** WaitIO(volatile BufferDesc *buf) *** 3464,3472 **** * play it safe. */ LockBufHdr(buf); ! sv_flags = buf->flags; UnlockBufHdr(buf); ! if (!(sv_flags & BM_IO_IN_PROGRESS)) break; LWLockAcquire(buf->io_in_progress_lock, LW_SHARED); LWLockRelease(buf->io_in_progress_lock); --- 3533,3542 ---- * play it safe. */ LockBufHdr(buf); ! state = pg_atomic_read_u32(&buf->state); UnlockBufHdr(buf); ! ! if (!(state & BM_IO_IN_PROGRESS)) break; LWLockAcquire(buf->io_in_progress_lock, LW_SHARED); LWLockRelease(buf->io_in_progress_lock); *************** WaitIO(volatile BufferDesc *buf) *** 3494,3499 **** --- 3564,3571 ---- static bool StartBufferIO(volatile BufferDesc *buf, bool forInput) { + uint32 state; + Assert(!InProgressBuf); for (;;) *************** StartBufferIO(volatile BufferDesc *buf, *** 3506,3512 **** LockBufHdr(buf); ! if (!(buf->flags & BM_IO_IN_PROGRESS)) break; /* --- 3578,3586 ---- LockBufHdr(buf); ! state = pg_atomic_read_u32(&buf->state); ! ! if (!(state & BM_IO_IN_PROGRESS)) break; /* *************** StartBufferIO(volatile BufferDesc *buf, *** 3522,3528 **** /* Once we get here, there is definitely no I/O active on this buffer */ ! if (forInput ? (buf->flags & BM_VALID) : !(buf->flags & BM_DIRTY)) { /* someone else already did the I/O */ UnlockBufHdr(buf); --- 3596,3602 ---- /* Once we get here, there is definitely no I/O active on this buffer */ ! if (forInput ? (state & BM_VALID) : !(state & BM_DIRTY)) { /* someone else already did the I/O */ UnlockBufHdr(buf); *************** StartBufferIO(volatile BufferDesc *buf, *** 3530,3536 **** return false; } ! buf->flags |= BM_IO_IN_PROGRESS; UnlockBufHdr(buf); --- 3604,3610 ---- return false; } ! pg_atomic_fetch_or_u32(&buf->state, BM_IO_IN_PROGRESS); UnlockBufHdr(buf); *************** TerminateBufferIO(volatile BufferDesc *b *** 3565,3575 **** LockBufHdr(buf); ! Assert(buf->flags & BM_IO_IN_PROGRESS); ! buf->flags &= ~(BM_IO_IN_PROGRESS | BM_IO_ERROR); ! if (clear_dirty && !(buf->flags & BM_JUST_DIRTIED)) ! buf->flags &= ~(BM_DIRTY | BM_CHECKPOINT_NEEDED); ! buf->flags |= set_flag_bits; UnlockBufHdr(buf); --- 3639,3651 ---- LockBufHdr(buf); ! Assert(pg_atomic_read_u32(&buf->state) & BM_IO_IN_PROGRESS); ! ! pg_atomic_fetch_and_u32(&buf->state, ~(BM_IO_IN_PROGRESS | BM_IO_ERROR)); ! if (clear_dirty && !(pg_atomic_read_u32(&buf->state) & BM_JUST_DIRTIED)) ! pg_atomic_fetch_and_u32(&buf->state, ~(BM_DIRTY | BM_CHECKPOINT_NEEDED)); ! ! pg_atomic_fetch_or_u32(&buf->state, set_flag_bits); UnlockBufHdr(buf); *************** AbortBufferIO(void) *** 3603,3625 **** LWLockAcquire(buf->io_in_progress_lock, LW_EXCLUSIVE); LockBufHdr(buf); ! Assert(buf->flags & BM_IO_IN_PROGRESS); if (IsForInput) { ! Assert(!(buf->flags & BM_DIRTY)); /* We'd better not think buffer is valid yet */ ! Assert(!(buf->flags & BM_VALID)); UnlockBufHdr(buf); } else { ! BufFlags sv_flags; ! sv_flags = buf->flags; ! Assert(sv_flags & BM_DIRTY); UnlockBufHdr(buf); /* Issue notice if this is not the first failure... */ ! if (sv_flags & BM_IO_ERROR) { /* Buffer is pinned, so we can read tag without spinlock */ char *path; --- 3679,3702 ---- LWLockAcquire(buf->io_in_progress_lock, LW_EXCLUSIVE); LockBufHdr(buf); ! Assert(pg_atomic_read_u32(&buf->state) & BM_IO_IN_PROGRESS); if (IsForInput) { ! Assert(!(pg_atomic_read_u32(&buf->state) & BM_DIRTY)); ! /* We'd better not think buffer is valid yet */ ! Assert(!(pg_atomic_read_u32(&buf->state) & BM_VALID)); UnlockBufHdr(buf); } else { ! uint32 state; ! state = pg_atomic_read_u32(&buf->state); ! Assert(state & BM_DIRTY); UnlockBufHdr(buf); /* Issue notice if this is not the first failure... */ ! if (state & BM_IO_ERROR) { /* Buffer is pinned, so we can read tag without spinlock */ char *path; *************** rnode_comparator(const void *p1, const v *** 3701,3703 **** --- 3778,3810 ---- else return 0; } + + void + LockBufHdr(volatile BufferDesc *desc) + { + uint32 state = pg_atomic_read_u32(&desc->state); + + for (;;) + { + /* wait till lock is free */ + while (unlikely(state & BM_LOCKED)) + { + pg_spin_delay(); + state = pg_atomic_read_u32(&desc->state); + + /* Add exponential backoff? Should seldomly be contended tho. */ + } + + /* and try to get lock */ + if (pg_atomic_compare_exchange_u32(&desc->state, &state, state | BM_LOCKED)) + break; + } + } + + void + UnlockBufHdr(volatile BufferDesc *desc) + { + Assert(pg_atomic_read_u32(&desc->state) & BM_LOCKED); + + pg_atomic_fetch_sub_u32(&desc->state, BM_LOCKED); + } diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c new file mode 100644 index bc2c773..3f2227b *** a/src/backend/storage/buffer/freelist.c --- b/src/backend/storage/buffer/freelist.c *************** StrategyGetBuffer(BufferAccessStrategy s *** 250,255 **** --- 250,257 ---- { while (true) { + uint32 state; + /* Acquire the spinlock to remove element from the freelist */ SpinLockAcquire(&StrategyControl->buffer_strategy_lock); *************** StrategyGetBuffer(BufferAccessStrategy s *** 280,286 **** * of 8.3, but we'd better check anyway.) */ LockBufHdr(buf); ! if (buf->refcount == 0 && buf->usage_count == 0) { if (strategy != NULL) AddBufferToRing(strategy, buf); --- 282,290 ---- * of 8.3, but we'd better check anyway.) */ LockBufHdr(buf); ! state = pg_atomic_read_u32(&buf->state); ! if ((state & BUF_REFCOUNT_MASK) == 0 ! && (state & BUF_USAGECOUNT_MASK) == 0) { if (strategy != NULL) AddBufferToRing(strategy, buf); *************** StrategyGetBuffer(BufferAccessStrategy s *** 295,300 **** --- 299,305 ---- trycounter = NBuffers; for (;;) { + uint32 state; buf = GetBufferDescriptor(ClockSweepTick()); *************** StrategyGetBuffer(BufferAccessStrategy s *** 303,313 **** * it; decrement the usage_count (unless pinned) and keep scanning. */ LockBufHdr(buf); ! if (buf->refcount == 0) { ! if (buf->usage_count > 0) { ! buf->usage_count--; trycounter = NBuffers; } else --- 308,322 ---- * it; decrement the usage_count (unless pinned) and keep scanning. */ LockBufHdr(buf); ! ! state = pg_atomic_read_u32(&buf->state); ! ! if ((state & BUF_REFCOUNT_MASK) == 0) { ! if ((state & BUF_USAGECOUNT_MASK) != 0) { ! pg_atomic_fetch_sub_u32(&buf->state, BUF_USAGECOUNT_ONE); ! trycounter = NBuffers; } else *************** GetBufferFromRing(BufferAccessStrategy s *** 589,594 **** --- 598,605 ---- { volatile BufferDesc *buf; Buffer bufnum; + uint32 state; + uint32 usagecount; /* Advance to next ring slot */ if (++strategy->current >= strategy->ring_size) *************** GetBufferFromRing(BufferAccessStrategy s *** 617,623 **** */ buf = GetBufferDescriptor(bufnum - 1); LockBufHdr(buf); ! if (buf->refcount == 0 && buf->usage_count <= 1) { strategy->current_was_in_ring = true; return buf; --- 628,637 ---- */ buf = GetBufferDescriptor(bufnum - 1); LockBufHdr(buf); ! state = pg_atomic_read_u32(&buf->state); ! usagecount = (state & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT; ! if ((state & BUF_REFCOUNT_MASK) == 0 ! && usagecount <= 1) { strategy->current_was_in_ring = true; return buf; diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c new file mode 100644 index 3144afe..1e11d71 *** a/src/backend/storage/buffer/localbuf.c --- b/src/backend/storage/buffer/localbuf.c *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 108,113 **** --- 108,114 ---- int b; int trycounter; bool found; + uint32 state; INIT_BUFFERTAG(newTag, smgr->smgr_rnode.node, forkNum, blockNum); *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 128,143 **** fprintf(stderr, "LB ALLOC (%u,%d,%d) %d\n", smgr->smgr_rnode.node.relNode, forkNum, blockNum, -b - 1); #endif /* this part is equivalent to PinBuffer for a shared buffer */ if (LocalRefCount[b] == 0) { ! if (bufHdr->usage_count < BM_MAX_USAGE_COUNT) ! bufHdr->usage_count++; } LocalRefCount[b]++; ResourceOwnerRememberBuffer(CurrentResourceOwner, BufferDescriptorGetBuffer(bufHdr)); ! if (bufHdr->flags & BM_VALID) *foundPtr = TRUE; else { --- 129,153 ---- fprintf(stderr, "LB ALLOC (%u,%d,%d) %d\n", smgr->smgr_rnode.node.relNode, forkNum, blockNum, -b - 1); #endif + state = pg_atomic_read_u32(&bufHdr->state); + /* this part is equivalent to PinBuffer for a shared buffer */ if (LocalRefCount[b] == 0) { ! int usagecount; ! ! usagecount = (state & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT; ! ! if (usagecount < BM_MAX_USAGE_COUNT) ! { ! state += BUF_USAGECOUNT_ONE; ! pg_atomic_write_u32(&bufHdr->state, state); ! } } LocalRefCount[b]++; ResourceOwnerRememberBuffer(CurrentResourceOwner, BufferDescriptorGetBuffer(bufHdr)); ! if (state & BM_VALID) *foundPtr = TRUE; else { *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 169,177 **** if (LocalRefCount[b] == 0) { ! if (bufHdr->usage_count > 0) { ! bufHdr->usage_count--; trycounter = NLocBuffer; } else --- 179,193 ---- if (LocalRefCount[b] == 0) { ! int usagecount; ! ! state = pg_atomic_read_u32(&bufHdr->state); ! usagecount = (state & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT; ! ! if (usagecount > 0) { ! state -= BUF_USAGECOUNT_ONE; ! pg_atomic_write_u32(&bufHdr->state, state); trycounter = NLocBuffer; } else *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 193,199 **** * this buffer is not referenced but it might still be dirty. if that's * the case, write it out before reusing it! */ ! if (bufHdr->flags & BM_DIRTY) { SMgrRelation oreln; Page localpage = (char *) LocalBufHdrGetBlock(bufHdr); --- 209,215 ---- * this buffer is not referenced but it might still be dirty. if that's * the case, write it out before reusing it! */ ! if (state & BM_DIRTY) { SMgrRelation oreln; Page localpage = (char *) LocalBufHdrGetBlock(bufHdr); *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 211,217 **** false); /* Mark not-dirty now in case we error out below */ ! bufHdr->flags &= ~BM_DIRTY; pgBufferUsage.local_blks_written++; } --- 227,234 ---- false); /* Mark not-dirty now in case we error out below */ ! state &= ~BM_DIRTY; ! pg_atomic_write_u32(&bufHdr->state, state); pgBufferUsage.local_blks_written++; } *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 228,234 **** /* * Update the hash table: remove old entry, if any, and make new one. */ ! if (bufHdr->flags & BM_TAG_VALID) { hresult = (LocalBufferLookupEnt *) hash_search(LocalBufHash, (void *) &bufHdr->tag, --- 245,251 ---- /* * Update the hash table: remove old entry, if any, and make new one. */ ! if (state & BM_TAG_VALID) { hresult = (LocalBufferLookupEnt *) hash_search(LocalBufHash, (void *) &bufHdr->tag, *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 237,243 **** elog(ERROR, "local buffer hash table corrupted"); /* mark buffer invalid just in case hash insert fails */ CLEAR_BUFFERTAG(bufHdr->tag); ! bufHdr->flags &= ~(BM_VALID | BM_TAG_VALID); } hresult = (LocalBufferLookupEnt *) --- 254,261 ---- elog(ERROR, "local buffer hash table corrupted"); /* mark buffer invalid just in case hash insert fails */ CLEAR_BUFFERTAG(bufHdr->tag); ! state &= ~(BM_VALID | BM_TAG_VALID); ! pg_atomic_write_u32(&bufHdr->state, state); } hresult = (LocalBufferLookupEnt *) *************** LocalBufferAlloc(SMgrRelation smgr, Fork *** 250,258 **** * it's all ours now. */ bufHdr->tag = newTag; ! bufHdr->flags &= ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED | BM_IO_ERROR); ! bufHdr->flags |= BM_TAG_VALID; ! bufHdr->usage_count = 1; *foundPtr = FALSE; return bufHdr; --- 268,278 ---- * it's all ours now. */ bufHdr->tag = newTag; ! state &= ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED | BM_IO_ERROR); ! state |= BM_TAG_VALID; ! state &= ~BUF_USAGECOUNT_MASK; ! state += BUF_USAGECOUNT_ONE; ! pg_atomic_write_u32(&bufHdr->state, state); *foundPtr = FALSE; return bufHdr; *************** MarkLocalBufferDirty(Buffer buffer) *** 267,272 **** --- 287,293 ---- { int bufid; BufferDesc *bufHdr; + uint32 state; Assert(BufferIsLocal(buffer)); *************** MarkLocalBufferDirty(Buffer buffer) *** 280,289 **** bufHdr = GetLocalBufferDescriptor(bufid); ! if (!(bufHdr->flags & BM_DIRTY)) pgBufferUsage.local_blks_dirtied++; ! bufHdr->flags |= BM_DIRTY; } /* --- 301,313 ---- bufHdr = GetLocalBufferDescriptor(bufid); ! state = pg_atomic_read_u32(&bufHdr->state); ! ! if (!(state & BM_DIRTY)) pgBufferUsage.local_blks_dirtied++; ! state |= BM_DIRTY; ! pg_atomic_write_u32(&bufHdr->state, state); } /* *************** DropRelFileNodeLocalBuffers(RelFileNode *** 307,314 **** { BufferDesc *bufHdr = GetLocalBufferDescriptor(i); LocalBufferLookupEnt *hresult; ! if ((bufHdr->flags & BM_TAG_VALID) && RelFileNodeEquals(bufHdr->tag.rnode, rnode) && bufHdr->tag.forkNum == forkNum && bufHdr->tag.blockNum >= firstDelBlock) --- 331,341 ---- { BufferDesc *bufHdr = GetLocalBufferDescriptor(i); LocalBufferLookupEnt *hresult; + uint32 state; ! state = pg_atomic_read_u32(&bufHdr->state); ! ! if ((state & BM_TAG_VALID) && RelFileNodeEquals(bufHdr->tag.rnode, rnode) && bufHdr->tag.forkNum == forkNum && bufHdr->tag.blockNum >= firstDelBlock) *************** DropRelFileNodeLocalBuffers(RelFileNode *** 327,334 **** elog(ERROR, "local buffer hash table corrupted"); /* Mark buffer invalid */ CLEAR_BUFFERTAG(bufHdr->tag); ! bufHdr->flags = 0; ! bufHdr->usage_count = 0; } } } --- 354,362 ---- elog(ERROR, "local buffer hash table corrupted"); /* Mark buffer invalid */ CLEAR_BUFFERTAG(bufHdr->tag); ! state &= ~BUF_FLAG_MASK; ! state &= ~BUF_USAGECOUNT_MASK; ! pg_atomic_write_u32(&bufHdr->state, state); } } } *************** DropRelFileNodeAllLocalBuffers(RelFileNo *** 349,356 **** { BufferDesc *bufHdr = GetLocalBufferDescriptor(i); LocalBufferLookupEnt *hresult; ! if ((bufHdr->flags & BM_TAG_VALID) && RelFileNodeEquals(bufHdr->tag.rnode, rnode)) { if (LocalRefCount[i] != 0) --- 377,387 ---- { BufferDesc *bufHdr = GetLocalBufferDescriptor(i); LocalBufferLookupEnt *hresult; + uint32 state; ! state = pg_atomic_read_u32(&bufHdr->state); ! ! if ((state & BM_TAG_VALID) && RelFileNodeEquals(bufHdr->tag.rnode, rnode)) { if (LocalRefCount[i] != 0) *************** DropRelFileNodeAllLocalBuffers(RelFileNo *** 367,374 **** elog(ERROR, "local buffer hash table corrupted"); /* Mark buffer invalid */ CLEAR_BUFFERTAG(bufHdr->tag); ! bufHdr->flags = 0; ! bufHdr->usage_count = 0; } } } --- 398,406 ---- elog(ERROR, "local buffer hash table corrupted"); /* Mark buffer invalid */ CLEAR_BUFFERTAG(bufHdr->tag); ! state &= ~BUF_FLAG_MASK; ! state &= ~BUF_USAGECOUNT_MASK; ! pg_atomic_write_u32(&bufHdr->state, state); } } } diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h new file mode 100644 index 521ee1c..92889e6 *** a/src/include/storage/buf_internals.h --- b/src/include/storage/buf_internals.h *************** *** 20,48 **** #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/smgr.h" #include "storage/spin.h" #include "utils/relcache.h" /* * Flags for buffer descriptors * * Note: TAG_VALID essentially means that there is a buffer hashtable * entry associated with the buffer's tag. */ ! #define BM_DIRTY (1 << 0) /* data needs writing */ ! #define BM_VALID (1 << 1) /* data is valid */ ! #define BM_TAG_VALID (1 << 2) /* tag is assigned */ ! #define BM_IO_IN_PROGRESS (1 << 3) /* read or write in progress */ ! #define BM_IO_ERROR (1 << 4) /* previous I/O failed */ ! #define BM_JUST_DIRTIED (1 << 5) /* dirtied since write started */ ! #define BM_PIN_COUNT_WAITER (1 << 6) /* have waiter for sole pin */ ! #define BM_CHECKPOINT_NEEDED (1 << 7) /* must write for checkpoint */ ! #define BM_PERMANENT (1 << 8) /* permanent relation (not * unlogged) */ - - typedef bits16 BufFlags; - /* * The maximum allowed value of usage_count represents a tradeoff between * accuracy and speed of the clock-sweep buffer management algorithm. A --- 20,59 ---- #include "storage/lwlock.h" #include "storage/shmem.h" #include "storage/smgr.h" + #include "port/atomics.h" #include "storage/spin.h" #include "utils/relcache.h" /* + * State is: + * 10 bit flags + * 4 bit usage count + * 18 bit refcount + */ + #define BUF_REFCOUNT_MASK ((1U << 18) - 1) + #define BUF_FLAG_MASK 0xFFC00000U + #define BUF_USAGECOUNT_MASK 0x003C0000U + #define BUF_USAGECOUNT_ONE (1U << 18) + #define BUF_USAGECOUNT_SHIFT 18 + + /* * Flags for buffer descriptors * * Note: TAG_VALID essentially means that there is a buffer hashtable * entry associated with the buffer's tag. */ ! #define BM_LOCKED (1U << 22) /* buffer header is locked */ ! #define BM_DIRTY (1U << 23) /* data needs writing */ ! #define BM_VALID (1U << 24) /* data is valid */ ! #define BM_TAG_VALID (1U << 25) /* tag is assigned */ ! #define BM_IO_IN_PROGRESS (1U << 26) /* read or write in progress */ ! #define BM_IO_ERROR (1U << 27) /* previous I/O failed */ ! #define BM_JUST_DIRTIED (1U << 28) /* dirtied since write started */ ! #define BM_PIN_COUNT_WAITER (1U << 29) /* have waiter for sole pin */ ! #define BM_CHECKPOINT_NEEDED (1U << 30) /* must write for checkpoint */ ! #define BM_PERMANENT (1U << 31) /* permanent relation (not * unlogged) */ /* * The maximum allowed value of usage_count represents a tradeoff between * accuracy and speed of the clock-sweep buffer management algorithm. A *************** typedef struct buftag *** 137,148 **** typedef struct BufferDesc { BufferTag tag; /* ID of page contained in buffer */ - BufFlags flags; /* see bit definitions above */ - uint16 usage_count; /* usage counter for clock sweep code */ - unsigned refcount; /* # of backends holding pins on buffer */ - int wait_backend_pid; /* backend PID of pin-count waiter */ ! slock_t buf_hdr_lock; /* protects the above fields */ int buf_id; /* buffer's index number (from 0) */ int freeNext; /* link in freelist chain */ --- 148,158 ---- typedef struct BufferDesc { BufferTag tag; /* ID of page contained in buffer */ ! /* state of the tag, containing flags, refcount and usagecount */ ! pg_atomic_uint32 state; ! ! int wait_backend_pid; /* backend PID of pin-count waiter */ int buf_id; /* buffer's index number (from 0) */ int freeNext; /* link in freelist chain */ *************** typedef union BufferDescPadded *** 192,207 **** #define FREENEXT_NOT_IN_LIST (-2) /* ! * Macros for acquiring/releasing a shared buffer header's spinlock. ! * Do not apply these to local buffers! ! * ! * Note: as a general coding rule, if you are using these then you probably ! * need to be using a volatile-qualified pointer to the buffer header, to ! * ensure that the compiler doesn't rearrange accesses to the header to ! * occur before or after the spinlock is acquired/released. */ ! #define LockBufHdr(bufHdr) SpinLockAcquire(&(bufHdr)->buf_hdr_lock) ! #define UnlockBufHdr(bufHdr) SpinLockRelease(&(bufHdr)->buf_hdr_lock) /* in buf_init.c */ --- 202,212 ---- #define FREENEXT_NOT_IN_LIST (-2) /* ! * Functions for acquiring/releasing a shared buffer header's spinlock. Do ! * not apply these to local buffers! FIXUP! */ ! extern void LockBufHdr(volatile BufferDesc *desc); ! extern void UnlockBufHdr(volatile BufferDesc *desc); /* in buf_init.c */