/* $Id: critsectrw-generic.cpp 56290 2015-06-09 14:01:31Z vboxsync $ */ /** @file * IPRT - Read/Write Critical Section, Generic. */ /* * Copyright (C) 2009-2015 Oracle Corporation * * This file is part of VirtualBox Open Source Edition (OSE), as * available from http://www.virtualbox.org. This file is free software; * you can redistribute it and/or modify it under the terms of the GNU * General Public License (GPL) as published by the Free Software * Foundation, in version 2 as it comes in the "COPYING" file of the * VirtualBox OSE distribution. VirtualBox OSE is distributed in the * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. * * The contents of this file may alternatively be used under the terms * of the Common Development and Distribution License Version 1.0 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the * VirtualBox OSE distribution, in which case the provisions of the * CDDL are applicable instead of those of the GPL. * * You may elect to license modified versions of this file under the * terms and conditions of either the GPL or the CDDL or both. */ /******************************************************************************* * Header Files * *******************************************************************************/ #define RTCRITSECTRW_WITHOUT_REMAPPING #define RTASSERT_QUIET #include #include "internal/iprt.h" #include #include #include #include #include #include #include #include "internal/magics.h" #include "internal/strict.h" RTDECL(int) RTCritSectRwInit(PRTCRITSECTRW pThis) { return RTCritSectRwInitEx(pThis, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "RTCritSectRw"); } RT_EXPORT_SYMBOL(RTCritSectRwInit); RTDECL(int) RTCritSectRwInitEx(PRTCRITSECTRW pThis, uint32_t fFlags, RTLOCKVALCLASS hClass, uint32_t uSubClass, const char *pszNameFmt, ...) { int rc; AssertReturn(!(fFlags & ~( RTCRITSECT_FLAGS_NO_NESTING | RTCRITSECT_FLAGS_NO_LOCK_VAL | RTCRITSECT_FLAGS_BOOTSTRAP_HACK | RTCRITSECT_FLAGS_NOP )), VERR_INVALID_PARAMETER); /* * Initialize the structure, allocate the lock validator stuff and sems. */ pThis->u32Magic = RTCRITSECTRW_MAGIC_DEAD; pThis->fNeedReset = false; pThis->u64State = 0; pThis->hNativeWriter = NIL_RTNATIVETHREAD; pThis->cWriterReads = 0; pThis->cWriteRecursions = 0; pThis->hEvtWrite = NIL_RTSEMEVENT; pThis->hEvtRead = NIL_RTSEMEVENTMULTI; pThis->pValidatorWrite = NULL; pThis->pValidatorRead = NULL; #if HC_ARCH_BITS == 32 pThis->HCPtrPadding = NIL_RTHCPTR; #endif #ifdef RTCRITSECTRW_STRICT bool const fLVEnabled = !(fFlags & RTCRITSECT_FLAGS_NO_LOCK_VAL); if (!pszNameFmt) { static uint32_t volatile s_iAnon = 0; uint32_t i = ASMAtomicIncU32(&s_iAnon) - 1; rc = RTLockValidatorRecExclCreate(&pThis->pValidatorWrite, hClass, uSubClass, pThis, fLVEnabled, "RTCritSectRw-%u", i); if (RT_SUCCESS(rc)) rc = RTLockValidatorRecSharedCreate(&pThis->pValidatorRead, hClass, uSubClass, pThis, false /*fSignaller*/, fLVEnabled, "RTCritSectRw-%u", i); } else { va_list va; va_start(va, pszNameFmt); rc = RTLockValidatorRecExclCreateV(&pThis->pValidatorWrite, hClass, uSubClass, pThis, fLVEnabled, pszNameFmt, va); va_end(va); if (RT_SUCCESS(rc)) { va_start(va, pszNameFmt); RTLockValidatorRecSharedCreateV(&pThis->pValidatorRead, hClass, uSubClass, pThis, false /*fSignaller*/, fLVEnabled, pszNameFmt, va); va_end(va); } } if (RT_SUCCESS(rc)) rc = RTLockValidatorRecMakeSiblings(&pThis->pValidatorWrite->Core, &pThis->pValidatorRead->Core); if (RT_SUCCESS(rc)) #endif { rc = RTSemEventMultiCreate(&pThis->hEvtRead); if (RT_SUCCESS(rc)) { rc = RTSemEventCreate(&pThis->hEvtWrite); if (RT_SUCCESS(rc)) { pThis->u32Magic = RTCRITSECTRW_MAGIC; return VINF_SUCCESS; } RTSemEventMultiDestroy(pThis->hEvtRead); } } #ifdef RTCRITSECTRW_STRICT RTLockValidatorRecSharedDestroy(&pThis->pValidatorRead); RTLockValidatorRecExclDestroy(&pThis->pValidatorWrite); #endif return rc; } RT_EXPORT_SYMBOL(RTCritSectRwInitEx); RTDECL(uint32_t) RTCritSectRwSetSubClass(PRTCRITSECTRW pThis, uint32_t uSubClass) { AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID); #ifdef RTCRITSECTRW_STRICT AssertReturn(!(pThis->fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID); RTLockValidatorRecSharedSetSubClass(pThis->pValidatorRead, uSubClass); return RTLockValidatorRecExclSetSubClass(pThis->pValidatorWrite, uSubClass); #else NOREF(uSubClass); return RTLOCKVAL_SUB_CLASS_INVALID; #endif } RT_EXPORT_SYMBOL(RTCritSectRwSetSubClass); static int rtCritSectRwEnterShared(PRTCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, bool fTryOnly) { /* * Validate input. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); #ifdef RTCRITSECTRW_STRICT RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt(); if (!fTryOnly) { int rc9; RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter); if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == RTThreadNativeSelf()) rc9 = RTLockValidatorRecExclCheckOrder(pThis->pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); else rc9 = RTLockValidatorRecSharedCheckOrder(pThis->pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); if (RT_FAILURE(rc9)) return rc9; } #endif /* * Get cracking... */ uint64_t u64State = ASMAtomicReadU64(&pThis->u64State); uint64_t u64OldState = u64State; for (;;) { if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) { /* It flows in the right direction, try follow it before it changes. */ uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; c++; Assert(c < RTCSRW_CNT_MASK / 2); u64State &= ~RTCSRW_CNT_RD_MASK; u64State |= c << RTCSRW_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) { #ifdef RTCRITSECTRW_STRICT RTLockValidatorRecSharedAddOwner(pThis->pValidatorRead, hThreadSelf, pSrcPos); #endif break; } } else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0) { /* Wrong direction, but we're alone here and can simply try switch the direction. */ u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); u64State |= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) | (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) { Assert(!pThis->fNeedReset); #ifdef RTCRITSECTRW_STRICT RTLockValidatorRecSharedAddOwner(pThis->pValidatorRead, hThreadSelf, pSrcPos); #endif break; } } else { /* Is the writer perhaps doing a read recursion? */ RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter); if (hNativeSelf == hNativeWriter) { #ifdef RTCRITSECTRW_STRICT int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->pValidatorWrite, &pThis->pValidatorRead->Core, pSrcPos); if (RT_FAILURE(rc9)) return rc9; #endif Assert(pThis->cWriterReads < UINT32_MAX / 2); ASMAtomicIncU32(&pThis->cWriterReads); return VINF_SUCCESS; /* don't break! */ } /* If we're only trying, return already. */ if (fTryOnly) return VERR_SEM_BUSY; /* Add ourselves to the queue and wait for the direction to change. */ uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; c++; Assert(c < RTCSRW_CNT_MASK / 2); uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; cWait++; Assert(cWait <= c); Assert(cWait < RTCSRW_CNT_MASK / 2); u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK); u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) { for (uint32_t iLoop = 0; ; iLoop++) { int rc; #ifdef RTCRITSECTRW_STRICT rc = RTLockValidatorRecSharedCheckBlocking(pThis->pValidatorRead, hThreadSelf, pSrcPos, true, RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false); if (RT_SUCCESS(rc)) #else RTTHREAD hThreadSelf = RTThreadSelf(); RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false); #endif { rc = RTSemEventMultiWait(pThis->hEvtRead, RT_INDEFINITE_WAIT); RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ); if (pThis->u32Magic != RTCRITSECTRW_MAGIC) return VERR_SEM_DESTROYED; } if (RT_FAILURE(rc)) { /* Decrement the counts and return the error. */ for (;;) { u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State); c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; Assert(c > 0); c--; cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; Assert(cWait > 0); cWait--; u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK); u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) break; } return rc; } Assert(pThis->fNeedReset); u64State = ASMAtomicReadU64(&pThis->u64State); if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) break; AssertMsg(iLoop < 1, ("%u\n", iLoop)); } /* Decrement the wait count and maybe reset the semaphore (if we're last). */ for (;;) { u64OldState = u64State; cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; Assert(cWait > 0); cWait--; u64State &= ~RTCSRW_WAIT_CNT_RD_MASK; u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) { if (cWait == 0) { if (ASMAtomicXchgBool(&pThis->fNeedReset, false)) { int rc = RTSemEventMultiReset(pThis->hEvtRead); AssertRCReturn(rc, rc); } } break; } u64State = ASMAtomicReadU64(&pThis->u64State); } #ifdef RTCRITSECTRW_STRICT RTLockValidatorRecSharedAddOwner(pThis->pValidatorRead, hThreadSelf, pSrcPos); #endif break; } } if (pThis->u32Magic != RTCRITSECTRW_MAGIC) return VERR_SEM_DESTROYED; ASMNopPause(); u64State = ASMAtomicReadU64(&pThis->u64State); u64OldState = u64State; } /* got it! */ Assert((ASMAtomicReadU64(&pThis->u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)); return VINF_SUCCESS; } RTDECL(int) RTCritSectRwEnterShared(PRTCRITSECTRW pThis) { #ifndef RTCRITSECTRW_STRICT return rtCritSectRwEnterShared(pThis, NULL, false /*fTryOnly*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return rtCritSectRwEnterShared(pThis, &SrcPos, false /*fTryOnly*/); #endif } RT_EXPORT_SYMBOL(RTCritSectRwEnterShared); RTDECL(int) RTCritSectRwEnterSharedDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) { RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return rtCritSectRwEnterShared(pThis, &SrcPos, false /*fTryOnly*/); } RT_EXPORT_SYMBOL(RTCritSectRwEnterSharedDebug); RTDECL(int) RTCritSectRwTryEnterShared(PRTCRITSECTRW pThis) { #ifndef RTCRITSECTRW_STRICT return rtCritSectRwEnterShared(pThis, NULL, true /*fTryOnly*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return rtCritSectRwEnterShared(pThis, &SrcPos, true /*fTryOnly*/); #endif } RT_EXPORT_SYMBOL(RTCritSectRwEnterShared); RTDECL(int) RTCritSectRwTryEnterSharedDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) { RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return rtCritSectRwEnterShared(pThis, &SrcPos, true /*fTryOnly*/); } RT_EXPORT_SYMBOL(RTCritSectRwEnterSharedDebug); RTDECL(int) RTCritSectRwLeaveShared(PRTCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); /* * Check the direction and take action accordingly. */ uint64_t u64State = ASMAtomicReadU64(&pThis->u64State); uint64_t u64OldState = u64State; if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) { #ifdef RTCRITSECTRW_STRICT int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->pValidatorRead, NIL_RTTHREAD); if (RT_FAILURE(rc9)) return rc9; #endif for (;;) { uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; AssertReturn(c > 0, VERR_NOT_OWNER); c--; if ( c > 0 || (u64State & RTCSRW_CNT_WR_MASK) == 0) { /* Don't change the direction. */ u64State &= ~RTCSRW_CNT_RD_MASK; u64State |= c << RTCSRW_CNT_RD_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) break; } else { /* Reverse the direction and signal the reader threads. */ u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_DIR_MASK); u64State |= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) { int rc = RTSemEventSignal(pThis->hEvtWrite); AssertRC(rc); break; } } ASMNopPause(); u64State = ASMAtomicReadU64(&pThis->u64State); u64OldState = u64State; } } else { RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter); AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER); AssertReturn(pThis->cWriterReads > 0, VERR_NOT_OWNER); #ifdef RTCRITSECTRW_STRICT int rc = RTLockValidatorRecExclUnwindMixed(pThis->pValidatorWrite, &pThis->pValidatorRead->Core); if (RT_FAILURE(rc)) return rc; #endif ASMAtomicDecU32(&pThis->cWriterReads); } return VINF_SUCCESS; } RT_EXPORT_SYMBOL(RTCritSectRwLeaveShared); static int rtCritSectRwEnterExcl(PRTCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, bool fTryOnly) { /* * Validate input. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); #ifdef RTCRITSECTRW_STRICT RTTHREAD hThreadSelf = NIL_RTTHREAD; if (!fTryOnly) { hThreadSelf = RTThreadSelfAutoAdopt(); int rc9 = RTLockValidatorRecExclCheckOrder(pThis->pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); if (RT_FAILURE(rc9)) return rc9; } #endif /* * Check if we're already the owner and just recursing. */ RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter); if (hNativeSelf == hNativeWriter) { Assert((ASMAtomicReadU64(&pThis->u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)); #ifdef RTCRITSECTRW_STRICT int rc9 = RTLockValidatorRecExclRecursion(pThis->pValidatorWrite, pSrcPos); if (RT_FAILURE(rc9)) return rc9; #endif Assert(pThis->cWriteRecursions < UINT32_MAX / 2); ASMAtomicIncU32(&pThis->cWriteRecursions); return VINF_SUCCESS; } /* * Get cracking. */ uint64_t u64State = ASMAtomicReadU64(&pThis->u64State); uint64_t u64OldState = u64State; for (;;) { if ( (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT) || (u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) != 0) { /* It flows in the right direction, try follow it before it changes. */ uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; c++; Assert(c < RTCSRW_CNT_MASK / 2); u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) break; } else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0) { /* Wrong direction, but we're alone here and can simply try switch the direction. */ u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); u64State |= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT); if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) break; } else if (fTryOnly) /* Wrong direction and we're not supposed to wait, just return. */ return VERR_SEM_BUSY; else { /* Add ourselves to the write count and break out to do the wait. */ uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; c++; Assert(c < RTCSRW_CNT_MASK / 2); u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) break; } if (pThis->u32Magic != RTCRITSECTRW_MAGIC) return VERR_SEM_DESTROYED; ASMNopPause(); u64State = ASMAtomicReadU64(&pThis->u64State); u64OldState = u64State; } /* * If we're in write mode now try grab the ownership. Play fair if there * are threads already waiting. */ bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT) && ( ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1 || fTryOnly); if (fDone) ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone); if (!fDone) { /* * If only trying, undo the above writer incrementation and return. */ if (fTryOnly) { for (;;) { u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State); uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; Assert(c > 0); c--; u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) break; } return VERR_SEM_BUSY; } /* * Wait for our turn. */ for (uint32_t iLoop = 0; ; iLoop++) { int rc; #ifdef RTCRITSECTRW_STRICT if (hThreadSelf == NIL_RTTHREAD) hThreadSelf = RTThreadSelfAutoAdopt(); rc = RTLockValidatorRecExclCheckBlocking(pThis->pValidatorWrite, hThreadSelf, pSrcPos, true, RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false); if (RT_SUCCESS(rc)) #else RTTHREAD hThreadSelf = RTThreadSelf(); RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false); #endif { rc = RTSemEventWait(pThis->hEvtWrite, RT_INDEFINITE_WAIT); RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); if (pThis->u32Magic != RTCRITSECTRW_MAGIC) return VERR_SEM_DESTROYED; } if (RT_FAILURE(rc)) { /* Decrement the counts and return the error. */ for (;;) { u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State); uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; Assert(c > 0); c--; u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) break; } return rc; } u64State = ASMAtomicReadU64(&pThis->u64State); if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) { ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone); if (fDone) break; } AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */ } } /* * Got it! */ Assert((ASMAtomicReadU64(&pThis->u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)); ASMAtomicWriteU32(&pThis->cWriteRecursions, 1); Assert(pThis->cWriterReads == 0); #ifdef RTCRITSECTRW_STRICT RTLockValidatorRecExclSetOwner(pThis->pValidatorWrite, hThreadSelf, pSrcPos, true); #endif return VINF_SUCCESS; } RTDECL(int) RTCritSectRwEnterExcl(PRTCRITSECTRW pThis) { #ifndef RTCRITSECTRW_STRICT return rtCritSectRwEnterExcl(pThis, NULL, false /*fTryAgain*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return rtCritSectRwEnterExcl(pThis, &SrcPos, false /*fTryAgain*/); #endif } RT_EXPORT_SYMBOL(RTCritSectRwEnterExcl); RTDECL(int) RTCritSectRwEnterExclDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) { RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return rtCritSectRwEnterExcl(pThis, &SrcPos, false /*fTryAgain*/); } RT_EXPORT_SYMBOL(RTCritSectRwEnterExclDebug); RTDECL(int) RTCritSectRwTryEnterExcl(PRTCRITSECTRW pThis) { #ifndef RTCRITSECTRW_STRICT return rtCritSectRwEnterExcl(pThis, NULL, true /*fTryAgain*/); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return rtCritSectRwEnterExcl(pThis, &SrcPos, true /*fTryAgain*/); #endif } RT_EXPORT_SYMBOL(RTCritSectRwTryEnterExcl); RTDECL(int) RTCritSectRwTryEnterExclDebug(PRTCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) { RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return rtCritSectRwEnterExcl(pThis, &SrcPos, true /*fTryAgain*/); } RT_EXPORT_SYMBOL(RTCritSectRwTryEnterExclDebug); RTDECL(int) RTCritSectRwLeaveExcl(PRTCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter); AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER); /* * Unwind a recursion. */ if (pThis->cWriteRecursions == 1) { AssertReturn(pThis->cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */ #ifdef RTCRITSECTRW_STRICT int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->pValidatorWrite, true); if (RT_FAILURE(rc9)) return rc9; #endif /* * Update the state. */ ASMAtomicWriteU32(&pThis->cWriteRecursions, 0); ASMAtomicWriteHandle(&pThis->hNativeWriter, NIL_RTNATIVETHREAD); for (;;) { uint64_t u64State = ASMAtomicReadU64(&pThis->u64State); uint64_t u64OldState = u64State; uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; Assert(c > 0); c--; if ( c > 0 || (u64State & RTCSRW_CNT_RD_MASK) == 0) { /* Don't change the direction, wait up the next writer if any. */ u64State &= ~RTCSRW_CNT_WR_MASK; u64State |= c << RTCSRW_CNT_WR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) { if (c > 0) { int rc = RTSemEventSignal(pThis->hEvtWrite); AssertRC(rc); } break; } } else { /* Reverse the direction and signal the reader threads. */ u64State &= ~(RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); u64State |= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT; if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState)) { Assert(!pThis->fNeedReset); ASMAtomicWriteBool(&pThis->fNeedReset, true); int rc = RTSemEventMultiSignal(pThis->hEvtRead); AssertRC(rc); break; } } ASMNopPause(); if (pThis->u32Magic != RTCRITSECTRW_MAGIC) return VERR_SEM_DESTROYED; } } else { Assert(pThis->cWriteRecursions != 0); #ifdef RTCRITSECTRW_STRICT int rc9 = RTLockValidatorRecExclUnwind(pThis->pValidatorWrite); if (RT_FAILURE(rc9)) return rc9; #endif ASMAtomicDecU32(&pThis->cWriteRecursions); } return VINF_SUCCESS; } RT_EXPORT_SYMBOL(RTCritSectRwLeaveExcl); RTDECL(bool) RTCritSectRwIsWriteOwner(PRTCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, false); /* * Check ownership. */ RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); RTNATIVETHREAD hNativeWriter; ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter); return hNativeWriter == hNativeSelf; } RT_EXPORT_SYMBOL(RTCritSectRwIsWriteOwner); RTDECL(bool) RTCritSectRwIsReadOwner(PRTCRITSECTRW pThis, bool fWannaHear) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, false); /* * Inspect the state. */ uint64_t u64State = ASMAtomicReadU64(&pThis->u64State); if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) { /* * It's in write mode, so we can only be a reader if we're also the * current writer. */ RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); RTNATIVETHREAD hWriter; ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hWriter); return hWriter == hNativeSelf; } /* * Read mode. If there are no current readers, then we cannot be a reader. */ if (!(u64State & RTCSRW_CNT_RD_MASK)) return false; #ifdef RTCRITSECTRW_STRICT /* * Ask the lock validator. */ return RTLockValidatorRecSharedIsOwner(pThis->pValidatorRead, NIL_RTTHREAD); #else /* * Ok, we don't know, just tell the caller what he want to hear. */ return fWannaHear; #endif } RT_EXPORT_SYMBOL(RTCritSectRwIsReadOwner); RTDECL(uint32_t) RTCritSectRwGetWriteRecursion(PRTCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, 0); /* * Return the requested data. */ return pThis->cWriteRecursions; } RT_EXPORT_SYMBOL(RTCritSectRwGetWriteRecursion); RTDECL(uint32_t) RTCritSectRwGetWriterReadRecursion(PRTCRITSECTRW pThis) { /* * Validate handle. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, 0); /* * Return the requested data. */ return pThis->cWriterReads; } RT_EXPORT_SYMBOL(RTCritSectRwGetWriterReadRecursion); RTDECL(uint32_t) RTCritSectRwGetReadCount(PRTCRITSECTRW pThis) { /* * Validate input. */ AssertPtr(pThis); AssertReturn(pThis->u32Magic == RTCRITSECTRW_MAGIC, 0); /* * Return the requested data. */ uint64_t u64State = ASMAtomicReadU64(&pThis->u64State); if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) return 0; return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; } RT_EXPORT_SYMBOL(RTCritSectRwGetReadCount); RTDECL(int) RTCritSectRwDelete(PRTCRITSECTRW pThis) { /* * Assert free waiters and so on. */ AssertPtr(pThis); Assert(pThis->u32Magic == RTCRITSECTRW_MAGIC); //Assert(pThis->cNestings == 0); //Assert(pThis->cLockers == -1); Assert(pThis->hNativeWriter == NIL_RTNATIVETHREAD); /* * Invalidate the structure and free the semaphores. */ if (!ASMAtomicCmpXchgU32(&pThis->u32Magic, RTCRITSECTRW_MAGIC_DEAD, RTCRITSECTRW_MAGIC)) return VERR_INVALID_PARAMETER; pThis->fFlags = 0; pThis->u64State = 0; RTSEMEVENT hEvtWrite = pThis->hEvtWrite; pThis->hEvtWrite = NIL_RTSEMEVENT; RTSEMEVENTMULTI hEvtRead = pThis->hEvtRead; pThis->hEvtRead = NIL_RTSEMEVENTMULTI; int rc1 = RTSemEventDestroy(hEvtWrite); AssertRC(rc1); int rc2 = RTSemEventMultiDestroy(hEvtRead); AssertRC(rc2); RTLockValidatorRecSharedDestroy(&pThis->pValidatorRead); RTLockValidatorRecExclDestroy(&pThis->pValidatorWrite); return RT_SUCCESS(rc1) ? rc2 : rc1; } RT_EXPORT_SYMBOL(RTCritSectRwDelete);