1
/* $Id: PDMAsyncCompletionFileCache.cpp $ */
3
* PDM Async I/O - Transport data asynchronous in R3 using EMT.
8
* Copyright (C) 2006-2008 Oracle Corporation
10
* This file is part of VirtualBox Open Source Edition (OSE), as
11
* available from http://www.virtualbox.org. This file is free software;
12
* you can redistribute it and/or modify it under the terms of the GNU
13
* General Public License (GPL) as published by the Free Software
14
* Foundation, in version 2 as it comes in the "COPYING" file of the
15
* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
16
* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
19
/** @page pg_pdm_async_completion_cache PDM Async Completion Cache - The file I/O cache
20
* This component implements an I/O cache for file endpoints based on the 2Q cache algorithm.
23
/*******************************************************************************
25
*******************************************************************************/
26
#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
29
#include <iprt/path.h>
31
#include <VBox/stam.h>
33
#include "PDMAsyncCompletionFileInternal.h"
36
* A I/O memory context.
38
typedef struct PDMIOMEMCTX
40
/** Pointer to the scatter/gather list. */
42
/** Number of segments. */
44
/** Current segment we are in. */
46
/** Pointer to the current buffer. */
48
/** Number of bytes left in the current buffer. */
50
} PDMIOMEMCTX, *PPDMIOMEMCTX;
53
# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
56
AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
57
("Thread does not own critical section\n"));\
60
# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
63
AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
64
("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
67
# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
70
AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
71
("Thread is not read owner of the per endpoint RW semaphore\n")); \
75
# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0)
76
# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while(0)
77
# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while(0)
80
/*******************************************************************************
81
* Internal Functions *
82
*******************************************************************************/
83
static void pdmacFileCacheTaskCompleted(PPDMACTASKFILE pTask, void *pvUser, int rc);
86
* Decrement the reference counter of the given cache entry.
89
* @param pEntry The entry to release.
91
DECLINLINE(void) pdmacFileEpCacheEntryRelease(PPDMACFILECACHEENTRY pEntry)
93
AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
94
ASMAtomicDecU32(&pEntry->cRefs);
98
* Increment the reference counter of the given cache entry.
101
* @param pEntry The entry to reference.
103
DECLINLINE(void) pdmacFileEpCacheEntryRef(PPDMACFILECACHEENTRY pEntry)
105
ASMAtomicIncU32(&pEntry->cRefs);
109
* Initialize a I/O memory context.
112
* @param pIoMemCtx Pointer to a unitialized I/O memory context.
113
* @param paDataSeg Pointer to the S/G list.
114
* @param cSegments Number of segments in the S/G list.
116
DECLINLINE(void) pdmIoMemCtxInit(PPDMIOMEMCTX pIoMemCtx, PCRTSGSEG paDataSeg, size_t cSegments)
118
AssertMsg((cSegments > 0) && paDataSeg, ("Trying to initialize a I/O memory context without a S/G list\n"));
120
pIoMemCtx->paDataSeg = paDataSeg;
121
pIoMemCtx->cSegments = cSegments;
122
pIoMemCtx->iSegIdx = 0;
123
pIoMemCtx->pbBuf = (uint8_t *)paDataSeg[0].pvSeg;
124
pIoMemCtx->cbBufLeft = paDataSeg[0].cbSeg;
128
* Return a buffer from the I/O memory context.
130
* @returns Pointer to the buffer
131
* @param pIoMemCtx Pointer to the I/O memory context.
132
* @param pcbData Pointer to the amount of byte requested.
133
* If the current buffer doesn't have enough bytes left
134
* the amount is returned in the variable.
136
DECLINLINE(uint8_t *) pdmIoMemCtxGetBuffer(PPDMIOMEMCTX pIoMemCtx, size_t *pcbData)
138
size_t cbData = RT_MIN(*pcbData, pIoMemCtx->cbBufLeft);
139
uint8_t *pbBuf = pIoMemCtx->pbBuf;
141
pIoMemCtx->cbBufLeft -= cbData;
143
/* Advance to the next segment if required. */
144
if (!pIoMemCtx->cbBufLeft)
146
pIoMemCtx->iSegIdx++;
148
if (RT_UNLIKELY(pIoMemCtx->iSegIdx == pIoMemCtx->cSegments))
150
pIoMemCtx->cbBufLeft = 0;
151
pIoMemCtx->pbBuf = NULL;
155
pIoMemCtx->pbBuf = (uint8_t *)pIoMemCtx->paDataSeg[pIoMemCtx->iSegIdx].pvSeg;
156
pIoMemCtx->cbBufLeft = pIoMemCtx->paDataSeg[pIoMemCtx->iSegIdx].cbSeg;
162
pIoMemCtx->pbBuf += cbData;
168
static void pdmacFileCacheValidate(PPDMACFILECACHEGLOBAL pCache)
170
/* Amount of cached data should never exceed the maximum amount. */
171
AssertMsg(pCache->cbCached <= pCache->cbMax,
172
("Current amount of cached data exceeds maximum\n"));
174
/* The amount of cached data in the LRU and FRU list should match cbCached */
175
AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
176
("Amount of cached data doesn't match\n"));
178
AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
179
("Paged out list exceeds maximum\n"));
183
DECLINLINE(void) pdmacFileCacheLockEnter(PPDMACFILECACHEGLOBAL pCache)
185
RTCritSectEnter(&pCache->CritSect);
187
pdmacFileCacheValidate(pCache);
191
DECLINLINE(void) pdmacFileCacheLockLeave(PPDMACFILECACHEGLOBAL pCache)
194
pdmacFileCacheValidate(pCache);
196
RTCritSectLeave(&pCache->CritSect);
199
DECLINLINE(void) pdmacFileCacheSub(PPDMACFILECACHEGLOBAL pCache, uint32_t cbAmount)
201
PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
202
pCache->cbCached -= cbAmount;
205
DECLINLINE(void) pdmacFileCacheAdd(PPDMACFILECACHEGLOBAL pCache, uint32_t cbAmount)
207
PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
208
pCache->cbCached += cbAmount;
211
DECLINLINE(void) pdmacFileCacheListAdd(PPDMACFILELRULIST pList, uint32_t cbAmount)
213
pList->cbCached += cbAmount;
216
DECLINLINE(void) pdmacFileCacheListSub(PPDMACFILELRULIST pList, uint32_t cbAmount)
218
pList->cbCached -= cbAmount;
221
#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
223
* Checks consistency of a LRU list.
226
* @param pList The LRU list to check.
227
* @param pNotInList Element which is not allowed to occur in the list.
229
static void pdmacFileCacheCheckList(PPDMACFILELRULIST pList, PPDMACFILECACHEENTRY pNotInList)
231
PPDMACFILECACHEENTRY pCurr = pList->pHead;
233
/* Check that there are no double entries and no cycles in the list. */
236
PPDMACFILECACHEENTRY pNext = pCurr->pNext;
240
AssertMsg(pCurr != pNext,
241
("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
243
pNext = pNext->pNext;
246
AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
249
AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
251
pCurr = pCurr->pNext;
257
* Unlinks a cache entry from the LRU list it is assigned to.
260
* @param pEntry The entry to unlink.
262
static void pdmacFileCacheEntryRemoveFromList(PPDMACFILECACHEENTRY pEntry)
264
PPDMACFILELRULIST pList = pEntry->pList;
265
PPDMACFILECACHEENTRY pPrev, pNext;
267
LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
271
#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
272
pdmacFileCacheCheckList(pList, NULL);
275
pPrev = pEntry->pPrev;
276
pNext = pEntry->pNext;
278
AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
279
AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
282
pPrev->pNext = pNext;
285
pList->pHead = pNext;
292
pNext->pPrev = pPrev;
295
pList->pTail = pPrev;
301
pEntry->pList = NULL;
302
pEntry->pPrev = NULL;
303
pEntry->pNext = NULL;
304
pdmacFileCacheListSub(pList, pEntry->cbData);
305
#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
306
pdmacFileCacheCheckList(pList, pEntry);
311
* Adds a cache entry to the given LRU list unlinking it from the currently
312
* assigned list if needed.
315
* @param pList List to the add entry to.
316
* @param pEntry Entry to add.
318
static void pdmacFileCacheEntryAddToList(PPDMACFILELRULIST pList, PPDMACFILECACHEENTRY pEntry)
320
LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
321
#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
322
pdmacFileCacheCheckList(pList, NULL);
325
/* Remove from old list if needed */
327
pdmacFileCacheEntryRemoveFromList(pEntry);
329
pEntry->pNext = pList->pHead;
331
pList->pHead->pPrev = pEntry;
334
Assert(!pList->pTail);
335
pList->pTail = pEntry;
338
pEntry->pPrev = NULL;
339
pList->pHead = pEntry;
340
pdmacFileCacheListAdd(pList, pEntry->cbData);
341
pEntry->pList = pList;
342
#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
343
pdmacFileCacheCheckList(pList, NULL);
348
* Destroys a LRU list freeing all entries.
351
* @param pList Pointer to the LRU list to destroy.
353
* @note The caller must own the critical section of the cache.
355
static void pdmacFileCacheDestroyList(PPDMACFILELRULIST pList)
359
PPDMACFILECACHEENTRY pEntry = pList->pHead;
361
pList->pHead = pEntry->pNext;
363
AssertMsg(!(pEntry->fFlags & (PDMACFILECACHE_ENTRY_IO_IN_PROGRESS | PDMACFILECACHE_ENTRY_IS_DIRTY)),
364
("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
366
RTMemPageFree(pEntry->pbData, pEntry->cbData);
372
* Tries to remove the given amount of bytes from a given list in the cache
373
* moving the entries to one of the given ghosts lists
375
* @returns Amount of data which could be freed.
376
* @param pCache Pointer to the global cache data.
377
* @param cbData The amount of the data to free.
378
* @param pListSrc The source list to evict data from.
379
* @param pGhostListSrc The ghost list removed entries should be moved to
380
* NULL if the entry should be freed.
381
* @param fReuseBuffer Flag whether a buffer should be reused if it has the same size
382
* @param ppbBuf Where to store the address of the buffer if an entry with the
383
* same size was found and fReuseBuffer is true.
385
* @note This function may return fewer bytes than requested because entries
386
* may be marked as non evictable if they are used for I/O at the
389
static size_t pdmacFileCacheEvictPagesFrom(PPDMACFILECACHEGLOBAL pCache, size_t cbData,
390
PPDMACFILELRULIST pListSrc, PPDMACFILELRULIST pGhostListDst,
391
bool fReuseBuffer, uint8_t **ppbBuffer)
393
size_t cbEvicted = 0;
395
PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
397
AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
398
AssertMsg( !pGhostListDst
399
|| (pGhostListDst == &pCache->LruRecentlyUsedOut),
400
("Destination list must be NULL or the recently used but paged out list\n"));
404
AssertPtr(ppbBuffer);
408
/* Start deleting from the tail. */
409
PPDMACFILECACHEENTRY pEntry = pListSrc->pTail;
411
while ((cbEvicted < cbData) && pEntry)
413
PPDMACFILECACHEENTRY pCurr = pEntry;
415
pEntry = pEntry->pPrev;
417
/* We can't evict pages which are currently in progress or dirty but not in progress */
418
if ( !(pCurr->fFlags & PDMACFILECACHE_NOT_EVICTABLE)
419
&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
421
/* Ok eviction candidate. Grab the endpoint semaphore and check again
422
* because somebody else might have raced us. */
423
PPDMACFILEENDPOINTCACHE pEndpointCache = &pCurr->pEndpoint->DataCache;
424
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
426
if (!(pCurr->fFlags & PDMACFILECACHE_NOT_EVICTABLE)
427
&& (ASMAtomicReadU32(&pCurr->cRefs) == 0))
429
LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
431
if (fReuseBuffer && (pCurr->cbData == cbData))
433
STAM_COUNTER_INC(&pCache->StatBuffersReused);
434
*ppbBuffer = pCurr->pbData;
436
else if (pCurr->pbData)
437
RTMemPageFree(pCurr->pbData, pCurr->cbData);
439
pCurr->pbData = NULL;
440
cbEvicted += pCurr->cbData;
442
pdmacFileCacheEntryRemoveFromList(pCurr);
443
pdmacFileCacheSub(pCache, pCurr->cbData);
447
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
449
PPDMACFILECACHEENTRY pGhostEntFree = pGhostListDst->pTail;
451
/* We have to remove the last entries from the paged out list. */
452
while ( ((pGhostListDst->cbCached + pCurr->cbData) > pCache->cbRecentlyUsedOutMax)
455
PPDMACFILECACHEENTRY pFree = pGhostEntFree;
456
PPDMACFILEENDPOINTCACHE pEndpointCacheFree = &pFree->pEndpoint->DataCache;
458
pGhostEntFree = pGhostEntFree->pPrev;
460
RTSemRWRequestWrite(pEndpointCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
462
if (ASMAtomicReadU32(&pFree->cRefs) == 0)
464
pdmacFileCacheEntryRemoveFromList(pFree);
466
STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
467
RTAvlrFileOffsetRemove(pEndpointCacheFree->pTree, pFree->Core.Key);
468
STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
473
RTSemRWReleaseWrite(pEndpointCacheFree->SemRWEntries);
476
if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
478
/* Couldn't remove enough entries. Delete */
479
STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
480
RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key);
481
STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
486
pdmacFileCacheEntryAddToList(pGhostListDst, pCurr);
490
/* Delete the entry from the AVL tree it is assigned to. */
491
STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
492
RTAvlrFileOffsetRemove(pCurr->pEndpoint->DataCache.pTree, pCurr->Core.Key);
493
STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
495
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
502
LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
508
static bool pdmacFileCacheReclaim(PPDMACFILECACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
510
size_t cbRemoved = 0;
512
if ((pCache->cbCached + cbData) < pCache->cbMax)
514
else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
516
/* Try to evict as many bytes as possible from A1in */
517
cbRemoved = pdmacFileCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
518
&pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
521
* If it was not possible to remove enough entries
522
* try the frequently accessed cache.
524
if (cbRemoved < cbData)
526
Assert(!fReuseBuffer || !*ppbBuffer); /* It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
529
* If we removed something we can't pass the reuse buffer flag anymore because
530
* we don't need to evict that much data
533
cbRemoved += pdmacFileCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
534
NULL, fReuseBuffer, ppbBuffer);
536
cbRemoved += pdmacFileCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
542
/* We have to remove entries from frequently access list. */
543
cbRemoved = pdmacFileCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
544
NULL, fReuseBuffer, ppbBuffer);
547
LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
548
return (cbRemoved >= cbData);
552
* Initiates a read I/O task for the given entry.
555
* @param pEntry The entry to fetch the data to.
557
static void pdmacFileCacheReadFromEndpoint(PPDMACFILECACHEENTRY pEntry)
559
LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
561
/* Make sure no one evicts the entry while it is accessed. */
562
pEntry->fFlags |= PDMACFILECACHE_ENTRY_IO_IN_PROGRESS;
564
PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEntry->pEndpoint);
567
AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
569
pIoTask->pEndpoint = pEntry->pEndpoint;
570
pIoTask->enmTransferType = PDMACTASKFILETRANSFER_READ;
571
pIoTask->Off = pEntry->Core.Key;
572
pIoTask->DataSeg.cbSeg = pEntry->cbData;
573
pIoTask->DataSeg.pvSeg = pEntry->pbData;
574
pIoTask->pvUser = pEntry;
575
pIoTask->pfnCompleted = pdmacFileCacheTaskCompleted;
577
/* Send it off to the I/O manager. */
578
pdmacFileEpAddTask(pEntry->pEndpoint, pIoTask);
582
* Initiates a write I/O task for the given entry.
585
* @param pEntry The entry to read the data from.
587
static void pdmacFileCacheWriteToEndpoint(PPDMACFILECACHEENTRY pEntry)
589
LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
591
/* Make sure no one evicts the entry while it is accessed. */
592
pEntry->fFlags |= PDMACFILECACHE_ENTRY_IO_IN_PROGRESS;
594
PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEntry->pEndpoint);
597
AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
599
pIoTask->pEndpoint = pEntry->pEndpoint;
600
pIoTask->enmTransferType = PDMACTASKFILETRANSFER_WRITE;
601
pIoTask->Off = pEntry->Core.Key;
602
pIoTask->DataSeg.cbSeg = pEntry->cbData;
603
pIoTask->DataSeg.pvSeg = pEntry->pbData;
604
pIoTask->pvUser = pEntry;
605
pIoTask->pfnCompleted = pdmacFileCacheTaskCompleted;
607
/* Send it off to the I/O manager. */
608
pdmacFileEpAddTask(pEntry->pEndpoint, pIoTask);
612
* Commit a single dirty entry to the endpoint
615
* @param pEntry The entry to commit.
617
static void pdmacFileCacheEntryCommit(PPDMACFILEENDPOINTCACHE pEndpointCache, PPDMACFILECACHEENTRY pEntry)
619
NOREF(pEndpointCache);
620
AssertMsg( (pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY)
621
&& !(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS),
622
("Invalid flags set for entry %#p\n", pEntry));
624
pdmacFileCacheWriteToEndpoint(pEntry);
628
* Commit all dirty entries for a single endpoint.
631
* @param pEndpointCache The endpoint cache to commit.
633
static void pdmacFileCacheEndpointCommit(PPDMACFILEENDPOINTCACHE pEndpointCache)
635
uint32_t cbCommitted = 0;
636
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
638
/* The list is moved to a new header to reduce locking overhead. */
639
RTLISTNODE ListDirtyNotCommitted;
642
RTListInit(&ListDirtyNotCommitted);
643
RTSpinlockAcquire(pEndpointCache->LockList, &Tmp);
644
RTListMove(&ListDirtyNotCommitted, &pEndpointCache->ListDirtyNotCommitted);
645
RTSpinlockRelease(pEndpointCache->LockList, &Tmp);
647
if (!RTListIsEmpty(&ListDirtyNotCommitted))
649
PPDMACFILECACHEENTRY pEntry = RTListNodeGetFirst(&ListDirtyNotCommitted,
653
while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
655
PPDMACFILECACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMACFILECACHEENTRY,
657
pdmacFileCacheEntryCommit(pEndpointCache, pEntry);
658
cbCommitted += pEntry->cbData;
659
RTListNodeRemove(&pEntry->NodeNotCommitted);
663
/* Commit the last endpoint */
664
Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
665
pdmacFileCacheEntryCommit(pEndpointCache, pEntry);
666
RTListNodeRemove(&pEntry->NodeNotCommitted);
667
AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
668
("Committed all entries but list is not empty\n"));
671
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
672
AssertMsg(pEndpointCache->pCache->cbDirty >= cbCommitted,
673
("Number of committed bytes exceeds number of dirty bytes\n"));
674
ASMAtomicSubU32(&pEndpointCache->pCache->cbDirty, cbCommitted);
678
* Commit all dirty entries in the cache.
681
* @param pCache The global cache instance.
683
static void pdmacFileCacheCommitDirtyEntries(PPDMACFILECACHEGLOBAL pCache)
685
bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
687
if (!fCommitInProgress)
689
pdmacFileCacheLockEnter(pCache);
690
Assert(!RTListIsEmpty(&pCache->ListEndpoints));
692
PPDMACFILEENDPOINTCACHE pEndpointCache = RTListNodeGetFirst(&pCache->ListEndpoints,
693
PDMACFILEENDPOINTCACHE,
695
AssertPtr(pEndpointCache);
697
while (!RTListNodeIsLast(&pCache->ListEndpoints, &pEndpointCache->NodeCacheEndpoint))
699
pdmacFileCacheEndpointCommit(pEndpointCache);
701
pEndpointCache = RTListNodeGetNext(&pEndpointCache->NodeCacheEndpoint, PDMACFILEENDPOINTCACHE,
705
/* Commit the last endpoint */
706
Assert(RTListNodeIsLast(&pCache->ListEndpoints, &pEndpointCache->NodeCacheEndpoint));
707
pdmacFileCacheEndpointCommit(pEndpointCache);
709
pdmacFileCacheLockLeave(pCache);
710
ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
715
* Adds the given entry as a dirty to the cache.
717
* @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
718
* @param pEndpointCache The endpoint cache the entry belongs to.
719
* @param pEntry The entry to add.
721
static bool pdmacFileCacheAddDirtyEntry(PPDMACFILEENDPOINTCACHE pEndpointCache, PPDMACFILECACHEENTRY pEntry)
723
bool fDirtyBytesExceeded = false;
724
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
726
/* If the commit timer is disabled we commit right away. */
727
if (pCache->u32CommitTimeoutMs == 0)
729
pEntry->fFlags |= PDMACFILECACHE_ENTRY_IS_DIRTY;
730
pdmacFileCacheEntryCommit(pEndpointCache, pEntry);
732
else if (!(pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY))
734
pEntry->fFlags |= PDMACFILECACHE_ENTRY_IS_DIRTY;
737
RTSpinlockAcquire(pEndpointCache->LockList, &Tmp);
738
RTListAppend(&pEndpointCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
739
RTSpinlockRelease(pEndpointCache->LockList, &Tmp);
741
uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
743
fDirtyBytesExceeded = (cbDirty >= pCache->cbCommitDirtyThreshold);
746
return fDirtyBytesExceeded;
751
* Completes a task segment freeing all ressources and completes the task handle
752
* if everything was transfered.
754
* @returns Next task segment handle.
755
* @param pTaskSeg Task segment to complete.
756
* @param rc Status code to set.
758
static PPDMACFILETASKSEG pdmacFileCacheTaskComplete(PPDMACFILETASKSEG pTaskSeg, int rc)
760
PPDMACFILETASKSEG pNext = pTaskSeg->pNext;
761
PPDMASYNCCOMPLETIONTASKFILE pTaskFile = pTaskSeg->pTask;
764
ASMAtomicCmpXchgS32(&pTaskFile->rc, rc, VINF_SUCCESS);
766
uint32_t uOld = ASMAtomicSubS32(&pTaskFile->cbTransferLeft, pTaskSeg->cbTransfer);
767
AssertMsg(uOld >= pTaskSeg->cbTransfer, ("New value would overflow\n"));
768
if (!(uOld - pTaskSeg->cbTransfer)
769
&& !ASMAtomicXchgBool(&pTaskFile->fCompleted, true))
770
pdmR3AsyncCompletionCompleteTask(&pTaskFile->Core, pTaskFile->rc, true);
778
* Completion callback for I/O tasks.
781
* @param pTask The completed task.
782
* @param pvUser Opaque user data.
783
* @param rc Status code of the completed request.
785
static void pdmacFileCacheTaskCompleted(PPDMACTASKFILE pTask, void *pvUser, int rc)
787
PPDMACFILECACHEENTRY pEntry = (PPDMACFILECACHEENTRY)pvUser;
788
PPDMACFILECACHEGLOBAL pCache = pEntry->pCache;
789
PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pEntry->pEndpoint;
790
PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
792
/* Reference the entry now as we are clearing the I/O in progres flag
793
* which protects the entry till now. */
794
pdmacFileEpCacheEntryRef(pEntry);
796
RTSemRWRequestWrite(pEndpoint->DataCache.SemRWEntries, RT_INDEFINITE_WAIT);
797
pEntry->fFlags &= ~PDMACFILECACHE_ENTRY_IO_IN_PROGRESS;
799
/* Process waiting segment list. The data in entry might have changed inbetween. */
801
PPDMACFILETASKSEG pComplete = pEntry->pWaitingHead;
802
PPDMACFILETASKSEG pCurr = pComplete;
804
AssertMsg((pCurr && pEntry->pWaitingTail) || (!pCurr && !pEntry->pWaitingTail),
805
("The list tail was not updated correctly\n"));
806
pEntry->pWaitingTail = NULL;
807
pEntry->pWaitingHead = NULL;
809
if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
812
* An error here is difficult to handle as the original request completed already.
813
* The error is logged for now and the VM is paused.
814
* If the user continues the entry is written again in the hope
815
* the user fixed the problem and the next write succeeds.
817
/** @todo r=aeichner: This solution doesn't work
818
* The user will get the message but the VM will hang afterwards
819
* VMR3Suspend() returns when the VM is suspended but suspending
820
* the VM will reopen the images readonly in DrvVD. They are closed first
821
* which will close the endpoints. This will block EMT while the
822
* I/O manager processes the close request but the IO manager is stuck
823
* in the VMR3Suspend call and can't process the request.
824
* Another problem is that closing the VM means flushing the cache
825
* but the entry failed and will probably fail again.
826
* No idea so far how to solve this problem... but the user gets informed
831
LogRel(("I/O cache: Error while writing entry at offset %RTfoff (%u bytes) to file \"%s\"\n",
832
pEntry->Core.Key, pEntry->cbData, pEndpoint->Core.pszUri));
834
rc = VMSetRuntimeError(pEndpoint->Core.pEpClass->pVM, 0, "CACHE_IOERR",
835
N_("The I/O cache encountered an error while updating data in file \"%s\" (rc=%Rrc). Make sure there is enough free space on the disk and that the disk is working properly. Operation can be resumed afterwards."), pEndpoint->Core.pszUri, rc);
837
rc = VMR3Suspend(pEndpoint->Core.pEpClass->pVM);
841
pEntry->fFlags &= ~PDMACFILECACHE_ENTRY_IS_DIRTY;
845
AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
847
memcpy(pEntry->pbData + pCurr->uBufOffset, pCurr->pvBuf, pCurr->cbTransfer);
850
pCurr = pCurr->pNext;
856
AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_READ, ("Invalid transfer type\n"));
857
AssertMsg(!(pEntry->fFlags & PDMACFILECACHE_ENTRY_IS_DIRTY),
858
("Invalid flags set\n"));
864
memcpy(pEntry->pbData + pCurr->uBufOffset, pCurr->pvBuf, pCurr->cbTransfer);
868
memcpy(pCurr->pvBuf, pEntry->pbData + pCurr->uBufOffset, pCurr->cbTransfer);
870
pCurr = pCurr->pNext;
874
bool fCommit = false;
876
fCommit = pdmacFileCacheAddDirtyEntry(pEndpointCache, pEntry);
878
RTSemRWReleaseWrite(pEndpoint->DataCache.SemRWEntries);
880
/* Dereference so that it isn't protected anymore except we issued anyother write for it. */
881
pdmacFileEpCacheEntryRelease(pEntry);
884
pdmacFileCacheCommitDirtyEntries(pCache);
886
/* Complete waiters now. */
888
pComplete = pdmacFileCacheTaskComplete(pComplete, rc);
892
* Commit timer callback.
894
static void pdmacFileCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
896
PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pvUser;
897
PPDMACFILECACHEGLOBAL pCache = &pClassFile->Cache;
899
LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
901
if (ASMAtomicReadU32(&pCache->cbDirty) > 0)
902
pdmacFileCacheCommitDirtyEntries(pCache);
904
TMTimerSetMillies(pTimer, pCache->u32CommitTimeoutMs);
905
LogFlowFunc(("Entries committed, going to sleep\n"));
909
* Initializies the I/O cache.
911
* returns VBox status code.
912
* @param pClassFile The global class data for file endpoints.
913
* @param pCfgNode CFGM node to query configuration data from.
915
int pdmacFileCacheInit(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile, PCFGMNODE pCfgNode)
917
int rc = VINF_SUCCESS;
918
PPDMACFILECACHEGLOBAL pCache = &pClassFile->Cache;
920
rc = CFGMR3QueryU32Def(pCfgNode, "CacheSize", &pCache->cbMax, 5 * _1M);
921
AssertLogRelRCReturn(rc, rc);
923
RTListInit(&pCache->ListEndpoints);
925
pCache->cbCached = 0;
926
pCache->fCommitInProgress = 0;
927
LogFlowFunc((": Maximum number of bytes cached %u\n", pCache->cbMax));
929
/* Initialize members */
930
pCache->LruRecentlyUsedIn.pHead = NULL;
931
pCache->LruRecentlyUsedIn.pTail = NULL;
932
pCache->LruRecentlyUsedIn.cbCached = 0;
934
pCache->LruRecentlyUsedOut.pHead = NULL;
935
pCache->LruRecentlyUsedOut.pTail = NULL;
936
pCache->LruRecentlyUsedOut.cbCached = 0;
938
pCache->LruFrequentlyUsed.pHead = NULL;
939
pCache->LruFrequentlyUsed.pTail = NULL;
940
pCache->LruFrequentlyUsed.cbCached = 0;
942
pCache->cbRecentlyUsedInMax = (pCache->cbMax / 100) * 25; /* 25% of the buffer size */
943
pCache->cbRecentlyUsedOutMax = (pCache->cbMax / 100) * 50; /* 50% of the buffer size */
944
LogFlowFunc((": cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n", pCache->cbRecentlyUsedInMax, pCache->cbRecentlyUsedOutMax));
946
/** @todo r=aeichner: Experiment to find optimal default values */
947
rc = CFGMR3QueryU32Def(pCfgNode, "CacheCommitIntervalMs", &pCache->u32CommitTimeoutMs, 10000 /* 10sec */);
948
AssertLogRelRCReturn(rc, rc);
949
rc = CFGMR3QueryU32(pCfgNode, "CacheCommitThreshold", &pCache->cbCommitDirtyThreshold);
950
if ( rc == VERR_CFGM_VALUE_NOT_FOUND
951
|| rc == VERR_CFGM_NO_PARENT)
953
/* Start committing after 50% of the cache are dirty */
954
pCache->cbCommitDirtyThreshold = pCache->cbMax / 2;
959
STAMR3Register(pClassFile->Core.pVM, &pCache->cbMax,
960
STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
961
"/PDM/AsyncCompletion/File/cbMax",
963
"Maximum cache size");
964
STAMR3Register(pClassFile->Core.pVM, &pCache->cbCached,
965
STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
966
"/PDM/AsyncCompletion/File/cbCached",
968
"Currently used cache");
969
STAMR3Register(pClassFile->Core.pVM, &pCache->LruRecentlyUsedIn.cbCached,
970
STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
971
"/PDM/AsyncCompletion/File/cbCachedMruIn",
973
"Number of bytes cached in MRU list");
974
STAMR3Register(pClassFile->Core.pVM, &pCache->LruRecentlyUsedOut.cbCached,
975
STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
976
"/PDM/AsyncCompletion/File/cbCachedMruOut",
978
"Number of bytes cached in FRU list");
979
STAMR3Register(pClassFile->Core.pVM, &pCache->LruFrequentlyUsed.cbCached,
980
STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
981
"/PDM/AsyncCompletion/File/cbCachedFru",
983
"Number of bytes cached in FRU ghost list");
985
#ifdef VBOX_WITH_STATISTICS
986
STAMR3Register(pClassFile->Core.pVM, &pCache->cHits,
987
STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
988
"/PDM/AsyncCompletion/File/CacheHits",
989
STAMUNIT_COUNT, "Number of hits in the cache");
990
STAMR3Register(pClassFile->Core.pVM, &pCache->cPartialHits,
991
STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
992
"/PDM/AsyncCompletion/File/CachePartialHits",
993
STAMUNIT_COUNT, "Number of partial hits in the cache");
994
STAMR3Register(pClassFile->Core.pVM, &pCache->cMisses,
995
STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
996
"/PDM/AsyncCompletion/File/CacheMisses",
997
STAMUNIT_COUNT, "Number of misses when accessing the cache");
998
STAMR3Register(pClassFile->Core.pVM, &pCache->StatRead,
999
STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1000
"/PDM/AsyncCompletion/File/CacheRead",
1001
STAMUNIT_BYTES, "Number of bytes read from the cache");
1002
STAMR3Register(pClassFile->Core.pVM, &pCache->StatWritten,
1003
STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1004
"/PDM/AsyncCompletion/File/CacheWritten",
1005
STAMUNIT_BYTES, "Number of bytes written to the cache");
1006
STAMR3Register(pClassFile->Core.pVM, &pCache->StatTreeGet,
1007
STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1008
"/PDM/AsyncCompletion/File/CacheTreeGet",
1009
STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
1010
STAMR3Register(pClassFile->Core.pVM, &pCache->StatTreeInsert,
1011
STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1012
"/PDM/AsyncCompletion/File/CacheTreeInsert",
1013
STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
1014
STAMR3Register(pClassFile->Core.pVM, &pCache->StatTreeRemove,
1015
STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
1016
"/PDM/AsyncCompletion/File/CacheTreeRemove",
1017
STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
1018
STAMR3Register(pClassFile->Core.pVM, &pCache->StatBuffersReused,
1019
STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1020
"/PDM/AsyncCompletion/File/CacheBuffersReused",
1021
STAMUNIT_COUNT, "Number of times a buffer could be reused");
1024
/* Initialize the critical section */
1025
rc = RTCritSectInit(&pCache->CritSect);
1029
/* Create the commit timer */
1030
if (pCache->u32CommitTimeoutMs > 0)
1031
rc = TMR3TimerCreateInternal(pClassFile->Core.pVM, TMCLOCK_REAL,
1032
pdmacFileCacheCommitTimerCallback,
1035
&pClassFile->Cache.pTimerCommit);
1039
LogRel(("AIOMgr: Cache successfully initialised. Cache size is %u bytes\n", pCache->cbMax));
1040
LogRel(("AIOMgr: Cache commit interval is %u ms\n", pCache->u32CommitTimeoutMs));
1041
LogRel(("AIOMgr: Cache commit threshold is %u bytes\n", pCache->cbCommitDirtyThreshold));
1042
return VINF_SUCCESS;
1045
RTCritSectDelete(&pCache->CritSect);
1052
* Destroysthe cache freeing all data.
1055
* @param pClassFile The global class data for file endpoints.
1057
void pdmacFileCacheDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile)
1059
PPDMACFILECACHEGLOBAL pCache = &pClassFile->Cache;
1061
/* Make sure no one else uses the cache now */
1062
pdmacFileCacheLockEnter(pCache);
1064
/* Cleanup deleting all cache entries waiting for in progress entries to finish. */
1065
pdmacFileCacheDestroyList(&pCache->LruRecentlyUsedIn);
1066
pdmacFileCacheDestroyList(&pCache->LruRecentlyUsedOut);
1067
pdmacFileCacheDestroyList(&pCache->LruFrequentlyUsed);
1069
pdmacFileCacheLockLeave(pCache);
1071
RTCritSectDelete(&pCache->CritSect);
1075
* Initializes per endpoint cache data
1076
* like the AVL tree used to access cached entries.
1078
* @returns VBox status code.
1079
* @param pEndpoint The endpoint to init the cache for,
1080
* @param pClassFile The global class data for file endpoints.
1082
int pdmacFileEpCacheInit(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile)
1084
PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
1086
pEndpointCache->pCache = &pClassFile->Cache;
1087
RTListInit(&pEndpointCache->ListDirtyNotCommitted);
1088
int rc = RTSpinlockCreate(&pEndpointCache->LockList);
1092
rc = RTSemRWCreate(&pEndpointCache->SemRWEntries);
1095
pEndpointCache->pTree = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
1096
if (pEndpointCache->pTree)
1098
pClassFile->Cache.cRefs++;
1099
RTListAppend(&pClassFile->Cache.ListEndpoints, &pEndpointCache->NodeCacheEndpoint);
1101
/* Arm the timer if this is the first endpoint. */
1102
if ( pClassFile->Cache.cRefs == 1
1103
&& pClassFile->Cache.u32CommitTimeoutMs > 0)
1104
rc = TMTimerSetMillies(pClassFile->Cache.pTimerCommit, pClassFile->Cache.u32CommitTimeoutMs);
1107
rc = VERR_NO_MEMORY;
1110
RTSemRWDestroy(pEndpointCache->SemRWEntries);
1114
RTSpinlockDestroy(pEndpointCache->LockList);
1117
#ifdef VBOX_WITH_STATISTICS
1120
STAMR3RegisterF(pClassFile->Core.pVM, &pEndpointCache->StatWriteDeferred,
1121
STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
1122
STAMUNIT_COUNT, "Number of deferred writes",
1123
"/PDM/AsyncCompletion/File/%s/Cache/DeferredWrites", RTPathFilename(pEndpoint->Core.pszUri));
1127
LogFlowFunc(("Leave rc=%Rrc\n", rc));
1132
* Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
1134
* @returns IPRT status code.
1135
* @param pNode The node to destroy.
1136
* @param pvUser Opaque user data.
1138
static int pdmacFileEpCacheEntryDestroy(PAVLRFOFFNODECORE pNode, void *pvUser)
1140
PPDMACFILECACHEENTRY pEntry = (PPDMACFILECACHEENTRY)pNode;
1141
PPDMACFILECACHEGLOBAL pCache = (PPDMACFILECACHEGLOBAL)pvUser;
1142
PPDMACFILEENDPOINTCACHE pEndpointCache = &pEntry->pEndpoint->DataCache;
1144
while (ASMAtomicReadU32(&pEntry->fFlags) & (PDMACFILECACHE_ENTRY_IO_IN_PROGRESS | PDMACFILECACHE_ENTRY_IS_DIRTY))
1146
/* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
1147
pdmacFileEpCacheEntryRef(pEntry);
1148
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1149
pdmacFileCacheLockLeave(pCache);
1153
/* Re-enter all locks */
1154
pdmacFileCacheLockEnter(pCache);
1155
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1156
pdmacFileEpCacheEntryRelease(pEntry);
1159
AssertMsg(!(pEntry->fFlags & (PDMACFILECACHE_ENTRY_IO_IN_PROGRESS | PDMACFILECACHE_ENTRY_IS_DIRTY)),
1160
("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
1162
bool fUpdateCache = pEntry->pList == &pCache->LruFrequentlyUsed
1163
|| pEntry->pList == &pCache->LruRecentlyUsedIn;
1165
pdmacFileCacheEntryRemoveFromList(pEntry);
1168
pdmacFileCacheSub(pCache, pEntry->cbData);
1170
RTMemPageFree(pEntry->pbData, pEntry->cbData);
1173
return VINF_SUCCESS;
1177
* Destroys all cache ressources used by the given endpoint.
1180
* @param pEndpoint The endpoint to the destroy.
1182
void pdmacFileEpCacheDestroy(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
1184
PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
1185
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1187
/* Make sure nobody is accessing the cache while we delete the tree. */
1188
pdmacFileCacheLockEnter(pCache);
1189
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1190
RTAvlrFileOffsetDestroy(pEndpointCache->pTree, pdmacFileEpCacheEntryDestroy, pCache);
1191
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1193
RTSpinlockDestroy(pEndpointCache->LockList);
1196
RTListNodeRemove(&pEndpointCache->NodeCacheEndpoint);
1199
&& pCache->u32CommitTimeoutMs > 0)
1200
TMTimerStop(pCache->pTimerCommit);
1202
pdmacFileCacheLockLeave(pCache);
1204
RTSemRWDestroy(pEndpointCache->SemRWEntries);
1206
#ifdef VBOX_WITH_STATISTICS
1207
PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
1209
STAMR3Deregister(pEpClassFile->Core.pVM, &pEndpointCache->StatWriteDeferred);
1213
static PPDMACFILECACHEENTRY pdmacFileEpCacheGetCacheEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off)
1215
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1216
PPDMACFILECACHEENTRY pEntry = NULL;
1218
STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1220
RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1221
pEntry = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetRangeGet(pEndpointCache->pTree, off);
1223
pdmacFileEpCacheEntryRef(pEntry);
1224
RTSemRWReleaseRead(pEndpointCache->SemRWEntries);
1226
STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1232
* Return the best fit cache entries for the given offset.
1235
* @param pEndpointCache The endpoint cache.
1236
* @param off The offset.
1237
* @param pEntryAbove Where to store the pointer to the best fit entry above the
1238
* the given offset. NULL if not required.
1239
* @param pEntryBelow Where to store the pointer to the best fit entry below the
1240
* the given offset. NULL if not required.
1242
static void pdmacFileEpCacheGetCacheBestFitEntryByOffset(PPDMACFILEENDPOINTCACHE pEndpointCache, RTFOFF off,
1243
PPDMACFILECACHEENTRY *ppEntryAbove,
1244
PPDMACFILECACHEENTRY *ppEntryBelow)
1246
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1248
STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
1250
RTSemRWRequestRead(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1253
*ppEntryAbove = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, true /*fAbove*/);
1255
pdmacFileEpCacheEntryRef(*ppEntryAbove);
1260
*ppEntryBelow = (PPDMACFILECACHEENTRY)RTAvlrFileOffsetGetBestFit(pEndpointCache->pTree, off, false /*fAbove*/);
1262
pdmacFileEpCacheEntryRef(*ppEntryBelow);
1264
RTSemRWReleaseRead(pEndpointCache->SemRWEntries);
1266
STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
1269
static void pdmacFileEpCacheInsertEntry(PPDMACFILEENDPOINTCACHE pEndpointCache, PPDMACFILECACHEENTRY pEntry)
1271
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1273
STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
1274
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1275
bool fInserted = RTAvlrFileOffsetInsert(pEndpointCache->pTree, &pEntry->Core);
1276
AssertMsg(fInserted, ("Node was not inserted into tree\n"));
1277
STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
1278
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1282
* Allocates and initializes a new entry for the cache.
1283
* The entry has a reference count of 1.
1285
* @returns Pointer to the new cache entry or NULL if out of memory.
1286
* @param pCache The cache the entry belongs to.
1287
* @param pEndoint The endpoint the entry holds data for.
1288
* @param off Start offset.
1289
* @param cbData Size of the cache entry.
1290
* @param pbBuffer Pointer to the buffer to use.
1291
* NULL if a new buffer should be allocated.
1292
* The buffer needs to have the same size of the entry.
1294
static PPDMACFILECACHEENTRY pdmacFileCacheEntryAlloc(PPDMACFILECACHEGLOBAL pCache,
1295
PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
1296
RTFOFF off, size_t cbData, uint8_t *pbBuffer)
1298
PPDMACFILECACHEENTRY pEntryNew = (PPDMACFILECACHEENTRY)RTMemAllocZ(sizeof(PDMACFILECACHEENTRY));
1300
if (RT_UNLIKELY(!pEntryNew))
1303
pEntryNew->Core.Key = off;
1304
pEntryNew->Core.KeyLast = off + cbData - 1;
1305
pEntryNew->pEndpoint = pEndpoint;
1306
pEntryNew->pCache = pCache;
1307
pEntryNew->fFlags = 0;
1308
pEntryNew->cRefs = 1; /* We are using it now. */
1309
pEntryNew->pList = NULL;
1310
pEntryNew->cbData = cbData;
1311
pEntryNew->pWaitingHead = NULL;
1312
pEntryNew->pWaitingTail = NULL;
1314
pEntryNew->pbData = pbBuffer;
1316
pEntryNew->pbData = (uint8_t *)RTMemPageAlloc(cbData);
1318
if (RT_UNLIKELY(!pEntryNew->pbData))
1320
RTMemFree(pEntryNew);
1328
* Adds a segment to the waiting list for a cache entry
1329
* which is currently in progress.
1332
* @param pEntry The cache entry to add the segment to.
1333
* @param pSeg The segment to add.
1335
DECLINLINE(void) pdmacFileEpCacheEntryAddWaitingSegment(PPDMACFILECACHEENTRY pEntry, PPDMACFILETASKSEG pSeg)
1339
if (pEntry->pWaitingHead)
1341
AssertPtr(pEntry->pWaitingTail);
1343
pEntry->pWaitingTail->pNext = pSeg;
1344
pEntry->pWaitingTail = pSeg;
1348
Assert(!pEntry->pWaitingTail);
1350
pEntry->pWaitingHead = pSeg;
1351
pEntry->pWaitingTail = pSeg;
1356
* Checks that a set of flags is set/clear acquiring the R/W semaphore
1357
* in exclusive mode.
1359
* @returns true if the flag in fSet is set and the one in fClear is clear.
1361
* The R/W semaphore is only held if true is returned.
1363
* @param pEndpointCache The endpoint cache instance data.
1364
* @param pEntry The entry to check the flags for.
1365
* @param fSet The flag which is tested to be set.
1366
* @param fClear The flag which is tested to be clear.
1368
DECLINLINE(bool) pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(PPDMACFILEENDPOINTCACHE pEndpointCache,
1369
PPDMACFILECACHEENTRY pEntry,
1370
uint32_t fSet, uint32_t fClear)
1372
uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1373
bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1377
/* Acquire the lock and check again becuase the completion callback might have raced us. */
1378
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1380
fFlags = ASMAtomicReadU32(&pEntry->fFlags);
1381
fPassed = ((fFlags & fSet) && !(fFlags & fClear));
1383
/* Drop the lock if we didn't passed the test. */
1385
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1392
* Copies data to a buffer described by a I/O memory context.
1395
* @param pIoMemCtx The I/O memory context to copy the data into.
1396
* @param pbData Pointer to the data data to copy.
1397
* @param cbData Amount of data to copy.
1399
static void pdmacFileEpCacheCopyToIoMemCtx(PPDMIOMEMCTX pIoMemCtx,
1405
size_t cbCopy = cbData;
1406
uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbCopy);
1410
memcpy(pbBuf, pbData, cbCopy);
1418
* Copies data from a buffer described by a I/O memory context.
1421
* @param pIoMemCtx The I/O memory context to copy the data from.
1422
* @param pbData Pointer to the destination buffer.
1423
* @param cbData Amount of data to copy.
1425
static void pdmacFileEpCacheCopyFromIoMemCtx(PPDMIOMEMCTX pIoMemCtx,
1431
size_t cbCopy = cbData;
1432
uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbCopy);
1436
memcpy(pbData, pbBuf, cbCopy);
1444
* Add a buffer described by the I/O memory context
1445
* to the entry waiting for completion.
1448
* @param pEntry The entry to add the buffer to.
1449
* @param pTask Task associated with the buffer.
1450
* @param pIoMemCtx The memory context to use.
1451
* @param OffDiff Offset from the start of the buffer
1453
* @param cbData Amount of data to wait for onthis entry.
1454
* @param fWrite Flag whether the task waits because it wants to write
1455
* to the cache entry.
1457
static void pdmacFileEpCacheEntryWaitersAdd(PPDMACFILECACHEENTRY pEntry,
1458
PPDMASYNCCOMPLETIONTASKFILE pTask,
1459
PPDMIOMEMCTX pIoMemCtx,
1466
PPDMACFILETASKSEG pSeg = (PPDMACFILETASKSEG)RTMemAllocZ(sizeof(PDMACFILETASKSEG));
1467
size_t cbSeg = cbData;
1468
uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbSeg);
1470
pSeg->pTask = pTask;
1471
pSeg->uBufOffset = OffDiff;
1472
pSeg->cbTransfer = cbSeg;
1473
pSeg->pvBuf = pbBuf;
1474
pSeg->fWrite = fWrite;
1476
pdmacFileEpCacheEntryAddWaitingSegment(pEntry, pSeg);
1484
* Passthrough a part of a request directly to the I/O manager
1485
* handling the endpoint.
1488
* @param pEndpoint The endpoint.
1489
* @param pTask The task.
1490
* @param pIoMemCtx The I/O memory context to use.
1491
* @param offStart Offset to start transfer from.
1492
* @param cbData Amount of data to transfer.
1493
* @param enmTransferType The transfer type (read/write)
1495
static void pdmacFileEpCacheRequestPassthrough(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
1496
PPDMASYNCCOMPLETIONTASKFILE pTask,
1497
PPDMIOMEMCTX pIoMemCtx,
1498
RTFOFF offStart, size_t cbData,
1499
PDMACTASKFILETRANSFER enmTransferType)
1503
size_t cbSeg = cbData;
1504
uint8_t *pbBuf = pdmIoMemCtxGetBuffer(pIoMemCtx, &cbSeg);
1505
PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEndpoint);
1508
pIoTask->pEndpoint = pEndpoint;
1509
pIoTask->enmTransferType = enmTransferType;
1510
pIoTask->Off = offStart;
1511
pIoTask->DataSeg.cbSeg = cbSeg;
1512
pIoTask->DataSeg.pvSeg = pbBuf;
1513
pIoTask->pvUser = pTask;
1514
pIoTask->pfnCompleted = pdmacFileEpTaskCompleted;
1519
/* Send it off to the I/O manager. */
1520
pdmacFileEpAddTask(pEndpoint, pIoTask);
1525
* Calculate aligned offset and size for a new cache entry
1526
* which do not intersect with an already existing entry and the
1529
* @returns The number of bytes the entry can hold of the requested amount
1531
* @param pEndpoint The endpoint.
1532
* @param pEndpointCache The endpoint cache.
1533
* @param off The start offset.
1534
* @param cb The number of bytes the entry needs to hold at least.
1535
* @param uAlignment Alignment of the boundary sizes.
1536
* @param poffAligned Where to store the aligned offset.
1537
* @param pcbAligned Where to store the aligned size of the entry.
1539
static size_t pdmacFileEpCacheEntryBoundariesCalc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
1540
PPDMACFILEENDPOINTCACHE pEndpointCache,
1541
RTFOFF off, size_t cb,
1542
unsigned uAlignment,
1543
RTFOFF *poffAligned, size_t *pcbAligned)
1546
size_t cbInEntry = 0;
1548
PPDMACFILECACHEENTRY pEntryAbove = NULL;
1549
PPDMACFILECACHEENTRY pEntryBelow = NULL;
1551
/* Get the best fit entries around the offset */
1552
pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off,
1553
&pEntryAbove, &pEntryBelow);
1556
LogFlow(("%sest fit entry below off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n",
1557
pEntryBelow ? "B" : "No b",
1559
pEntryBelow ? pEntryBelow->Core.Key : 0,
1560
pEntryBelow ? pEntryBelow->Core.KeyLast : 0,
1561
pEntryBelow ? pEntryBelow->cbData : 0));
1563
LogFlow(("%sest fit entry above off=%RTfoff (BestFit=%RTfoff BestFitEnd=%RTfoff BestFitSize=%u)\n",
1564
pEntryAbove ? "B" : "No b",
1566
pEntryAbove ? pEntryAbove->Core.Key : 0,
1567
pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
1568
pEntryAbove ? pEntryAbove->cbData : 0));
1570
/* Align the offset first. */
1571
offAligned = off & ~(RTFOFF)(512-1);
1573
&& offAligned <= pEntryBelow->Core.KeyLast)
1574
offAligned = pEntryBelow->Core.KeyLast;
1577
&& off + (RTFOFF)cb > pEntryAbove->Core.Key)
1579
cbInEntry = pEntryAbove->Core.Key - off;
1580
cbAligned = pEntryAbove->Core.Key - offAligned;
1585
* Align the size to a 4KB boundary.
1586
* Memory size is aligned to a page boundary
1587
* and memory is wasted if the size is rather small.
1588
* (For example reads with a size of 512 bytes).
1591
cbAligned = RT_ALIGN_Z(cb + (off - offAligned), uAlignment);
1594
* Clip to file size if the original request doesn't
1595
* exceed the file (not an appending write)
1597
uint64_t cbReq = off + (RTFOFF)cb;
1598
if (cbReq >= pEndpoint->cbFile)
1599
cbAligned = cbReq - offAligned;
1601
cbAligned = RT_MIN(pEndpoint->cbFile - offAligned, cbAligned);
1604
Assert(pEntryAbove->Core.Key >= off);
1605
cbAligned = RT_MIN(cbAligned, (uint64_t)pEntryAbove->Core.Key - offAligned);
1609
/* A few sanity checks */
1610
AssertMsg(!pEntryBelow || pEntryBelow->Core.KeyLast < offAligned,
1611
("Aligned start offset intersects with another cache entry\n"));
1612
AssertMsg(!pEntryAbove || (offAligned + (RTFOFF)cbAligned) <= pEntryAbove->Core.Key,
1613
("Aligned size intersects with another cache entry\n"));
1614
Assert(cbInEntry <= cbAligned);
1615
AssertMsg( ( offAligned + (RTFOFF)cbAligned <= (RTFOFF)pEndpoint->cbFile
1616
&& off + (RTFOFF)cb <= (RTFOFF)pEndpoint->cbFile)
1617
|| (offAligned + (RTFOFF)cbAligned <= off + (RTFOFF)cb),
1618
("Unwanted file size increase\n"));
1621
pdmacFileEpCacheEntryRelease(pEntryBelow);
1623
pdmacFileEpCacheEntryRelease(pEntryAbove);
1625
LogFlow(("offAligned=%RTfoff cbAligned=%u\n", offAligned, cbAligned));
1627
*poffAligned = offAligned;
1628
*pcbAligned = cbAligned;
1634
* Create a new cache entry evicting data from the cache if required.
1636
* @returns Pointer to the new cache entry or NULL
1637
* if not enough bytes could be evicted from the cache.
1638
* @param pEndpoint The endpoint.
1639
* @param pEndpointCache The endpoint cache.
1640
* @param off The offset.
1641
* @param cb Number of bytes the cache entry should have.
1642
* @param uAlignment Alignment the size of the entry should have.
1643
* @param pcbData Where to store the number of bytes the new
1644
* entry can hold. May be lower than actually requested
1645
* due to another entry intersecting the access range.
1647
static PPDMACFILECACHEENTRY pdmacFileEpCacheEntryCreate(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
1648
PPDMACFILEENDPOINTCACHE pEndpointCache,
1649
RTFOFF off, size_t cb,
1650
unsigned uAlignment,
1653
RTFOFF offStart = 0;
1655
PPDMACFILECACHEENTRY pEntryNew = NULL;
1656
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1657
uint8_t *pbBuffer = NULL;
1659
*pcbData = pdmacFileEpCacheEntryBoundariesCalc(pEndpoint,
1663
&offStart, &cbEntry);
1665
pdmacFileCacheLockEnter(pCache);
1666
bool fEnough = pdmacFileCacheReclaim(pCache, cbEntry, true, &pbBuffer);
1670
LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
1672
pEntryNew = pdmacFileCacheEntryAlloc(pCache, pEndpoint,
1675
if (RT_LIKELY(pEntryNew))
1677
pdmacFileCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
1678
pdmacFileCacheAdd(pCache, cbEntry);
1679
pdmacFileCacheLockLeave(pCache);
1681
pdmacFileEpCacheInsertEntry(pEndpointCache, pEntryNew);
1683
AssertMsg( (off >= pEntryNew->Core.Key)
1684
&& (off + (RTFOFF)*pcbData <= pEntryNew->Core.KeyLast + 1),
1685
("Overflow in calculation off=%RTfoff OffsetAligned=%RTfoff\n",
1686
off, pEntryNew->Core.Key));
1689
pdmacFileCacheLockLeave(pCache);
1692
pdmacFileCacheLockLeave(pCache);
1698
* Reads the specified data from the endpoint using the cache if possible.
1700
* @returns VBox status code.
1701
* @param pEndpoint The endpoint to read from.
1702
* @param pTask The task structure used as identifier for this request.
1703
* @param off The offset to start reading from.
1704
* @param paSegments Pointer to the array holding the destination buffers.
1705
* @param cSegments Number of segments in the array.
1706
* @param cbRead Number of bytes to read.
1708
int pdmacFileEpCacheRead(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
1709
RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
1712
int rc = VINF_SUCCESS;
1713
PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
1714
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1715
PPDMACFILECACHEENTRY pEntry;
1717
LogFlowFunc((": pEndpoint=%#p{%s} pTask=%#p off=%RTfoff paSegments=%#p cSegments=%u cbRead=%u\n",
1718
pEndpoint, pEndpoint->Core.pszUri, pTask, off, paSegments, cSegments, cbRead));
1720
/* Set to completed to make sure that the task is valid while we access it. */
1721
ASMAtomicWriteBool(&pTask->fCompleted, true);
1723
/* Init the I/O memory context */
1724
PDMIOMEMCTX IoMemCtx;
1725
pdmIoMemCtxInit(&IoMemCtx, paSegments, cSegments);
1731
pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off);
1734
* If there is no entry we try to create a new one eviciting unused pages
1735
* if the cache is full. If this is not possible we will pass the request through
1736
* and skip the caching (all entries may be still in progress so they can't
1738
* If we have an entry it can be in one of the LRU lists where the entry
1739
* contains data (recently used or frequently used LRU) so we can just read
1740
* the data we need and put the entry at the head of the frequently used LRU list.
1741
* In case the entry is in one of the ghost lists it doesn't contain any data.
1742
* We have to fetch it again evicting pages from either T1 or T2 to make room.
1746
RTFOFF OffDiff = off - pEntry->Core.Key;
1748
AssertMsg(off >= pEntry->Core.Key,
1749
("Overflow in calculation off=%RTfoff OffsetAligned=%RTfoff\n",
1750
off, pEntry->Core.Key));
1752
AssertPtr(pEntry->pList);
1754
cbToRead = RT_MIN(pEntry->cbData - OffDiff, cbRead);
1756
AssertMsg(off + (RTFOFF)cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
1757
("Buffer of cache entry exceeded off=%RTfoff cbToRead=%d\n",
1763
STAM_COUNTER_INC(&pCache->cHits);
1765
STAM_COUNTER_INC(&pCache->cPartialHits);
1767
STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
1769
/* Ghost lists contain no data. */
1770
if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1771
|| (pEntry->pList == &pCache->LruFrequentlyUsed))
1773
if (pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
1774
PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
1775
PDMACFILECACHE_ENTRY_IS_DIRTY))
1777
/* Entry didn't completed yet. Append to the list */
1778
pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
1781
false /* fWrite */);
1782
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1786
/* Read as much as we can from the entry. */
1787
pdmacFileEpCacheCopyToIoMemCtx(&IoMemCtx, pEntry->pbData + OffDiff, cbToRead);
1788
ASMAtomicSubS32(&pTask->cbTransferLeft, cbToRead);
1791
/* Move this entry to the top position */
1792
if (pEntry->pList == &pCache->LruFrequentlyUsed)
1794
pdmacFileCacheLockEnter(pCache);
1795
pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1796
pdmacFileCacheLockLeave(pCache);
1798
/* Release the entry */
1799
pdmacFileEpCacheEntryRelease(pEntry);
1803
uint8_t *pbBuffer = NULL;
1805
LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
1807
pdmacFileCacheLockEnter(pCache);
1808
pdmacFileCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
1809
bool fEnough = pdmacFileCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
1811
/* Move the entry to Am and fetch it to the cache. */
1814
pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
1815
pdmacFileCacheAdd(pCache, pEntry->cbData);
1816
pdmacFileCacheLockLeave(pCache);
1819
pEntry->pbData = pbBuffer;
1821
pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
1822
AssertPtr(pEntry->pbData);
1824
pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
1827
false /* fWrite */);
1828
pdmacFileCacheReadFromEndpoint(pEntry);
1829
/* Release the entry */
1830
pdmacFileEpCacheEntryRelease(pEntry);
1834
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
1835
STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
1836
RTAvlrFileOffsetRemove(pEndpointCache->pTree, pEntry->Core.Key);
1837
STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
1838
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
1840
pdmacFileCacheLockLeave(pCache);
1844
pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
1845
&IoMemCtx, off, cbToRead,
1846
PDMACTASKFILETRANSFER_READ);
1852
#ifdef VBOX_WITH_IO_READ_CACHE
1853
/* No entry found for this offset. Create a new entry and fetch the data to the cache. */
1854
PPDMACFILECACHEENTRY pEntryNew = pdmacFileEpCacheEntryCreate(pEndpoint,
1865
STAM_COUNTER_INC(&pCache->cMisses);
1867
STAM_COUNTER_INC(&pCache->cPartialHits);
1869
pdmacFileEpCacheEntryWaitersAdd(pEntryNew, pTask,
1871
off - pEntryNew->Core.Key,
1873
false /* fWrite */);
1874
pdmacFileCacheReadFromEndpoint(pEntryNew);
1875
pdmacFileEpCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
1880
* There is not enough free space in the cache.
1881
* Pass the request directly to the I/O manager.
1883
LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
1885
pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
1886
&IoMemCtx, off, cbToRead,
1887
PDMACTASKFILETRANSFER_READ);
1890
/* Clip read size if neccessary. */
1891
PPDMACFILECACHEENTRY pEntryAbove;
1892
pdmacFileEpCacheGetCacheBestFitEntryByOffset(pEndpointCache, off,
1893
&pEntryAbove, NULL);
1897
if (off + (RTFOFF)cbRead > pEntryAbove->Core.Key)
1898
cbToRead = pEntryAbove->Core.Key - off;
1902
pdmacFileEpCacheEntryRelease(pEntryAbove);
1908
pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
1909
&IoMemCtx, off, cbToRead,
1910
PDMACTASKFILETRANSFER_READ);
1916
ASMAtomicWriteBool(&pTask->fCompleted, false);
1918
if (ASMAtomicReadS32(&pTask->cbTransferLeft) == 0
1919
&& !ASMAtomicXchgBool(&pTask->fCompleted, true))
1920
pdmR3AsyncCompletionCompleteTask(&pTask->Core, VINF_SUCCESS, false);
1922
rc = VINF_AIO_TASK_PENDING;
1924
LogFlowFunc((": Leave rc=%Rrc\n", rc));
1930
* Writes the given data to the endpoint using the cache if possible.
1932
* @returns VBox status code.
1933
* @param pEndpoint The endpoint to write to.
1934
* @param pTask The task structure used as identifier for this request.
1935
* @param off The offset to start writing to
1936
* @param paSegments Pointer to the array holding the source buffers.
1937
* @param cSegments Number of segments in the array.
1938
* @param cbWrite Number of bytes to write.
1940
int pdmacFileEpCacheWrite(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
1941
RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
1944
int rc = VINF_SUCCESS;
1945
PPDMACFILEENDPOINTCACHE pEndpointCache = &pEndpoint->DataCache;
1946
PPDMACFILECACHEGLOBAL pCache = pEndpointCache->pCache;
1947
PPDMACFILECACHEENTRY pEntry;
1949
LogFlowFunc((": pEndpoint=%#p{%s} pTask=%#p off=%RTfoff paSegments=%#p cSegments=%u cbWrite=%u\n",
1950
pEndpoint, pEndpoint->Core.pszUri, pTask, off, paSegments, cSegments, cbWrite));
1952
/* Set to completed to make sure that the task is valid while we access it. */
1953
ASMAtomicWriteBool(&pTask->fCompleted, true);
1955
/* Init the I/O memory context */
1956
PDMIOMEMCTX IoMemCtx;
1957
pdmIoMemCtxInit(&IoMemCtx, paSegments, cSegments);
1963
pEntry = pdmacFileEpCacheGetCacheEntryByOffset(pEndpointCache, off);
1967
/* Write the data into the entry and mark it as dirty */
1968
AssertPtr(pEntry->pList);
1970
RTFOFF OffDiff = off - pEntry->Core.Key;
1972
AssertMsg(off >= pEntry->Core.Key,
1973
("Overflow in calculation off=%RTfoff OffsetAligned=%RTfoff\n",
1974
off, pEntry->Core.Key));
1976
cbToWrite = RT_MIN(pEntry->cbData - OffDiff, cbWrite);
1977
cbWrite -= cbToWrite;
1980
STAM_COUNTER_INC(&pCache->cHits);
1982
STAM_COUNTER_INC(&pCache->cPartialHits);
1984
STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
1986
/* Ghost lists contain no data. */
1987
if ( (pEntry->pList == &pCache->LruRecentlyUsedIn)
1988
|| (pEntry->pList == &pCache->LruFrequentlyUsed))
1990
/* Check if the entry is dirty. */
1991
if(pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
1992
PDMACFILECACHE_ENTRY_IS_DIRTY,
1995
/* If it is dirty but not in progrss just update the data. */
1996
if (!(pEntry->fFlags & PDMACFILECACHE_ENTRY_IO_IN_PROGRESS))
1998
pdmacFileEpCacheCopyFromIoMemCtx(&IoMemCtx,
1999
pEntry->pbData + OffDiff,
2001
ASMAtomicSubS32(&pTask->cbTransferLeft, cbToWrite);
2005
/* The data isn't written to the file yet */
2006
pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
2010
STAM_COUNTER_INC(&pEndpointCache->StatWriteDeferred);
2013
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
2015
else /* Dirty bit not set */
2018
* Check if a read is in progress for this entry.
2019
* We have to defer processing in that case.
2021
if(pdmacFileEpCacheEntryFlagIsSetClearAcquireLock(pEndpointCache, pEntry,
2022
PDMACFILECACHE_ENTRY_IO_IN_PROGRESS,
2025
pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
2029
STAM_COUNTER_INC(&pEndpointCache->StatWriteDeferred);
2030
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
2032
else /* I/O in progress flag not set */
2034
/* Write as much as we can into the entry and update the file. */
2035
pdmacFileEpCacheCopyFromIoMemCtx(&IoMemCtx,
2036
pEntry->pbData + OffDiff,
2038
ASMAtomicSubS32(&pTask->cbTransferLeft, cbToWrite);
2040
bool fCommit = pdmacFileCacheAddDirtyEntry(pEndpointCache, pEntry);
2042
pdmacFileCacheCommitDirtyEntries(pCache);
2044
} /* Dirty bit not set */
2046
/* Move this entry to the top position */
2047
if (pEntry->pList == &pCache->LruFrequentlyUsed)
2049
pdmacFileCacheLockEnter(pCache);
2050
pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2051
pdmacFileCacheLockLeave(pCache);
2054
pdmacFileEpCacheEntryRelease(pEntry);
2056
else /* Entry is on the ghost list */
2058
uint8_t *pbBuffer = NULL;
2060
pdmacFileCacheLockEnter(pCache);
2061
pdmacFileCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
2062
bool fEnough = pdmacFileCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
2066
/* Move the entry to Am and fetch it to the cache. */
2067
pdmacFileCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
2068
pdmacFileCacheAdd(pCache, pEntry->cbData);
2069
pdmacFileCacheLockLeave(pCache);
2072
pEntry->pbData = pbBuffer;
2074
pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
2075
AssertPtr(pEntry->pbData);
2077
pdmacFileEpCacheEntryWaitersAdd(pEntry, pTask,
2081
STAM_COUNTER_INC(&pEndpointCache->StatWriteDeferred);
2082
pdmacFileCacheReadFromEndpoint(pEntry);
2084
/* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
2085
pdmacFileEpCacheEntryRelease(pEntry);
2089
RTSemRWRequestWrite(pEndpointCache->SemRWEntries, RT_INDEFINITE_WAIT);
2090
STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
2091
RTAvlrFileOffsetRemove(pEndpointCache->pTree, pEntry->Core.Key);
2092
STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
2093
RTSemRWReleaseWrite(pEndpointCache->SemRWEntries);
2095
pdmacFileCacheLockLeave(pCache);
2098
pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
2099
&IoMemCtx, off, cbToWrite,
2100
PDMACTASKFILETRANSFER_WRITE);
2104
else /* No entry found */
2107
* No entry found. Try to create a new cache entry to store the data in and if that fails
2108
* write directly to the file.
2110
PPDMACFILECACHEENTRY pEntryNew = pdmacFileEpCacheEntryCreate(pEndpoint,
2116
cbWrite -= cbToWrite;
2120
RTFOFF offDiff = off - pEntryNew->Core.Key;
2122
STAM_COUNTER_INC(&pCache->cHits);
2125
* Check if it is possible to just write the data without waiting
2126
* for it to get fetched first.
2128
if (!offDiff && pEntryNew->cbData == cbToWrite)
2130
pdmacFileEpCacheCopyFromIoMemCtx(&IoMemCtx,
2133
ASMAtomicSubS32(&pTask->cbTransferLeft, cbToWrite);
2135
bool fCommit = pdmacFileCacheAddDirtyEntry(pEndpointCache, pEntryNew);
2137
pdmacFileCacheCommitDirtyEntries(pCache);
2138
STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
2142
/* Defer the write and fetch the data from the endpoint. */
2143
pdmacFileEpCacheEntryWaitersAdd(pEntryNew, pTask,
2147
STAM_COUNTER_INC(&pEndpointCache->StatWriteDeferred);
2148
pdmacFileCacheReadFromEndpoint(pEntryNew);
2151
pdmacFileEpCacheEntryRelease(pEntryNew);
2156
* There is not enough free space in the cache.
2157
* Pass the request directly to the I/O manager.
2159
LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
2161
STAM_COUNTER_INC(&pCache->cMisses);
2163
pdmacFileEpCacheRequestPassthrough(pEndpoint, pTask,
2164
&IoMemCtx, off, cbToWrite,
2165
PDMACTASKFILETRANSFER_WRITE);
2172
ASMAtomicWriteBool(&pTask->fCompleted, false);
2174
if (ASMAtomicReadS32(&pTask->cbTransferLeft) == 0
2175
&& !ASMAtomicXchgBool(&pTask->fCompleted, true))
2176
pdmR3AsyncCompletionCompleteTask(&pTask->Core, VINF_SUCCESS, false);
2178
rc = VINF_AIO_TASK_PENDING;
2180
LogFlowFunc((": Leave rc=%Rrc\n", rc));
2185
int pdmacFileEpCacheFlush(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
2187
int rc = VINF_SUCCESS;
2189
LogFlowFunc((": pEndpoint=%#p{%s}\n", pEndpoint, pEndpoint->Core.pszUri));
2191
/* Commit dirty entries in the cache. */
2192
pdmacFileCacheEndpointCommit(&pEndpoint->DataCache);
2194
LogFlowFunc((": Leave rc=%Rrc\n", rc));
added
removed
src/VBox/VMM/PDMAsyncCompletionFileCache.cpp
