Index: trunk/src/VBox/VMM/MMUkHeap.cpp
===================================================================
--- trunk/src/VBox/VMM/MMHeap.cpp	(revision 18287)
+++ trunk/src/VBox/VMM/MMUkHeap.cpp	(revision 18792)
@@ -1,9 +1,9 @@
 /* $Id$ */
 /** @file
- * MM - Memory Manager - Heap.
+ * MM - Memory Manager - Ring-3 Heap with kernel accessible mapping.
  */
 
 /*
- * Copyright (C) 2006-2007 Sun Microsystems, Inc.
+ * Copyright (C) 2006-2009 Sun Microsystems, Inc.
  *
  * This file is part of VirtualBox Open Source Edition (OSE), as
@@ -27,5 +27,4 @@
 #include <VBox/mm.h>
 #include <VBox/stam.h>
-#include <VBox/pgm.h>
 #include "MMInternal.h"
 #include <VBox/vm.h>
@@ -35,7 +34,7 @@
 #include <VBox/log.h>
 
-#include <iprt/alloc.h>
 #include <iprt/assert.h>
 #include <iprt/string.h>
+#include <iprt/heap.h>
 
 
@@ -43,10 +42,13 @@
 *   Internal Functions                                                         *
 *******************************************************************************/
-static void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero);
-
-
-
-/**
- * Allocate and initialize a heap structure and it's associated substructures.
+static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr);
+
+
+
+/**
+ * Create a User-kernel heap.
+ *
+ * This does not require SUPLib to be initialized as we'll lazily allocate the
+ * kernel accessible memory on the first alloc call.
  *
  * @returns VBox status.
@@ -54,7 +56,7 @@
  * @param   ppHeap  Where to store the heap pointer.
  */
-int mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap)
-{
-    PMMHEAP pHeap = (PMMHEAP)RTMemAllocZ(sizeof(MMHEAP) + sizeof(MMHEAPSTAT));
+int mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap)
+{
+    PMMUKHEAP pHeap = (PMMUKHEAP)MMR3HeapAllocZU(pUVM, MM_TAG_MM, sizeof(MMUKHEAP));
     if (pHeap)
     {
@@ -66,15 +68,14 @@
              */
             pHeap->pUVM = pUVM;
-            pHeap->Stat.pHeap = pHeap;
-#ifdef MMR3HEAP_WITH_STATISTICS
-            PMMHEAPSTAT pStat = &pHeap->Stat;
-            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFrees",           STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
+#ifdef MMUKHEAP_WITH_STATISTICS
+            PMMUKHEAPSTAT pStat = &pHeap->Stat;
+            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFrees",           STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
             STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
-                                                                        STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
-            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
-            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
+                                                                        STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
 #endif
             *ppHeap = pHeap;
@@ -82,5 +83,5 @@
         }
         AssertRC(rc);
-        RTMemFree(pHeap);
+        MMR3HeapFree(pHeap);
     }
     AssertMsgFailed(("failed to allocate heap structure\n"));
@@ -90,9 +91,9 @@
 
 /**
- * Destroy a heap.
+ * Destroy a User-kernel heap.
  *
  * @param   pHeap   Heap handle.
  */
-void mmR3HeapDestroy(PMMHEAP pHeap)
+void mmR3UkHeapDestroy(PMMUKHEAP pHeap)
 {
     /*
@@ -103,43 +104,15 @@
 
     /*
-     * Walk the node list and free all the memory.
-     */
-    PMMHEAPHDR  pHdr = pHeap->pHead;
-    while (pHdr)
-    {
-        void *pv = pHdr;
-        pHdr = pHdr->pNext;
-        RTMemFree(pv);
-    }
-
-    /*
-     * Free the stat nodes.
-     */
-    /** @todo free all nodes in a AVL tree. */
-    RTMemFree(pHeap);
-}
-
-
-/**
- * Allocate memory associating it with the VM for collective cleanup.
- *
- * The memory will be allocated from the default heap but a header
- * is added in which we keep track of which VM it belongs to and chain
- * all the allocations together so they can be freed in one go.
- *
- * This interface is typically used for memory block which will not be
- * freed during the life of the VM.
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAllocU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
-{
-    Assert(pUVM->mm.s.pHeap);
-    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
+     * Walk the sub-heaps and free them.
+     */
+    while (pHeap->pSubHeapHead)
+    {
+        PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+        pHeap->pSubHeapHead = pSubHeap->pNext;
+        SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
+        //MMR3HeapFree(pSubHeap); - rely on the automatic cleanup.
+    }
+    //MMR3HeapFree(pHeap->stats);
+    //MMR3HeapFree(pHeap);
 }
 
@@ -161,37 +134,14 @@
  *                      identify how memory is used by the VM.
  * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize)
-{
-    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
-}
-
-
-/**
- * Same as MMR3HeapAllocU().
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    Assert(pUVM->mm.s.pHeap);
-    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Same as MMR3HeapAlloc().
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(void *) MMR3UkHeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
+{
+    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
+}
+
+
+/**
+ * Same as MMR3UkHeapAlloc().
  *
  * @returns Pointer to allocated memory.
@@ -202,8 +152,9 @@
  * @param   cbSize      Size of the block.
  * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(int) MMR3UkHeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
+{
+    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
     if (pv)
     {
@@ -216,21 +167,5 @@
 
 /**
- * Same as MMR3HeapAlloc() only the memory is zeroed.
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAllocZU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
-{
-    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
-}
-
-
-/**
- * Same as MMR3HeapAlloc() only the memory is zeroed.
+ * Same as MMR3UkHeapAlloc() only the memory is zeroed.
  *
  * @returns Pointer to allocated memory.
@@ -240,37 +175,14 @@
  *                      identify how memory is used by the VM.
  * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize)
-{
-    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
-}
-
-
-/**
- * Same as MMR3HeapAllocZ().
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocZExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    Assert(pUVM->mm.s.pHeap);
-    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Same as MMR3HeapAllocZ().
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(void *) MMR3UkHeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
+{
+    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
+}
+
+
+/**
+ * Same as MMR3UkHeapAllocZ().
  *
  * @returns Pointer to allocated memory.
@@ -281,8 +193,9 @@
  * @param   cbSize      Size of the block.
  * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(int) MMR3UkHeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
+{
+    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
     if (pv)
     {
@@ -291,4 +204,37 @@
     }
     return VERR_NO_MEMORY;
+}
+
+
+/***
+ * Worker for mmR3UkHeapAlloc that creates and adds a new sub-heap.
+ *
+ * @returns Pointer to the new sub-heap.
+ * @param   pHeap       The heap
+ * @param   cbSubHeap   The size of the sub-heap.
+ */
+static PMMUKHEAPSUB mmR3UkHeapAddSubHeap(PMMUKHEAP pHeap, size_t cbSubHeap)
+{
+    PMMUKHEAPSUB pSubHeap = (PMMUKHEAPSUB)MMR3HeapAllocU(pHeap->pUVM, MM_TAG_MM/*_UK_HEAP*/, sizeof(*pSubHeap));
+    if (pSubHeap)
+    {
+        pSubHeap->cb = cbSubHeap;
+        int rc = SUPR3PageAllocEx(pSubHeap->cb >> PAGE_SHIFT, 0, &pSubHeap->pv, &pSubHeap->pvR0, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTHeapSimpleInit(&pSubHeap->hSimple, pSubHeap->pv, pSubHeap->cb);
+            if (RT_SUCCESS(rc))
+            {
+                pSubHeap->pNext = pHeap->pSubHeapHead;
+                pHeap->pSubHeapHead = pSubHeap;
+                return pSubHeap;
+            }
+
+            /* bail out */
+            SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
+        }
+        MMR3HeapFree(pSubHeap);
+    }
+    return NULL;
 }
 
@@ -302,26 +248,25 @@
  *                      basis in addition to a global one. Thus we can easily
  *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
+ * @param   cb          Size of the block.
  * @param   fZero       Whether or not to zero the memory block.
- */
-void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
-{
-#ifdef MMR3HEAP_WITH_STATISTICS
+ * @param   pR0Ptr      Where to return the ring-0 pointer.
+ */
+static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr)
+{
+    if (pR0Ptr)
+        *pR0Ptr = NIL_RTR0PTR;
     RTCritSectEnter(&pHeap->Lock);
 
+#ifdef MMUKHEAP_WITH_STATISTICS
     /*
      * Find/alloc statistics nodes.
      */
     pHeap->Stat.cAllocations++;
-    PMMHEAPSTAT pStat = (PMMHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+    PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
     if (pStat)
-    {
         pStat->cAllocations++;
-
-        RTCritSectLeave(&pHeap->Lock);
-    }
     else
     {
-        pStat = (PMMHEAPSTAT)RTMemAllocZ(sizeof(MMHEAPSTAT));
+        pStat = (PMMUKHEAPSTAT)MMR3HeapAllocZU(pHeap->pUVM, MM_TAG_MM, sizeof(MMUKHEAPSTAT));
         if (!pStat)
         {
@@ -332,20 +277,18 @@
         }
         pStat->Core.Key = (AVLULKEY)enmTag;
-        pStat->pHeap    = pHeap;
         RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
 
         pStat->cAllocations++;
-        RTCritSectLeave(&pHeap->Lock);
 
         /* register the statistics */
         PUVM pUVM = pHeap->pUVM;
         const char *pszTag = mmR3GetTagName(enmTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/R3Heap/%s", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.",        "/MM/R3Heap/%s/cAllocations", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.",      "/MM/R3Heap/%s/cReallocations", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.",         "/MM/R3Heap/%s/cFrees", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/R3Heap/%s/cFailures", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/R3Heap/%s/cbAllocated", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/R3Heap/%s/cbFreed", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/UkHeap/%s", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.",      "/MM/UkHeap/%s/cAllocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.",    "/MM/UkHeap/%s/cReallocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.",       "/MM/UkHeap/%s/cFrees", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/UkHeap/%s/cFailures", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/UkHeap/%s/cbAllocated", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/UkHeap/%s/cbFreed", pszTag);
     }
 #endif
@@ -354,12 +297,11 @@
      * Validate input.
      */
-    if (cbSize == 0)
-    {
-#ifdef MMR3HEAP_WITH_STATISTICS
-        RTCritSectEnter(&pHeap->Lock);
+    if (cb == 0)
+    {
+#ifdef MMUKHEAP_WITH_STATISTICS
         pStat->cFailures++;
         pHeap->Stat.cFailures++;
+#endif
         RTCritSectLeave(&pHeap->Lock);
-#endif
         return NULL;
     }
@@ -368,278 +310,78 @@
      * Allocate heap block.
      */
-    cbSize = RT_ALIGN_Z(cbSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
-    PMMHEAPHDR  pHdr = (PMMHEAPHDR)(fZero ? RTMemAllocZ(cbSize) : RTMemAlloc(cbSize));
-    if (!pHdr)
-    {
-        AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cbSize, enmTag, &enmTag));
-#ifdef MMR3HEAP_WITH_STATISTICS
-        RTCritSectEnter(&pHeap->Lock);
-        pStat->cFailures++;
-        pHeap->Stat.cFailures++;
-        RTCritSectLeave(&pHeap->Lock);
-#endif
-        return NULL;
-    }
-    Assert(!((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)));
-
-    RTCritSectEnter(&pHeap->Lock);
-
-    /*
-     * Init and link in the header.
-     */
-    pHdr->pNext = NULL;
-    pHdr->pPrev = pHeap->pTail;
-    if (pHdr->pPrev)
-        pHdr->pPrev->pNext = pHdr;
-    else
-        pHeap->pHead = pHdr;
-    pHeap->pTail = pHdr;
-#ifdef MMR3HEAP_WITH_STATISTICS
-    pHdr->pStat  = pStat;
-#else
-    pHdr->pStat  = &pHeap->Stat;
-#endif
-    pHdr->cbSize = cbSize;
+    cb = RT_ALIGN_Z(cb, MMUKHEAP_SIZE_ALIGNMENT);
+    void *pv = NULL;
+    PMMUKHEAPSUB pSubHeapPrev = NULL;
+    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+    while (pSubHeap)
+    {
+        if (fZero)
+            pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        else
+            pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        if (pv)
+        {
+            /* Move the sub-heap with free memory to the head. */
+            if (pSubHeapPrev)
+            {
+                pSubHeapPrev->pNext = pSubHeap->pNext;
+                pSubHeap->pNext = pHeap->pSubHeapHead;
+                pHeap->pSubHeapHead = pSubHeap;
+            }
+            break;
+        }
+        pSubHeapPrev = pSubHeap;
+        pSubHeap = pSubHeap->pNext;
+    }
+    if (RT_UNLIKELY(!pv))
+    {
+        /*
+         * Add another sub-heap.
+         */
+        pSubHeap = mmR3UkHeapAddSubHeap(pHeap, RT_MAX(RT_ALIGN_Z(cb, PAGE_SIZE) + PAGE_SIZE * 16, _256K));
+        if (pSubHeap)
+        {
+            if (fZero)
+                pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+            else
+                pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        }
+        if (RT_UNLIKELY(!pv))
+        {
+            AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cb, enmTag, &enmTag));
+#ifdef MMUKHEAP_WITH_STATISTICS
+            pStat->cFailures++;
+            pHeap->Stat.cFailures++;
+#endif
+            RTCritSectLeave(&pHeap->Lock);
+            return NULL;
+        }
+    }
 
     /*
      * Update statistics
      */
-#ifdef MMR3HEAP_WITH_STATISTICS
-    pStat->cbAllocated          += cbSize;
-    pStat->cbCurAllocated       += cbSize;
-    pHeap->Stat.cbAllocated     += cbSize;
-    pHeap->Stat.cbCurAllocated  += cbSize;
-#endif
-
+#ifdef MMUKHEAP_WITH_STATISTICS
+    size_t cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
+    pStat->cbAllocated          += cbActual;
+    pStat->cbCurAllocated       += cbActual;
+    pHeap->Stat.cbAllocated     += cbActual;
+    pHeap->Stat.cbCurAllocated  += cbActual;
+#endif
+
+    if (pR0Ptr)
+        *pR0Ptr = (uintptr_t)pv - (uintptr_t)pSubHeap->pv + pSubHeap->pvR0;
     RTCritSectLeave(&pHeap->Lock);
-
-    return pHdr + 1;
-}
-
-
-/**
- * Reallocate memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
- *
- * @returns Pointer to reallocated memory.
- * @param   pv          Pointer to the memory block to reallocate.
- *                      Must not be NULL!
- * @param   cbNewSize   New block size.
- */
-VMMR3DECL(void *) MMR3HeapRealloc(void *pv, size_t cbNewSize)
-{
-    AssertMsg(pv, ("Invalid pointer pv=%p\n", pv));
-    if (!pv)
-        return NULL;
-
-    /*
-     * If newsize is zero then this is a free.
-     */
-    if (!cbNewSize)
-    {
-        MMR3HeapFree(pv);
-        return NULL;
-    }
-
-    /*
-     * Validate header.
-     */
-    PMMHEAPHDR  pHdr = (PMMHEAPHDR)pv - 1;
-    if (    pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1)
-        ||  (uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))
-    {
-        AssertMsgFailed(("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
-        return NULL;
-    }
-    Assert(pHdr->pStat != NULL);
-    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
-    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
-
-    PMMHEAP pHeap = pHdr->pStat->pHeap;
-
-#ifdef MMR3HEAP_WITH_STATISTICS
-    RTCritSectEnter(&pHeap->Lock);
-    pHdr->pStat->cReallocations++;
-    pHeap->Stat.cReallocations++;
-    RTCritSectLeave(&pHeap->Lock);
-#endif
-
-    /*
-     * Rellocate the block.
-     */
-    cbNewSize = RT_ALIGN_Z(cbNewSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
-    PMMHEAPHDR pHdrNew = (PMMHEAPHDR)RTMemRealloc(pHdr, cbNewSize);
-    if (!pHdrNew)
-    {
-#ifdef MMR3HEAP_WITH_STATISTICS
-        RTCritSectEnter(&pHeap->Lock);
-        pHdr->pStat->cFailures++;
-        pHeap->Stat.cFailures++;
-        RTCritSectLeave(&pHeap->Lock);
-#endif
-        return NULL;
-    }
-
-    /*
-     * Update pointers.
-     */
-    if (pHdrNew != pHdr)
-    {
-        RTCritSectEnter(&pHeap->Lock);
-        if (pHdrNew->pPrev)
-            pHdrNew->pPrev->pNext = pHdrNew;
-        else
-            pHeap->pHead = pHdrNew;
-
-        if (pHdrNew->pNext)
-            pHdrNew->pNext->pPrev = pHdrNew;
-        else
-            pHeap->pTail = pHdrNew;
-        RTCritSectLeave(&pHeap->Lock);
-    }
-
-    /*
-     * Update statistics.
-     */
-#ifdef MMR3HEAP_WITH_STATISTICS
-    RTCritSectEnter(&pHeap->Lock);
-    pHdrNew->pStat->cbAllocated += cbNewSize - pHdrNew->cbSize;
-    pHeap->Stat.cbAllocated += cbNewSize - pHdrNew->cbSize;
-    RTCritSectLeave(&pHeap->Lock);
-#endif
-
-    pHdrNew->cbSize = cbNewSize;
-
-    return pHdrNew + 1;
-}
-
-
-/**
- * Duplicates the specified string.
- *
- * @returns Pointer to the duplicate.
- * @returns NULL on failure or when input NULL.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   psz         The string to duplicate. NULL is allowed.
- */
-VMMR3DECL(char *) MMR3HeapStrDupU(PUVM pUVM, MMTAG enmTag, const char *psz)
-{
-    if (!psz)
-        return NULL;
-    AssertPtr(psz);
-
-    size_t cch = strlen(psz) + 1;
-    char *pszDup = (char *)MMR3HeapAllocU(pUVM, enmTag, cch);
-    if (pszDup)
-        memcpy(pszDup, psz, cch);
-    return pszDup;
-}
-
-
-/**
- * Duplicates the specified string.
- *
- * @returns Pointer to the duplicate.
- * @returns NULL on failure or when input NULL.
+    return pv;
+}
+
+
+/**
+ * Releases memory allocated with MMR3UkHeapAlloc() and MMR3UkHeapAllocZ()
+ *
  * @param   pVM         The VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   psz         The string to duplicate. NULL is allowed.
- */
-VMMR3DECL(char *) MMR3HeapStrDup(PVM pVM, MMTAG enmTag, const char *psz)
-{
-    return MMR3HeapStrDupU(pVM->pUVM, enmTag, psz);
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pVM         The VM
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   ...         Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintf(PVM pVM, MMTAG enmTag, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    char *psz = MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
-    va_end(va);
-    return psz;
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   ...         Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintfU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    char *psz = MMR3HeapAPrintfVU(pUVM, enmTag, pszFormat, va);
-    va_end(va);
-    return psz;
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pVM         The VM
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   va          Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintfV(PVM pVM, MMTAG enmTag, const char *pszFormat, va_list va)
-{
-    return MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   va          Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintfVU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, va_list va)
-{
-    /*
-     * The lazy bird way.
-     */
-    char *psz;
-    int cch = RTStrAPrintfV(&psz, pszFormat, va);
-    if (cch < 0)
-        return NULL;
-    Assert(psz[cch] == '\0');
-    char *pszRet = (char *)MMR3HeapAllocU(pUVM, enmTag, cch + 1);
-    if (pszRet)
-        memcpy(pszRet, psz, cch + 1);
-    RTStrFree(psz);
-    return pszRet;
-}
-
-
-/**
- * Releases memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
- *
  * @param   pv          Pointer to the memory block to free.
  */
-VMMR3DECL(void) MMR3HeapFree(void *pv)
+VMMR3DECL(void) MMR3UkHeapFree(PVM pVM, void *pv, MMTAG enmTag)
 {
     /* Ignore NULL pointers. */
@@ -647,149 +389,38 @@
         return;
 
-    /*
-     * Validate header.
-     */
-    PMMHEAPHDR  pHdr = (PMMHEAPHDR)pv - 1;
-    if (    pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1)
-        ||  (uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))
-    {
-        AssertMsgFailed(("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
-        return;
-    }
-    Assert(pHdr->pStat != NULL);
-    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
-    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
-
-    /*
-     * Update statistics
-     */
-    PMMHEAP pHeap = pHdr->pStat->pHeap;
+    PMMUKHEAP pHeap = pVM->pUVM->mm.s.pUkHeap;
     RTCritSectEnter(&pHeap->Lock);
 
-#ifdef MMR3HEAP_WITH_STATISTICS
-    pHdr->pStat->cFrees++;
-    pHeap->Stat.cFrees++;
-    pHdr->pStat->cbFreed            += pHdr->cbSize;
-    pHeap->Stat.cbFreed             += pHdr->cbSize;
-    pHdr->pStat->cbCurAllocated     -= pHdr->cbSize;
-    pHeap->Stat.cbCurAllocated      -= pHdr->cbSize;
-#endif
-
-    /*
-     * Unlink it.
-     */
-    if (pHdr->pPrev)
-        pHdr->pPrev->pNext = pHdr->pNext;
-    else
-        pHeap->pHead = pHdr->pNext;
-
-    if (pHdr->pNext)
-        pHdr->pNext->pPrev = pHdr->pPrev;
-    else
-        pHeap->pTail = pHdr->pPrev;
-
-    RTCritSectLeave(&pHeap->Lock);
-
-    /*
-     * Free the memory.
-     */
-    RTMemFree(pHdr);
-}
-
-
-/**
- * Gets the string name of a memory tag.
- *
- * @returns name of enmTag.
- * @param   enmTag      The tag.
- */
-const char *mmR3GetTagName(MMTAG enmTag)
-{
-    switch (enmTag)
-    {
-        #define TAG2STR(tag) case MM_TAG_##tag: return #tag
-
-        TAG2STR(CFGM);
-        TAG2STR(CFGM_BYTES);
-        TAG2STR(CFGM_STRING);
-        TAG2STR(CFGM_USER);
-
-        TAG2STR(CSAM);
-        TAG2STR(CSAM_PATCH);
-
-        TAG2STR(DBGF);
-        TAG2STR(DBGF_INFO);
-        TAG2STR(DBGF_LINE);
-        TAG2STR(DBGF_LINE_DUP);
-        TAG2STR(DBGF_MODULE);
-        TAG2STR(DBGF_OS);
-        TAG2STR(DBGF_STACK);
-        TAG2STR(DBGF_SYMBOL);
-        TAG2STR(DBGF_SYMBOL_DUP);
-
-        TAG2STR(EM);
-
-        TAG2STR(IOM);
-        TAG2STR(IOM_STATS);
-
-        TAG2STR(MM);
-        TAG2STR(MM_LOOKUP_GUEST);
-        TAG2STR(MM_LOOKUP_PHYS);
-        TAG2STR(MM_LOOKUP_VIRT);
-        TAG2STR(MM_PAGE);
-
-        TAG2STR(PARAV);
-
-        TAG2STR(PATM);
-        TAG2STR(PATM_PATCH);
-
-        TAG2STR(PDM);
-        TAG2STR(PDM_DEVICE);
-        TAG2STR(PDM_DEVICE_USER);
-        TAG2STR(PDM_DRIVER);
-        TAG2STR(PDM_DRIVER_USER);
-        TAG2STR(PDM_USB);
-        TAG2STR(PDM_USB_USER);
-        TAG2STR(PDM_LUN);
-        TAG2STR(PDM_QUEUE);
-        TAG2STR(PDM_THREAD);
-        TAG2STR(PDM_ASYNC_COMPLETION);
-
-        TAG2STR(PGM);
-        TAG2STR(PGM_CHUNK_MAPPING);
-        TAG2STR(PGM_HANDLERS);
-        TAG2STR(PGM_MAPPINGS);
-        TAG2STR(PGM_PHYS);
-        TAG2STR(PGM_POOL);
-
-        TAG2STR(REM);
-
-        TAG2STR(SELM);
-
-        TAG2STR(SSM);
-
-        TAG2STR(STAM);
-
-        TAG2STR(TM);
-
-        TAG2STR(TRPM);
-
-        TAG2STR(VM);
-        TAG2STR(VM_REQ);
-
-        TAG2STR(VMM);
-
-        TAG2STR(HWACCM);
-
-        #undef TAG2STR
-
-        default:
-        {
-            AssertMsgFailed(("Unknown tag %d! forgot to add it to the switch?\n", enmTag));
-            static char sz[48];
-            RTStrPrintf(sz, sizeof(sz), "%d", enmTag);
-            return sz;
-        }
-    }
-}
-
+    /*
+     * Find the sub-heap and block
+     */
+#ifdef MMUKHEAP_WITH_STATISTICS
+    size_t cbActual = 0;
+#endif
+    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+    while (pSubHeap)
+    {
+        if ((uintptr_t)pv - (uintptr_t)pSubHeap->pv < pSubHeap->cb)
+        {
+#ifdef MMUKHEAP_WITH_STATISTICS
+            cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
+            PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+            if (pStat)
+            {
+                pStat->cFrees++;
+                pStat->cbCurAllocated     -= cbActual;
+                pStat->cbFreed            += cbActual;
+            }
+            pHeap->Stat.cFrees++;
+            pHeap->Stat.cbFreed           += cbActual;
+            pHeap->Stat.cbCurAllocated    -= cbActual;
+#endif
+            RTHeapSimpleFree(pSubHeap->hSimple, pv);
+
+            RTCritSectLeave(&pHeap->Lock);
+            return;
+        }
+    }
+    AssertMsgFailed(("pv=%p\n", pv));
+}
+
