From f1950fa9330b1bc1c8e2075638f2556a5f6fffbd Mon Sep 17 00:00:00 2001 From: "Eric J. Bowersox" Date: Sun, 9 Jun 2013 19:50:14 -0600 Subject: [PATCH] completed porting forward the chunks and base layers of the heap management code --- include/comrogue/heap.h | 8 +- include/comrogue/intlib.h | 1 + include/comrogue/types.h | 4 + kernel/lib/Makefile | 4 +- kernel/lib/heap_base.c | 93 ++++++++- kernel/lib/heap_chunks.c | 362 +++++++++++++++++++++++++++++++++++- kernel/lib/heap_internals.h | 75 +++++++- kernel/lib/heap_rtree.c | 246 ++++++++++++++++++++++++ kernel/lib/heap_toplevel.c | 22 ++- kernel/lib/heap_utils.c | 148 +++++++++++++++ kernel/lib/intlib.c | 24 +++ 11 files changed, 964 insertions(+), 23 deletions(-) create mode 100644 kernel/lib/heap_rtree.c create mode 100644 kernel/lib/heap_utils.c diff --git a/include/comrogue/heap.h b/include/comrogue/heap.h index 0eb7945..d8ce6d5 100644 --- a/include/comrogue/heap.h +++ b/include/comrogue/heap.h @@ -50,7 +50,8 @@ typedef struct tagRAWHEAPDATA UINT32 opaque[32]; /* opaque data, do not modify */ } RAWHEAPDATA, *PRAWHEAPDATA; -typedef void (*PFNRAWHEAPDATAFREE)(PRAWHEAPDATA prhd); /* function that optionally frees the heap data */ +typedef void (*PFNRAWHEAPDATAFREE)(PRAWHEAPDATA); /* function that optionally frees the heap data */ +typedef void (*PFNHEAPABORT)(PVOID); /* function called to abort on serious error */ /*-------------------- * External functions @@ -61,8 +62,9 @@ typedef void (*PFNRAWHEAPDATAFREE)(PRAWHEAPDATA prhd); /* function that optional CDECL_BEGIN -extern HRESULT HeapCreate(PRAWHEAPDATA prhd, PFNRAWHEAPDATAFREE pfnFree, IChunkAllocator *pChunkAllocator, - IMutexFactory *pMutexFactory, UINT32 nChunkBits, IMalloc **ppHeap); +extern HRESULT HeapCreate(PRAWHEAPDATA prhd, PFNRAWHEAPDATAFREE pfnFree, PFNHEAPABORT pfnAbort, PVOID pvAbortArg, + IChunkAllocator *pChunkAllocator, IMutexFactory *pMutexFactory, UINT32 nChunkBits, + IMalloc **ppHeap); CDECL_END diff --git a/include/comrogue/intlib.h b/include/comrogue/intlib.h index 1cd56bf..addcfe5 100644 --- a/include/comrogue/intlib.h +++ b/include/comrogue/intlib.h @@ -49,6 +49,7 @@ typedef struct tagLDIV { CDECL_BEGIN extern HRESULT IntLDiv(PLDIV pResult, INT64 num, INT64 denom); +extern INT32 IntFirstSet(INT32 nMask); CDECL_END diff --git a/include/comrogue/types.h b/include/comrogue/types.h index fe1758e..b779231 100644 --- a/include/comrogue/types.h +++ b/include/comrogue/types.h @@ -59,6 +59,10 @@ #define INT64_BITS 64 #define UINT64_BITS 64 +#define LOG_PTRSIZE 2 /* log2(sizeof(void *)) */ +#define LOG_INTSIZE 2 /* log2(sizeof(int)) */ +#define LOG_INT64SIZE 3 /* log2(sizeof(long long)) */ + /* Boolean values */ #define TRUE 1 #define FALSE 0 diff --git a/kernel/lib/Makefile b/kernel/lib/Makefile index 8f6bb49..d51fb7e 100644 --- a/kernel/lib/Makefile +++ b/kernel/lib/Makefile @@ -32,8 +32,8 @@ MAKEFLAGS += -rR CRBASEDIR := $(abspath ../..) include $(CRBASEDIR)/armcompile.mk -LIB_OBJS = divide.o qdivrem.o heap_toplevel.o heap_base.o heap_chunks.o intlib.o objhelp.o objhelp_enumconn.o \ - objhelp_enumgeneric.o objhelp_fixedcp.o rbtree.o str.o strcopymem.o strcomparemem.o \ +LIB_OBJS = divide.o qdivrem.o heap_toplevel.o heap_base.o heap_chunks.o heap_rtree.o heap_utils.o intlib.o objhelp.o \ + objhelp_enumconn.o objhelp_enumgeneric.o objhelp_fixedcp.o rbtree.o str.o strcopymem.o strcomparemem.o \ strsetmem.o lib_guids.o all: kernel-lib.o diff --git a/kernel/lib/heap_base.c b/kernel/lib/heap_base.c index 9cebd2b..9922511 100644 --- a/kernel/lib/heap_base.c +++ b/kernel/lib/heap_base.c @@ -41,23 +41,53 @@ #include #include "heap_internals.h" +/*----------------------------------- + * Base memory allocation functions. + *----------------------------------- + */ + +/* + * Allocate a new chunk of memory for use by the base allocator, and point to it with the allocator pointers. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - szMinimum = Minimum number of bytes we need to allocate. + * + * Returns: + * - TRUE = Allocation failed. + * - FALSE = Allocation succeeded. + */ static BOOL base_alloc_new_chunk(PHEAPDATA phd, SIZE_T szMinimum) { - BOOL fZero = FALSE; /* do we need this zeroed? (no) */ SIZE_T szAdjusted; /* adjusted size to multiple of chunk size */ PVOID pvNewChunk; /* pointer to new chunk */ + PVOID pvTemp; /* temporary pointer */ + BOOL fZero = FALSE; /* do we need this zeroed? (no) */ szAdjusted = CHUNK_CEILING(phd, szMinimum); pvNewChunk = _HeapChunkAlloc(phd, szAdjusted, phd->szChunk, TRUE, &fZero); if (!pvNewChunk) return TRUE; /* allocation failed */ *((PPVOID)pvNewChunk) = phd->pvBasePages; + pvTemp = (PVOID)(((UINT_PTR)pvNewChunk) + sizeof(PVOID)); + *((SIZE_T *)pvTemp) = szAdjusted; phd->pvBasePages = pvNewChunk; - phd->pvBaseNext = (PVOID)(((UINT_PTR)pvNewChunk) + sizeof(PVOID)); + phd->pvBaseNext = (PVOID)(((UINT_PTR)pvNewChunk) + sizeof(PVOID) + sizeof(SIZE_T)); phd->pvBasePast = (PVOID)(((UINT_PTR)pvNewChunk) + szAdjusted); return FALSE; } +/* + * Allocate a chunk of memory from the base allocator. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - sz = Number of bytes to allocate. + * + * Returns: + * - NULL = Allocation failed. + * - Other = Pointer to the allocated block of memory. + */ PVOID _HeapBaseAlloc(PHEAPDATA phd, SIZE_T sz) { PVOID rc = NULL; /* return from this function */ @@ -68,8 +98,8 @@ PVOID _HeapBaseAlloc(PHEAPDATA phd, SIZE_T sz) IMutex_Lock(phd->pmtxBase); if (((UINT_PTR)(phd->pvBaseNext) + szAdjusted) > (UINT_PTR)(phd->pvBasePast)) { /* allocate new chunk if we don't have enough space for the allocation */ - if (base_alloc_new_chunk(phd, szAdjusted + sizeof(PVOID))) - goto error0; + if (base_alloc_new_chunk(phd, szAdjusted + sizeof(PVOID) + sizeof(SIZE_T))) + goto error0; /* failed */ } rc = phd->pvBaseNext; /* perform the allocation */ @@ -80,6 +110,16 @@ error0: return rc; } +/* + * Allocate a new EXTENT_NODE from the base allocator. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * + * Returns: + * - NULL = Allocation failed. + * - Other = Pointer to the new EXTENT_NODE. + */ PEXTENT_NODE _HeapBaseNodeAlloc(PHEAPDATA phd) { PEXTENT_NODE rc = NULL; /* return from this function */ @@ -97,6 +137,16 @@ PEXTENT_NODE _HeapBaseNodeAlloc(PHEAPDATA phd) return rc; } +/* + * Returns an EXTENT_NODE to the free list. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - pexn = Pointer to the EXTENT_NODE to be freed. + * + * Returns: + * Nothing. + */ void _HeapBaseNodeDeAlloc(PHEAPDATA phd, PEXTENT_NODE pexn) { /* add it to the free list */ @@ -106,6 +156,15 @@ void _HeapBaseNodeDeAlloc(PHEAPDATA phd, PEXTENT_NODE pexn) IMutex_Unlock(phd->pmtxBase); } +/* + * Set up the base allocator. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * + * Returns: + * Standard HRESULT success/failure indicator. + */ HRESULT _HeapBaseSetup(PHEAPDATA phd) { HRESULT hr = IMutexFactory_CreateMutex(phd->pMutexFactory, &(phd->pmtxBase)); @@ -115,8 +174,32 @@ HRESULT _HeapBaseSetup(PHEAPDATA phd) return S_OK; } +/* + * Shut down the base allocator. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * + * Returns: + * Nothing. + */ void _HeapBaseShutdown(PHEAPDATA phd) { - /* TODO */ + PVOID pChunk; /* the chunk we're working on */ + PVOID pNextChunk; /* the next chunk to be dealt with */ + PVOID pvTemp; /* temporary pointer */ + SIZE_T szChunk; /* size of allocated chunk */ + + /* All allocated "base" chunks come straight from the IChunkAllocator, so it's safe to free them this way. */ + pChunk = phd->pvBasePages; + while (pChunk) + { + pNextChunk = *((PPVOID)pChunk); + pvTemp = (PVOID)(((UINT_PTR)pChunk) + sizeof(PVOID)); + szChunk = *((SIZE_T *)pvTemp); + IChunkAllocator_FreeChunk(phd->pChunkAllocator, pChunk, szChunk); + pChunk = pNextChunk; + } + IUnknown_Release(phd->pmtxBase); } diff --git a/kernel/lib/heap_chunks.c b/kernel/lib/heap_chunks.c index 1c92b91..aa7534a 100644 --- a/kernel/lib/heap_chunks.c +++ b/kernel/lib/heap_chunks.c @@ -34,25 +34,42 @@ */ #include #include +#include #include #include #include #include #include "heap_internals.h" +#ifdef _H_THIS_FILE +#undef _H_THIS_FILE +_DECLARE_H_THIS_FILE +#endif + /*------------------------------------------------------------------------ * Functions driving the red-black trees that contain EXTENT_NODE objects *------------------------------------------------------------------------ */ +/* + * Compare two extent nodes, first by size, then by address. + * + * Parameters: + * - pexnA = First extent node to compare. + * - pexnB = Second extent node to compare. + * + * Returns: + * A value less than, equal to, or greater than 0 as pexnA is less than, equal to, or greater than pexnB. + */ static INT32 compare_sizeaddr(PEXTENT_NODE pexnA, PEXTENT_NODE pexnB) { + /* compare sizes first */ SIZE_T szA = pexnA->sz; SIZE_T szB = pexnB->sz; INT32 rc = (szA > szB) - (szA < szB); if (rc == 0) - { + { /* compare addresses */ UINT_PTR addrA = (UINT_PTR)(pexnA->pv); UINT_PTR addrB = (UINT_PTR)(pexnB->pv); rc = (addrA > addrB) - (addrA < addrB); @@ -60,6 +77,16 @@ static INT32 compare_sizeaddr(PEXTENT_NODE pexnA, PEXTENT_NODE pexnB) return rc; } +/* + * Compare two extent nodes by address. + * + * Parameters: + * - pexnA = First extent node to compare. + * - pexnB = Second extent node to compare. + * + * Returns: + * A value less than, equal to, or greater than 0 as pexnA is less than, equal to, or greater than pexnB. + */ static INT32 compare_addr(PEXTENT_NODE pexnA, PEXTENT_NODE pexnB) { UINT_PTR addrA = (UINT_PTR)(pexnA->pv); @@ -67,21 +94,57 @@ static INT32 compare_addr(PEXTENT_NODE pexnA, PEXTENT_NODE pexnB) return (addrA > addrB) - (addrA < addrB); } +/* + * Given a pointer to an extent node, returns a pointer to its RBTREENODE for the size-address tree. + * + * Parameters: + * - pexn = Pointer to the extent node. + * + * Returns: + * Pointer to the extent node's "size-address" RBTREENODE. + */ static PRBTREENODE get_sizeaddr_node(PEXTENT_NODE pexn) { return &(pexn->rbtnSizeAddress); } +/* + * Given a pointer to an extent node, returns a pointer to its RBTREENODE for the address tree. + * + * Parameters: + * - pexn = Pointer to the extent node. + * + * Returns: + * Pointer to the extent node's "address" RBTREENODE. + */ static PRBTREENODE get_addr_node(PEXTENT_NODE pexn) { return &(pexn->rbtnAddress); } +/* + * Given a pointer to an extent node's "size-address" RBTREENODE, returns a pointer to the extent node. + * + * Parameters: + * - prbtn = Pointer to the "size-address" RBTREENODE. + * + * Returns: + * Pointer to the base extent node. + */ static PEXTENT_NODE get_from_sizeaddr_node(PRBTREENODE prbtn) { return (PEXTENT_NODE)(((UINT_PTR)prbtn) - OFFSETOF(EXTENT_NODE, rbtnSizeAddress)); } +/* + * Given a pointer to an extent node's "address" RBTREENODE, returns a pointer to the extent node. + * + * Parameters: + * - prbtn = Pointer to the "address" RBTREENODE. + * + * Returns: + * Pointer to the base extent node. + */ static PEXTENT_NODE get_from_addr_node(PRBTREENODE prbtn) { return (PEXTENT_NODE)(((UINT_PTR)prbtn) - OFFSETOF(EXTENT_NODE, rbtnAddress)); @@ -92,31 +155,320 @@ static PEXTENT_NODE get_from_addr_node(PRBTREENODE prbtn) *---------------------- */ +/* + * Looks for a chunk in the "free extents" list that can be used to satisfy an allocation, and returns it + * if one is found. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - sz = Size of the chunk to be allocated. + * - szAlignment = Specifies the alignment that the allocated chunk should have. + * - pfZeroed = Points to a flag which indicates if the allocated chunk should be zeroed. If the actual + * chunk we find is zeroed, this gets toggled to TRUE. + * + * Returns: + * - NULL = A recycled chunk could not be found. + * - Other = Pointer to the recycled chunk. + */ +static PVOID chunk_recycle(PHEAPDATA phd, SIZE_T sz, SIZE_T szAlignment, BOOL *pfZeroed) +{ + PVOID rc; /* return from this function */ + SIZE_T szAlloc; /* allocation size to look for */ + SIZE_T szLeading; /* leading size of the chunk that can be returned */ + SIZE_T szTrailing; /* trailing size of the chunk that can be returned */ + BOOL fZeroed; /* is this block zeroed? */ + PEXTENT_NODE pexn; /* pointer to extent node we find */ + EXTENT_NODE exnKey; /* key for tree search */ + + /* Look for a big enough block in the tree. */ + szAlloc = sz + szAlignment - phd->szChunk; + if (szAlloc < sz) + return NULL; /* the allocator wrapped around */ + exnKey.pv = NULL; + exnKey.sz = szAlloc; + IMutex_Lock(phd->pmtxChunks); + pexn = (PEXTENT_NODE)RbtFindSuccessor(&(phd->rbtExtSizeAddr), (TREEKEY)(&exnKey)); + if (!pexn) + { /* no big enough block found, bug out */ + IMutex_Unlock(phd->pmtxChunks); + return NULL; + } + + /* Compute leading and trailing sizes in this block and point to return data. */ + szLeading = ALIGNMENT_CEILING((UINT_PTR)(pexn->pv), szAlignment - 1) - (UINT_PTR)(pexn->pv); + _H_ASSERT(phd, pexn->sz >= szLeading + sz); + szTrailing = pexn->sz - szLeading - sz; + rc = (PVOID)((UINT_PTR)(pexn->pv) + szLeading); + fZeroed = pexn->fZeroed; + if (fZeroed) + *pfZeroed = TRUE; + + /* Remove existing node from the trees. */ + RbtDelete(&(phd->rbtExtSizeAddr), (TREEKEY)pexn); + RbtDelete(&(phd->rbtExtAddr), (TREEKEY)pexn); + + if (szLeading > 0) + { /* insert leading space as smaller chunk */ + pexn->sz = szLeading; + rbtNewNode(&(pexn->rbtnSizeAddress)); + rbtNewNode(&(pexn->rbtnAddress)); + RbtInsert(&(phd->rbtExtSizeAddr), pexn); + RbtInsert(&(phd->rbtExtAddr), pexn); + pexn = NULL; /* reused the node, no longer there */ + } + + if (szTrailing > 0) + { /* insert trailing space as smaller chunk */ + if (!pexn) + { /* allocate another node, dropping mutex first to avoid deadlock */ + IMutex_Unlock(phd->pmtxChunks); + pexn = _HeapBaseNodeAlloc(phd); + if (!pexn) + { /* node allocation failed, reverse allocation and bug out */ + _HeapChunkDeAlloc(phd, rc, sz, TRUE); + return NULL; + } + IMutex_Lock(phd->pmtxChunks); + } + pexn->pv = (PVOID)(((UINT_PTR)rc) + sz); + pexn->sz = szTrailing; + pexn->fZeroed = fZeroed; + rbtNewNode(&(pexn->rbtnSizeAddress)); + rbtNewNode(&(pexn->rbtnAddress)); + RbtInsert(&(phd->rbtExtSizeAddr), pexn); + RbtInsert(&(phd->rbtExtAddr), pexn); + pexn = NULL; /* used the node */ + } + + IMutex_Unlock(phd->pmtxChunks); + if (pexn) + _HeapBaseNodeDeAlloc(phd, pexn); /* deallocate unused node, if any */ + if (*pfZeroed && !fZeroed) + StrSetMem(rc, 0, sz); /* zero memory if we wanted it and don't have it */ + return rc; +} + +/* + * Allocate a new chunk for heap use, either from the current "free" chunks or from the underlying + * IChunkAllocator interface. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - sz = Size of the chunk to be allocated. + * - szAlignment = Specifies the alignment that the allocated chunk should have. + * - fBase = TRUE if this is an allocation from the heap base code, FALSE if not. + * - pfZeroed = Points to a flag which indicates if the allocated chunk should be zeroed. If the actual + * chunk we find is zeroed, this gets toggled to TRUE. + * + * Returns: + * - NULL = A chunk could not be found (out of memory). + * - Other = Pointer to the new chunk. + */ PVOID _HeapChunkAlloc(PHEAPDATA phd, SIZE_T sz, SIZE_T szAlignment, BOOL fBase, BOOL *pfZeroed) { - return NULL; /* TODO */ + PVOID rc = NULL; /* return from this function */ + HRESULT hr; /* return from chunk allocator */ + + _H_ASSERT(phd, sz != 0); + _H_ASSERT(phd, (sz & phd->uiChunkSizeMask) == 0); + _H_ASSERT(phd, szAlignment != 0); + _H_ASSERT(phd, (szAlignment & phd->uiChunkSizeMask) == 0); + + /* try getting a recycled chunk first */ + if (!fBase) /* don't try to recycle if we're allocating on behalf of the base */ + rc = chunk_recycle(phd, sz, szAlignment, pfZeroed); + if (rc) + goto returning; + + /* call the chunk allocator for a new chunk */ + hr = IChunkAllocator_AllocChunk(phd->pChunkAllocator, sz, szAlignment, &rc); + if (SUCCEEDED(hr)) + { /* got it! */ + if (hr != MEMMGR_S_NONZEROED) + *pfZeroed = TRUE; /* we got zeroed memory even though we didn't ask for it */ + goto returning; + } + + /* allocation completely failed */ + rc = NULL; +returning: + if (rc && !fBase) + { /* log the chunk we return in our radix tree */ + if (_HeapRTreeSet(phd, phd->prtChunks, (UINT_PTR)rc, rc)) + { /* bogus! deallocate the chunk */ + _HeapChunkDeAlloc(phd, rc, sz, TRUE); + return NULL; + } + } + + _H_ASSERT(phd, CHUNK_ADDR2BASE(phd, rc) == rc); + return rc; } +/* + * Record a chunk in the "free extents" list. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - pvChunk = The chunk to be recorded in the free list. + * - sz = The size of the chunk to be recorded in the free list. + * + * Returns: + * Nothing. + */ +static void chunk_record(PHEAPDATA phd, PVOID pvChunk, SIZE_T sz) +{ + HRESULT hr; /* allocator return value */ + BOOL fUnzeroed; /* TRUE if region is not zeroed */ + PEXTENT_NODE pexnNew; /* new extent node */ + PEXTENT_NODE pexn; /* pointer to found extent node */ + PEXTENT_NODE pexnPrev; /* pointer to previous node */ + EXTENT_NODE exnKey; /* key value to search for */ + + /* tell the allocator to purge the region */ + hr = IChunkAllocator_PurgeUnusedRegion(phd->pChunkAllocator, pvChunk, sz); + fUnzeroed = MAKEBOOL(hr == MEMMGR_S_NONZEROED); + + /* allocate new node which we may need */ + pexnNew = _HeapBaseNodeAlloc(phd); + IMutex_Lock(phd->pmtxChunks); + + /* try looking to coalesce with next node */ + exnKey.pv = (PVOID)(((UINT_PTR)pvChunk) + sz); + pexn = (PEXTENT_NODE)RbtFindSuccessor(&(phd->rbtExtAddr), (TREEKEY)(&exnKey)); + if (pexn && (pexn->pv == exnKey.pv)) + { /* coalesce node forward with address range */ + RbtDelete(&(phd->rbtExtSizeAddr), (TREEKEY)pexn); + pexn->pv = pvChunk; + pexn->sz += sz; + pexn->fZeroed = MAKEBOOL(pexn->fZeroed && !fUnzeroed); + rbtNewNode(&(pexn->rbtnSizeAddress)); + RbtInsert(&(phd->rbtExtSizeAddr), pexn); + if (pexnNew) /* didn't need new node after all */ + _HeapBaseNodeDeAlloc(phd, pexnNew); + } + else + { /* could not coalesce forward, insert new node */ + if (!pexnNew) + { /* this is not likely but in kernel context it could be significant */ + _H_ASSERT(phd, FALSE); /* record an error here */ + IMutex_Unlock(phd->pmtxChunks); + return; + } + pexn = pexnNew; + pexn->pv = pvChunk; + pexn->sz = sz; + pexn->fZeroed = MAKEBOOL(!fUnzeroed); + rbtNewNode(&(pexn->rbtnSizeAddress)); + rbtNewNode(&(pexn->rbtnAddress)); + RbtInsert(&(phd->rbtExtSizeAddr), pexn); + RbtInsert(&(phd->rbtExtAddr), pexn); + } + + /* Try to coalesce backwards now. */ + pexnPrev = (PEXTENT_NODE)RbtFindPredecessor(&(phd->rbtExtAddr), (TREEKEY)pexn); + if (pexnPrev && ((PVOID)(((UINT_PTR)(pexnPrev->pv)) + pexnPrev->sz)) == pvChunk) + { /* Coalesce chunk with previous address range. */ + RbtDelete(&(phd->rbtExtSizeAddr), (TREEKEY)pexnPrev); + RbtDelete(&(phd->rbtExtAddr), (TREEKEY)pexnPrev); + + RbtDelete(&(phd->rbtExtSizeAddr), (TREEKEY)pexn); + pexn->pv = pexnPrev->pv; + pexn->sz += pexnPrev->sz; + pexn->fZeroed = MAKEBOOL(pexn->fZeroed && pexnPrev->fZeroed); + rbtNewNode(&(pexn->rbtnSizeAddress)); + RbtInsert(&(phd->rbtExtSizeAddr), pexn); + _HeapBaseNodeDeAlloc(phd, pexnPrev); + } + + IMutex_Unlock(phd->pmtxChunks); /* done */ +} + +/* + * "Unmap" a chunk and record it as "free." + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - pvChunk = The chunk to be unmapped. + * - sz = The size of the chunk to be unmapped. + * + * Returns: + * Nothing. + */ void _HeapChunkUnmap(PHEAPDATA phd, PVOID pvChunk, SIZE_T sz) { - /* TODO */ + _H_ASSERT(phd, pvChunk); + _H_ASSERT(phd, CHUNK_ADDR2BASE(phd, pvChunk) == pvChunk); + _H_ASSERT(phd, sz); + _H_ASSERT(phd, (sz && phd->uiChunkSizeMask) == 0); + chunk_record(phd, pvChunk, sz); } +/* + * Deallocate a chunk of memory. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - pvChunk = The chunk to be deallocated. + * - sz = The size of the chunk to be deallocated. + * - fUnmap = If TRUE, this chunk will be unmapped. + * + * Returns: + * Nothing. + */ void _HeapChunkDeAlloc(PHEAPDATA phd, PVOID pvChunk, SIZE_T sz, BOOL fUnmap) { - /* TODO */ + _H_ASSERT(phd, pvChunk); + _H_ASSERT(phd, CHUNK_ADDR2BASE(phd, pvChunk) == pvChunk); + _H_ASSERT(phd, sz); + _H_ASSERT(phd, (sz && phd->uiChunkSizeMask) == 0); + + _HeapRTreeSet(phd, phd->prtChunks, (UINT_PTR)pvChunk, NULL); + if (fUnmap) + _HeapChunkUnmap(phd, pvChunk, sz); } +/* + * Set up the chunk allocation code in the heap. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * + * Returns: + * Standard HRESULT success/failure indicator. + */ HRESULT _HeapChunkSetup(PHEAPDATA phd) { + HRESULT hr; /* intermediate result */ + + hr = IMutexFactory_CreateMutex(phd->pMutexFactory, &(phd->pmtxChunks)); + if (FAILED(hr)) + return hr; rbtInitTree(&(phd->rbtExtSizeAddr), (PFNTREECOMPARE)compare_sizeaddr, (PFNGETTREEKEY)get_from_sizeaddr_node, (PFNGETTREENODEPTR)get_sizeaddr_node, (PFNGETFROMTREENODEPTR)get_from_sizeaddr_node); rbtInitTree(&(phd->rbtExtAddr), (PFNTREECOMPARE)compare_addr, (PFNGETTREEKEY)get_from_addr_node, (PFNGETTREENODEPTR)get_addr_node, (PFNGETFROMTREENODEPTR)get_from_addr_node); + phd->prtChunks = _HeapRTreeNew(phd, (1U << (LOG_PTRSIZE + 3)) - phd->nChunkBits); + if (!(phd->prtChunks)) + { + IUnknown_Release(phd->pmtxChunks); + return E_OUTOFMEMORY; + } return S_OK; } +/* + * Shut down the chunk allocation code in the heap. + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * + * Returns: + * Nothing. + */ extern void _HeapChunkShutdown(PHEAPDATA phd) { - /* TODO */ + /* TODO: find chunks owned by either the extent tree or the radix tree and attempt to return them */ + _HeapRTreeDestroy(phd->prtChunks); + IUnknown_Release(phd->pmtxChunks); } diff --git a/kernel/lib/heap_internals.h b/kernel/lib/heap_internals.h index 5362ae9..5c73b01 100644 --- a/kernel/lib/heap_internals.h +++ b/kernel/lib/heap_internals.h @@ -39,15 +39,33 @@ #ifndef __ASM__ +#include #include #include #include #include #include +#include #include #include #include #include +#include + +/*-------------------------------------------------------------- + * Radix tree implementation for keeping track of memory chunks + *-------------------------------------------------------------- + */ + +#define RTREE_NODESIZE (1U << 14) /* node size for tree */ + +typedef struct tagMEMRTREE +{ + IMutex *pmtx; /* mutex for tree */ + PPVOID ppRoot; /* tree root */ + UINT32 uiHeight; /* tree height */ + UINT32 auiLevel2Bits[0]; /* bits at level 2 - dynamically sized */ +} MEMRTREE, *PMEMRTREE; /*------------------- * Extent management @@ -77,6 +95,8 @@ typedef struct tagHEAPDATA { UINT32 uiRefCount; /* reference count */ UINT32 uiFlags; /* flags word */ PFNRAWHEAPDATAFREE pfnFreeRawHeapData; /* pointer to function that frees the raw heap data, if any */ + PFNHEAPABORT pfnAbort; /* pointer to abort function */ + PVOID pvAbortArg; /* argument to abort function */ IChunkAllocator *pChunkAllocator; /* chunk allocator pointer */ IMutexFactory *pMutexFactory; /* mutex factory pointer */ FIXEDCPDATA fcpMallocSpy; /* connection point for IMallocSpy */ @@ -87,8 +107,10 @@ typedef struct tagHEAPDATA { UINT32 szChunk; /* size of a chunk */ UINT32 uiChunkSizeMask; /* bitmask for a chunk */ UINT32 cpgChunk; /* number of pages in a chunk */ + IMutex *pmtxChunks; /* chunks mutex */ RBTREE rbtExtSizeAddr; /* tree ordering extents by size and address */ RBTREE rbtExtAddr; /* tree ordering extents by address */ + PMEMRTREE prtChunks; /* radix tree containing all chunk values */ IMutex *pmtxBase; /* base mutex */ PVOID pvBasePages; /* pages being used for internal memory allocation */ PVOID pvBaseNext; /* next allocation location */ @@ -96,19 +118,64 @@ typedef struct tagHEAPDATA { PEXTENT_NODE pexnBaseNodes; /* pointer to base nodes */ } HEAPDATA, *PHEAPDATA; +/*--------------------------------- + * Utility and debugging functions + *--------------------------------- + */ + +/* Get nearest aligned address at or below a. */ +#define ALIGNMENT_ADDR2BASE(a, alignment) ((PVOID)(((UINT_PTR)(a)) & ~(alignment))) + +/* Get offset between a and ALIGNMENT_ADDR2BASE(a, alignment). */ +#define ALIGNMENT_ADDR2OFFSET(a, alignment) ((SIZE_T)(((UINT_PTR)(a)) & (alignment))) + +/* Returns the smallest alignment multiple greater than sz. */ +#define ALIGNMENT_CEILING(sz, alignment) (((sz) + (alignment)) & ~(alignment)) + +CDECL_BEGIN + +extern void _HeapDbgWrite(PHEAPDATA phd, PCSTR sz); +extern void _HeapPrintf(PHEAPDATA phd, PCSTR szFormat, ...); +extern void _HeapAssertFailed(PHEAPDATA phd, PCSTR szFile, INT32 nLine); +extern SIZE_T _HeapPow2Ceiling(SIZE_T x); + +CDECL_END + +#define _H_THIS_FILE __FILE__ +#define _DECLARE_H_THIS_FILE static const char SEG_RODATA _H_THIS_FILE[] = __FILE__; + +#define _H_ASSERT(phd, expr) ((expr) ? (void)0 : _HeapAssertFailed(phd, _H_THIS_FILE, __LINE__)) + +/*--------------------------------- + * Radix tree management functions + *--------------------------------- + */ + +CDECL_BEGIN + +extern PMEMRTREE _HeapRTreeNew(PHEAPDATA phd, UINT32 cBits); +extern void _HeapRTreeDestroy(PMEMRTREE prt); +extern PVOID _HeapRTreeGetLocked(PHEAPDATA phd, PMEMRTREE prt, UINT_PTR uiKey); +extern PVOID _HeapRTreeGet(PHEAPDATA phd, PMEMRTREE prt, UINT_PTR uiKey); +extern BOOL _HeapRTreeSet(PHEAPDATA phd, PMEMRTREE prt, UINT_PTR uiKey, PVOID pv); + +CDECL_END + /*------------------------------------- * Internal chunk management functions *------------------------------------- */ /* Get chunk address for allocated address a. */ -#define CHUNK_ADDR2BASE(phd, a) ((PVOID)(((UINT_PTR)(a)) & ~((phd)->uiChunkSizeMask))) +#define CHUNK_ADDR2BASE(phd, a) ALIGNMENT_ADDR2BASE((a), (phd)->uiChunkSizeMask) /* Get chunk offset of allocated address a. */ -#define CHUNK_ADDR2OFFSET(phd, a) ((SIZE_T)(((UINT_PTR)(a)) & (phd)->uiChunkSizeMask)) +#define CHUNK_ADDR2OFFSET(phd, a) ALIGNMENT_ADDR2OFFSET((a), (phd)->uiChunkSizeMask) /* Return the smallest chunk size multiple that can contain a certain size. */ -#define CHUNK_CEILING(phd, sz) (((sz) + (phd)->uiChunkSizeMask) & ~((phd)->uiChunkSizeMask)) +#define CHUNK_CEILING(phd, sz) ALIGNMENT_CEILING((sz), (phd)->uiChunkSizeMask) + +CDECL_BEGIN extern PVOID _HeapChunkAlloc(PHEAPDATA phd, SIZE_T sz, SIZE_T szAlignment, BOOL fBase, BOOL *pfZeroed); extern void _HeapChunkUnmap(PHEAPDATA phd, PVOID pvChunk, SIZE_T sz); @@ -116,6 +183,8 @@ extern void _HeapChunkDeAlloc(PHEAPDATA phd, PVOID pvChunk, SIZE_T sz, BOOL fUnm extern HRESULT _HeapChunkSetup(PHEAPDATA phd); extern void _HeapChunkShutdown(PHEAPDATA phd); +CDECL_END + /*------------------------------------ * Internal base management functions *------------------------------------ diff --git a/kernel/lib/heap_rtree.c b/kernel/lib/heap_rtree.c new file mode 100644 index 0000000..7868e61 --- /dev/null +++ b/kernel/lib/heap_rtree.c @@ -0,0 +1,246 @@ +/* + * This file is part of the COMROGUE Operating System for Raspberry Pi + * + * Copyright (c) 2013, Eric J. Bowersox / Erbosoft Enterprises + * All rights reserved. + * + * This program is free for commercial and non-commercial use as long as the following conditions are + * adhered to. + * + * Copyright in this file remains Eric J. Bowersox and/or Erbosoft, and as such any copyright notices + * in the code are not to be removed. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this list of conditions and + * the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and + * the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * "Raspberry Pi" is a trademark of the Raspberry Pi Foundation. + */ +/* + * This code is based on/inspired by jemalloc-3.3.1. Please see LICENSE.jemalloc for further details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "heap_internals.h" + +#ifdef _H_THIS_FILE +#undef _H_THIS_FILE +_DECLARE_H_THIS_FILE +#endif + +/*--------------------------------- + * Radix tree management functions + *--------------------------------- + */ + +/* + * Allocate a new radix tree structure and its root node. + * + * Parameters: + * - phd = Pointer to HEAPDATA block. + * - cBits = Number of bits of key information to store in the tree. + * + * Returns: + * - NULL = Allocation failed. + * - Other = Pointer to the new tree structure. + */ +PMEMRTREE _HeapRTreeNew(PHEAPDATA phd, UINT32 cBits) +{ + PMEMRTREE rc; /* return from this function */ + UINT32 cBitsPerLevel; /* number of bits per level of the tree */ + UINT32 uiHeight; /* tree height */ + SIZE_T cb; /* number of bytes for allocation */ + register UINT32 i; /* loop counter */ + + /* Compute the number of bits per level and the height of the tree. */ + cBitsPerLevel = IntFirstSet(_HeapPow2Ceiling(RTREE_NODESIZE / sizeof(PVOID))) - 1; + uiHeight = cBits / cBitsPerLevel; + if ((uiHeight * cBitsPerLevel) != cBits) + uiHeight++; + _H_ASSERT(phd, (uiHeight * cBitsPerLevel) >= cBits); + + /* Allocate the tree node. */ + cb = sizeof(MEMRTREE) + (sizeof(UINT32) * uiHeight); + rc = (PMEMRTREE)_HeapBaseAlloc(phd, cb); + if (!rc) + return NULL; + StrSetMem(rc, 0, cb); + + /* Allocate the mutex and fill the tree data. */ + if (FAILED(IMutexFactory_CreateMutex(phd->pMutexFactory, &(rc->pmtx)))) + return NULL; /* just leak the allocation */ + rc->uiHeight = uiHeight; + if ((uiHeight * cBitsPerLevel) > cBits) + rc->auiLevel2Bits[0] = cBits % cBitsPerLevel; + else + rc->auiLevel2Bits[0] = cBitsPerLevel; + for (i = 1; i < uiHeight; i++) + rc->auiLevel2Bits[i] = cBitsPerLevel; + + /* Allocate the tree root data. */ + cb = sizeof(PVOID) << rc->auiLevel2Bits[0]; + rc->ppRoot = (PPVOID)_HeapBaseAlloc(phd, cb); + if (!(rc->ppRoot)) + { /* failed allocation, just leak the base node */ + IUnknown_Release(rc->pmtx); + return NULL; + } + StrSetMem(rc->ppRoot, 0, cb); + return rc; +} + +/* + * Destroy a radix tree data structure. + * + * Parameters: + * - prt = Pointer to the radix tree structure to be destroyed. + * + * Returns: + * Nothing. + */ +void _HeapRTreeDestroy(PMEMRTREE prt) +{ + IUnknown_Release(prt->pmtx); /* destroy the mutex */ + /* don't need to destroy anything else, it will get killed with the base allocations */ +} + +/* + * Retrieve a value from the radix tree. (Can either do it with the tree locked or not.) + * + * Parameters: + * - phd = Pointer to HEAPDATA block. + * - prt = Pointer to the radix tree structure to search. + * - uiKey = Key value to look for in the tree. + * + * Returns: + * - NULL = Key value not found. + * - Other = Pointer value stashed in the tree under that key value. + */ +#define GENERATE_GET_FUNC(name) \ +PVOID name (PHEAPDATA phd, PMEMRTREE prt, UINT_PTR uiKey) \ +{ \ + PVOID rc; /* return from this function */ \ + UINT_PTR uiSubkey; /* subkey value */ \ + UINT32 i; /* loop counter */ \ + UINT32 nLeftShift; /* number of bits to shift key left */ \ + UINT32 uiHeight; /* tree height value */ \ + UINT32 cBits; /* number of bits at this level */ \ + PPVOID ppNode; /* pointer to current node */ \ + PPVOID ppChild; /* pointer to child node */ \ + \ + DO_LOCK(prt->pmtx); \ + /* Walk the tree to find the right leaf node */ \ + for (i = nLeftShift = 0, uiHeight = prt->uiHeight, ppNode = prt->ppRoot; \ + i < (uiHeight - 1); \ + i++, nLeftShift += cBits, ppNode = ppChild) \ + { \ + cBits = prt->auiLevel2Bits[i]; \ + uiSubkey = (uiKey << nLeftShift) >> ((1U << (LOG_PTRSIZE + 3)) - cBits); \ + ppChild = (PPVOID)(ppNode[uiSubkey]); \ + if (!ppChild) \ + { \ + DO_UNLOCK(prt->pmtx); \ + return NULL; \ + } \ + } \ + \ + /* this is a leaf node, it contains values, not pointers */ \ + cBits = prt->auiLevel2Bits[i]; \ + uiSubkey = (uiKey << nLeftShift) >> ((1U << (LOG_PTRSIZE + 3)) - cBits); \ + rc = ppNode[uiSubkey]; \ + DO_UNLOCK(prt->pmtx); \ + DO_GET_VALIDATE \ + return rc; \ +} + +/* Generate locked version of function */ +#define DO_LOCK(mtx) IMutex_Lock(mtx) +#define DO_UNLOCK(mtx) IMutex_Unlock(mtx) +#define DO_GET_VALIDATE +GENERATE_GET_FUNC(_HeapRTreeGetLocked) +#undef DO_LOCK +#undef DO_UNLOCK +#undef DO_GET_VALIDATE + +/* Generate unlocked version of function */ +#define DO_LOCK(mtx) +#define DO_UNLOCK(mtx) +#define DO_GET_VALIDATE _H_ASSERT(phd, _HeapRTreeGetLocked(phd, prt, uiKey) == rc); +GENERATE_GET_FUNC(_HeapRTreeGet) +#undef DO_LOCK +#undef DO_UNLOCK +#undef DO_GET_VALIDATE + +/* + * Sets a pointer value into the radix tree under a specified key. + * + * Parameters: + * - phd = Pointer to HEAPDATA block. + * - prt = Pointer to the radix tree structure to set. + * - uiKey = Key value to set in the tree. + * - pv = Pointer value to set in the tree. + * + * Returns: + * - TRUE = Allocation failed, tree value not set. + * - FALSE = Tree value set. + */ +BOOL _HeapRTreeSet(PHEAPDATA phd, PMEMRTREE prt, UINT_PTR uiKey, PVOID pv) +{ + UINT_PTR uiSubkey; /* subkey value */ + UINT32 i; /* loop counter */ + UINT32 nLeftShift; /* number of bits to shift key left */ + UINT32 uiHeight; /* tree height value */ + UINT32 cBits; /* number of bits at this level */ + PPVOID ppNode; /* pointer to current node */ + PPVOID ppChild; /* pointer to child node */ + SIZE_T cb; /* number of bytes to allocate */ + + IMutex_Lock(prt->pmtx); + /* Walk the tree to find the right leaf node (creating where applicable) */ + for (i = nLeftShift = 0, uiHeight = prt->uiHeight, ppNode = prt->ppRoot; + i < (uiHeight - 1); + i++, nLeftShift += cBits, ppNode = ppChild) + { + cBits = prt->auiLevel2Bits[i]; + uiSubkey = (uiKey << nLeftShift) >> ((1U << (LOG_PTRSIZE + 3)) - cBits); + ppChild = (PPVOID)(ppNode[uiSubkey]); + if (!ppChild) + { /* allocate new node for next level */ + cb = sizeof(PVOID) << prt->auiLevel2Bits[i + 1]; + ppChild = (PPVOID)_HeapBaseAlloc(phd, cb); + if (!ppChild) + { /* allocation failed, bug out */ + IMutex_Unlock(prt->pmtx); + return TRUE; + } + StrSetMem(ppChild, 0, cb); + ppNode[uiSubkey] = (PVOID)ppChild; + } + } + + /* this is a leaf node, it contains values, not pointers */ + cBits = prt->auiLevel2Bits[i]; + uiSubkey = (uiKey << nLeftShift) >> ((1U << (LOG_PTRSIZE + 3)) - cBits); + ppNode[uiSubkey] = pv; + IMutex_Unlock(prt->pmtx); + return FALSE; +} diff --git a/kernel/lib/heap_toplevel.c b/kernel/lib/heap_toplevel.c index 8a4025f..4d6a806 100644 --- a/kernel/lib/heap_toplevel.c +++ b/kernel/lib/heap_toplevel.c @@ -131,10 +131,12 @@ static UINT32 malloc_Release(IUnknown *pThis) if ((rc == 0) && !(phd->uiFlags & PHDFLAGS_DELETING)) { phd->uiFlags |= PHDFLAGS_DELETING; + /* Do subsystem shutdown. */ + _HeapChunkShutdown(phd); _HeapBaseShutdown(phd); toplevel_shutdown(phd); if (phd->pfnFreeRawHeapData) - { + { /* free the raw heap data */ pfnFree = phd->pfnFreeRawHeapData; (*pfnFree)((PRAWHEAPDATA)phd); } @@ -453,6 +455,8 @@ static const SEG_RODATA struct IConnectionPointContainerVTable vtblConnectionPoi * for the heap. * - pfnFree = Pointer to a function called as the last stage of releasing the heap, which frees the * "prhd" block. May be NULL. + * - pfnAbort = Pointer to a function called when there's a serious error in the heap. May be NULL. + * - pvAbortArg = Argument passed to the pfnAbort function when it's called. May be NULL. * - pChunkAllocator = Pointer to the IChunkAllocator interface used by the heap to allocate chunks of memory * for carving up by the heap. * - pMutexFactory = Pointer to the IMutexFactory interface used to allocate IMutex objects. @@ -462,8 +466,9 @@ static const SEG_RODATA struct IConnectionPointContainerVTable vtblConnectionPoi * Returns: * Standard HRESULT success/failure. */ -HRESULT HeapCreate(PRAWHEAPDATA prhd, PFNRAWHEAPDATAFREE pfnFree, IChunkAllocator *pChunkAllocator, - IMutexFactory *pMutexFactory, UINT32 nChunkBits, IMalloc **ppHeap) +HRESULT HeapCreate(PRAWHEAPDATA prhd, PFNRAWHEAPDATAFREE pfnFree, PFNHEAPABORT pfnAbort, PVOID pvAbortArg, + IChunkAllocator *pChunkAllocator, IMutexFactory *pMutexFactory, UINT32 nChunkBits, + IMalloc **ppHeap) { PHEAPDATA phd; /* pointer to actual heap data */ HRESULT hr; /* HRESULT of intermediate operations */ @@ -481,6 +486,8 @@ HRESULT HeapCreate(PRAWHEAPDATA prhd, PFNRAWHEAPDATAFREE pfnFree, IChunkAllocato phd->uiRefCount = 1; phd->uiFlags = 0; phd->pfnFreeRawHeapData = pfnFree; + phd->pfnAbort = pfnAbort; + phd->pvAbortArg = pvAbortArg; phd->nChunkBits = nChunkBits; phd->szChunk = 1 << nChunkBits; if (phd->szChunk < SYS_PAGE_SIZE) @@ -497,10 +504,13 @@ HRESULT HeapCreate(PRAWHEAPDATA prhd, PFNRAWHEAPDATAFREE pfnFree, IChunkAllocato ObjHlpFixedCpSetup(&(phd->fcpSequentialStream), (PUNKNOWN)phd, &IID_ISequentialStream, (IUnknown **)(&(phd->pDebugStream)), 1, NULL); - /* Setup base pointers. */ + /* Setup all the various "subsystems." */ hr = _HeapBaseSetup(phd); if (FAILED(hr)) goto error0; + hr = _HeapChunkSetup(phd); + if (FAILED(hr)) + goto error1; @@ -508,7 +518,9 @@ HRESULT HeapCreate(PRAWHEAPDATA prhd, PFNRAWHEAPDATAFREE pfnFree, IChunkAllocato *ppHeap = (IMalloc *)phd; return S_OK; - /*error1:*/ + /*error2:*/ + _HeapChunkShutdown(phd); +error1: _HeapBaseShutdown(phd); error0: toplevel_shutdown(phd); diff --git a/kernel/lib/heap_utils.c b/kernel/lib/heap_utils.c new file mode 100644 index 0000000..fdadf68 --- /dev/null +++ b/kernel/lib/heap_utils.c @@ -0,0 +1,148 @@ +/* + * This file is part of the COMROGUE Operating System for Raspberry Pi + * + * Copyright (c) 2013, Eric J. Bowersox / Erbosoft Enterprises + * All rights reserved. + * + * This program is free for commercial and non-commercial use as long as the following conditions are + * adhered to. + * + * Copyright in this file remains Eric J. Bowersox and/or Erbosoft, and as such any copyright notices + * in the code are not to be removed. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this list of conditions and + * the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and + * the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * "Raspberry Pi" is a trademark of the Raspberry Pi Foundation. + */ +/* + * This code is based on/inspired by jemalloc-3.3.1. Please see LICENSE.jemalloc for further details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "heap_internals.h" + +#ifdef _H_THIS_FILE +#undef _H_THIS_FILE +_DECLARE_H_THIS_FILE +#endif + +/*--------------------------------- + * Utility and debugging functions + *--------------------------------- + */ + +/* + * Write a string to debugging output (if it's been configured). + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - sz = String to be written. + * + * Returns: + * Nothing. + */ +void _HeapDbgWrite(PHEAPDATA phd, PCSTR sz) +{ + if (phd->pDebugStream) + ISequentialStream_Write(phd->pDebugStream, sz, StrLength8(sz), NULL); +} + +/* + * Internal function called to write formatted output to the debugging output stream. + * + * Parameters: + * - ppvArg = Pointer argument to StrFormatV8 (actually the ISequentialStream pointer). + * - pchData = Pointer to data to be written. + * - cbData = Number of characters to be written. + * + * Returns: + * Standard HRESULT success/failure indicator. + */ +static HRESULT heap_printf_func(PPVOID ppvArg, PCCHAR pchData, UINT32 cbData) +{ + return ISequentialStream_Write(((PSEQUENTIALSTREAM)ppvArg), pchData, cbData, NULL); +} + +/* + * Formats data to the debugging output (if it's been configured). + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - szFormat = Printf-style format string. + * - = Arguments to be substituted into the printf-style format string. + * + * Returns: + * Nothing. + */ +void _HeapPrintf(PHEAPDATA phd, PCSTR szFormat, ...) +{ + va_list pargs; + + if (phd->pDebugStream) + { + va_start(pargs, szFormat); + StrFormatV8(heap_printf_func, (PPVOID)(phd->pDebugStream), szFormat, pargs); + va_end(pargs); + } +} + +/* + * Called when an assertion in the code fails; it prints the assertion failure to the debugging output (if set) + * and calls the heap's abort function (if set). + * + * Parameters: + * - phd = Pointer to the HEAPDATA block. + * - szFile = File name where the assertion failed. + * - szLine = Line number where the assertion failed. + * + * Returns: + * Nothing. + */ +void _HeapAssertFailed(PHEAPDATA phd, PCSTR szFile, INT32 nLine) +{ + static const char SEG_RODATA szMessage[] = "_HeapAssertFailed at %s:%d\n"; + _HeapPrintf(phd, szMessage, szFile, nLine); + if (phd->pfnAbort) + (*(phd->pfnAbort))(phd->pvAbortArg); +} + +/* + * Compute the smallest power of 2 greater than or equal to its argument. + * + * Parameters: + * - x = The value to compute. + * + * Returns: + * The value that is the smallest power of 2 greater than or equal to x. + */ +SIZE_T _HeapPow2Ceiling(SIZE_T x) +{ + x--; + x |= (x >> 1); /* this sequence sets all bits at or below the topmost one to 1 */ + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + return ++x; /* which ensures that this is a power of 2 */ +} diff --git a/kernel/lib/intlib.c b/kernel/lib/intlib.c index efc97d8..e198a9c 100644 --- a/kernel/lib/intlib.c +++ b/kernel/lib/intlib.c @@ -78,3 +78,27 @@ HRESULT IntLDiv(PLDIV pResult, INT64 num, INT64 denom) pResult->rem = -(pResult->rem); return S_OK; } + +/* + * Returns the index of the first bit set in the specified mask, where the least-significant bit is considered + * bit #1. + * + * Parameters: + * - nMask = The mask to be tested. + * + * Returns: + * - 0 = If nMask is 0. + * - Other = Index of the first set bit in nMask. + */ +INT32 IntFirstSet(INT32 nMask) +{ + register INT32 nBit; /* bit index to return */ + if (nMask) + { + for (nBit = 1; !(nMask & 1); nBit++) + nMask = ((UINT32)nMask) >> 1; + return nBit; + } + else + return 0; /* corner case */ +}