From 682aa60c013aa3dc2d8f10cd312609e162e0359d Mon Sep 17 00:00:00 2001 From: "Eric J. Bowersox" Date: Sun, 9 Jun 2013 21:41:17 -0600 Subject: [PATCH] started adding stuff for the arena layer of the heap --- include/comrogue/types.h | 1 + kernel/lib/Makefile | 6 +- kernel/lib/heap_arena.c | 61 +++++++++++++ kernel/lib/heap_internals.h | 171 ++++++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 kernel/lib/heap_arena.c diff --git a/include/comrogue/types.h b/include/comrogue/types.h index b779231..ee75e10 100644 --- a/include/comrogue/types.h +++ b/include/comrogue/types.h @@ -61,6 +61,7 @@ #define LOG_PTRSIZE 2 /* log2(sizeof(void *)) */ #define LOG_INTSIZE 2 /* log2(sizeof(int)) */ +#define LOG_UINTSIZE 2 /* log2(sizeof(UINT32)) */ #define LOG_INT64SIZE 3 /* log2(sizeof(long long)) */ /* Boolean values */ diff --git a/kernel/lib/Makefile b/kernel/lib/Makefile index d51fb7e..867718b 100644 --- a/kernel/lib/Makefile +++ b/kernel/lib/Makefile @@ -32,9 +32,9 @@ MAKEFLAGS += -rR CRBASEDIR := $(abspath ../..) include $(CRBASEDIR)/armcompile.mk -LIB_OBJS = divide.o qdivrem.o heap_toplevel.o heap_base.o heap_chunks.o heap_rtree.o heap_utils.o intlib.o objhelp.o \ - objhelp_enumconn.o objhelp_enumgeneric.o objhelp_fixedcp.o rbtree.o str.o strcopymem.o strcomparemem.o \ - strsetmem.o lib_guids.o +LIB_OBJS = divide.o qdivrem.o heap_toplevel.o heap_arena.o heap_base.o heap_chunks.o heap_rtree.o heap_utils.o \ + intlib.o objhelp.o objhelp_enumconn.o objhelp_enumgeneric.o objhelp_fixedcp.o rbtree.o str.o \ + strcopymem.o strcomparemem.o strsetmem.o lib_guids.o all: kernel-lib.o diff --git a/kernel/lib/heap_arena.c b/kernel/lib/heap_arena.c new file mode 100644 index 0000000..65c8fe3 --- /dev/null +++ b/kernel/lib/heap_arena.c @@ -0,0 +1,61 @@ +/* + * This file is part of the COMROGUE Operating System for Raspberry Pi + * + * Copyright (c) 2013, Eric J. Bowersox / Erbosoft Enterprises + * All rights reserved. + * + * This program is free for commercial and non-commercial use as long as the following conditions are + * adhered to. + * + * Copyright in this file remains Eric J. Bowersox and/or Erbosoft, and as such any copyright notices + * in the code are not to be removed. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this list of conditions and + * the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and + * the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * "Raspberry Pi" is a trademark of the Raspberry Pi Foundation. + */ +/* + * This code is based on/inspired by jemalloc-3.3.1. Please see LICENSE.jemalloc for further details. + */ +#include +#include +#include "heap_internals.h" + +#ifdef _H_THIS_FILE +#undef _H_THIS_FILE +_DECLARE_H_THIS_FILE +#endif + +/*---------------------------- + * Arena management functions + *---------------------------- + */ + + + + + +HRESULT _HeapArenaSetup(PHEAPDATA phd) +{ + return S_OK; /* TODO */ +} + +void _HeapArenaShutdown(PHEAPDATA phd) +{ + /* TODO */ +} diff --git a/kernel/lib/heap_internals.h b/kernel/lib/heap_internals.h index 5c73b01..0843bc2 100644 --- a/kernel/lib/heap_internals.h +++ b/kernel/lib/heap_internals.h @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -84,6 +85,164 @@ typedef struct tagEXTENT_NODE } EXTENT_NODE, *PEXTENT_NODE; typedef PEXTENT_NODE *PPEXTENT_NODE; +/*--------------------- + * Bitmap declarations + *--------------------- + */ + +/* Maximum number of regions per run */ +#define LG_RUN_MAXREGS 11 +#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) + +/* Maximum bitmap count */ +#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS + +typedef UINT32 BITMAP; /* bitmap type definition */ +#define LG_SIZEOF_BITMAP LOG_UINTSIZE + +/* Number of bits per group */ +#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3) +#define BITMAP_GROUP_NBITS (1U << LG_BITMAP_GROUP_NBITS) +#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS - 1) + +/* Maximum number of bitmap levels */ +#define BITMAP_MAX_LEVELS \ + ((LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) + !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP)) + +/* Bitmap level structure */ +typedef struct tagBITMAPLEVEL +{ + SIZE_T ofsGroup; /* offset of groups for this level within array */ +} BITMAPLEVEL, *PBITMAPLEVEL; + +/* Bitmap information structure */ +typedef struct tagBITMAPINFO +{ + SIZE_T cBits; /* number of bits in bitmap */ + UINT32 nLevels; /* number of levels required for bits */ + BITMAPLEVEL aLevels[BITMAP_MAX_LEVELS + 1]; /* the levels - only first (nLevels + 1) used */ +} BITMAPINFO, *PBITMAPINFO; + +/*--------------------------------- + * Thread-level cache declarations + *--------------------------------- + */ + +/* statistics per cache bin */ +typedef struct tagTCACHEBINSTATS +{ + UINT64 nRequests; /* number of requests in this particular bin */ +} TCACHEBINSTATS, *PTCACHEBINSTATS; + +/* single bin of the cache */ +typedef struct tagTCACHEBIN +{ + TCACHEBINSTATS stats; /* statistics for this bin */ + INT32 nLowWatermark; /* minimum number cached since last GC */ + UINT32 cbitFill; /* fill level */ + UINT32 nCached; /* number of cached objects */ + PPVOID ppvAvail; /* stack of cached objects */ +} TCACHEBIN, *PTCACHEBIN; + +typedef struct tagARENA ARENA, *PARENA; /* forward declaration */ + +/* thread cache */ +typedef struct tagTCACHE +{ + DLIST_FIELD_DECLARE(struct tagTCACHE, link); /* link aggregator */ + UINT64 cbProfAccum; /* accumulated bytes */ + PARENA parena; /* this thread's arena */ + UINT32 cEvents; /* event count since incremental GC */ + UINT32 ndxNextGCBin; /* next bin to be GC'd */ + TCACHEBIN aBins[0]; /* cache bins (dynamically sized) */ +} TCACHE, *PTCACHE; + +/*------------------------ + * Arena data definitions + *------------------------ + */ + +/* Chunk map, each element corresponds to one page within the chunk */ +typedef struct tagARENACHUNKMAP +{ + union + { + RBTREENODE rbtn; /* tree of runs */ + DLIST_FIELD_DECLARE(struct tagARENACHUNKMAP, link); /* list of runs in purgatory */ + } u; + SIZE_T bits; /* run address and various flags */ +} ARENACHUNKMAP, *PARENACHUNKMAP; + +#define CHUNK_MAP_BININD_SHIFT 4 /* shift count for bin index mask */ +#define BININD_INVALID ((SIZE_T)0xFFU) /* invalid bin index */ +#define CHUNK_MAP_BININD_MASK ((SIZE_T)0xFF0U) /* bin index mask */ +#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK /* invalid bin marker */ +#define CHUNK_MAP_FLAGS_MASK ((SIZE_T)0xCU) /* flag bits mask */ +#define CHUNK_MAP_DIRTY ((SIZE_T)0x8U) /* dirty flag */ +#define CHUNK_MAP_UNZEROED ((SIZE_T)0x4U) /* non-zeroed flag */ +#define CHUNK_MAP_LARGE ((SIZE_T)0x2U) /* large allocation flag */ +#define CHUNK_MAP_ALLOCATED ((SIZE_T)0x1U) /* allocated flag */ +#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED + +/* chunk header within an arena */ +typedef struct tagARENACHUNK +{ + PARENA parena; /* arena that owns the chunk */ + RBTREENODE rbtnDirty; /* linkage for tree of chunks with dirty runs */ + SIZE_T cpgDirty; /* number of dirty pages */ + SIZE_T cAvailRuns; /* number of available runs */ + SIZE_T cAvailRunAdjacent; /* number of available run adjacencies */ + ARENACHUNKMAP aMaps[0]; /* map of pages within chunk */ +} ARENACHUNK, *PARENACHUNK; + +/* large allocation statistics */ +typedef struct tagMALLOCLARGESTATS +{ + UINT64 nMalloc; /* number of allocation requests */ + UINT64 nDalloc; /* number of deallocation requests */ + UINT64 nRequests; /* number of allocation requests */ + SIZE_T cRuns; /* count of runs of this size class */ +} MALLOCLARGESTATS, *PMALLOCLARGESTATS; + +/* Arena statistics data. */ +typedef struct tagARENASTATS +{ + SIZE_T cbMapped; /* number of bytes currently mapped */ + UINT64 cPurges; /* number of purge sweeps made */ + UINT64 cAdvise; /* number of memory advise calls made */ + UINT64 cPagesPurged; /* number of pages purged */ + SIZE_T cbAllocatedLarge; /* number of bytes of large allocations */ + UINT64 cLargeMalloc; /* number of large allocations */ + UINT64 cLargeDalloc; /* number of large deallocations */ + UINT64 cLargeRequests; /* number of large allocation requests */ + PMALLOCLARGESTATS amls; /* array of stat elements, one per size class */ +} ARENASTATS, *PARENASTATS; + +struct tagARENABININFO +{ + SIZE_T cbRegions; /* size of regions in a run */ + SIZE_T cbRedzone; /* size of the red zone */ + SIZE_T cbInterval; /* interval between regions */ + SIZE_T cbRunSize; /* total size of a run for this size class */ + UINT32 nRegions; /* number of regions in a run for this size class */ + UINT32 ofsBitmap; /* offset of bitmap element in run header */ + BITMAPINFO bitmapinfo; /* manipulates bitmaps associated with this bin's runs */ + UINT32 ofsCtx0; /* offset of context in run header, or 0 */ + UINT32 ofsRegion0; /* offse of first region in a run for size class */ +} ARENABININFO, *PARENABININFO; + +/* The actual arena definition. */ +struct tagARENA +{ + UINT32 nIndex; /* index of this arena within array */ + UINT32 nThreads; /* number of threads assigned to this arena */ + IMutex *pmtxLock; /* arena lock */ + ARENASTATS stats; /* arena statistics */ + DLIST_HEAD_DECLARE(TCACHE, dlistTCache); /* list of tcaches for threads in arena */ + UINT64 cbProfAccum; /* accumulated bytes */ + RBTREE rbtDirtyChunks; /* tree of dirty page-containing chunks */ +}; + /*---------------------------------- * The actual heap data declaration *---------------------------------- @@ -200,6 +359,18 @@ extern void _HeapBaseShutdown(PHEAPDATA phd); CDECL_END +/*---------------------------- + * Arena management functions + *---------------------------- + */ + +CDECL_BEGIN + +extern HRESULT _HeapArenaSetup(PHEAPDATA phd); +extern void _HeapArenaShutdown(PHEAPDATA phd); + +CDECL_END + #endif /* __ASM__ */ #endif /* __COMROGUE_INTERNALS__ */