singrdk/base/Kernel/Bartok/GCs/SegregatedFreeList.cs

1267 lines
49 KiB
C#
Raw Normal View History

2008-03-05 09:52:00 -05:00
/*******************************************************************/
/* WARNING */
/* This file should be identical in the Bartok and Singularity */
/* depots. Master copy resides in Bartok Depot. Changes should be */
/* made to Bartok Depot and propagated to Singularity Depot. */
/*******************************************************************/
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
namespace System.GCs {
using Microsoft.Bartok.Options;
using Microsoft.Bartok.Runtime;
using System.Runtime.CompilerServices;
using System.Threading;
#if SINGULARITY
using Microsoft.Singularity;
#endif
/// <summary>
/// This class supports Allocate/Free operations using memory obtained
/// from the PageManager.
///
/// Objects that are small enough that multiple instances of them can fit
/// on a page are allocated from pages of identically sized memory cells.
///
/// Large objects are allocated on separate pages.
///
/// This class is designed to be thread safe on both allocation and free
/// operations. At this stage it is required that RecycleGlobalPages is
/// called periodically to reconsider previously full pages for inclusion
/// in allocation lists.
///
/// The free operation is currently more expensive than required due to
/// space accounting.
///
/// This class also keeps up to date memory accounting information. Note
/// that modifications to this data is not synchronized so it is possible
/// that the counts drift from actual figures over time.
/// </summary>
internal unsafe struct SegregatedFreeList /* : Allocator */
{
#region Mixins
[MixinConditional("SegregatedFreeList")]
[MixinConditional("AllThreadMixins")]
[Mixin(typeof(Thread))]
private class SegregatedFreeListThread : Object {
// Thread-specific segregated free-list allocator
[RequiredByBartok]
internal SegregatedFreeList segregatedFreeList;
}
[Inline]
private static SegregatedFreeListThread MixinThread(Thread t)
{
return (SegregatedFreeListThread) (Object) t;
}
#endregion
#region Global (Safe from the context of any thread)
#region Constants
/// <summary>
/// Small objects.
/// </summary>
internal const PageType SMALL_OBJ_PAGE = PageType.Owner2;
/// <summary>
/// This is a page that is being set up.
/// </summary>
internal const PageType INIT_PAGE = PageType.Owner4;
/// <summary>
/// Large objects.
/// </summary>
internal const PageType LARGE_OBJ_PAGE = PageType.Owner3;
/// <summary>
/// BUGBUG: Duplicated constant.
/// </summary>
internal const int LOG_PAGE_SIZE = 12;
/// <summary>
/// The size of each block (which contains a single object size)
/// </summary>
internal const int BLOCK_SIZE = (1 << LOG_PAGE_SIZE) - PageHeader.SIZEOF;
/// <summary>
/// The threshold where objects become large objects.
/// </summary>
internal const int LARGE_OBJECT_THRESHOLD = (BLOCK_SIZE >> 1) + 1;
/// <summary>
/// The largest possible size class.
/// </summary>
internal const int SIZE_CLASSES = 33 + ((LARGE_OBJECT_THRESHOLD-1) >> 7);
#endregion
#region Size Class Mappings
/// <summary>
/// Returns the size class index for an object of the specified size.
///
/// BUGBUG: We hope that from an inlined allocation site this is
/// resolved as a constant. That is why this was changed to remove
/// indirection through an index lookup table.
/// </summary>
[Inline]
private static int GetSizeClass(UIntPtr bytes) {
// Exact request rounds down to lower size class.
bytes = bytes - new UIntPtr(1);
// Sizes 0 -> 64 are in classes 0 -> 15
// Sizes 72 -> 128 are in classes 16 -> 23
// Sizes 140 -> 256 are in classes 24 -> 31
// Sizes 256 -> 512 are in classes 32 -> 35
// We skip a power of 2 because large objects are rare.
//Sizes 512 and up are in classes 36 -> 41
if (bytes < 64) return 0 + ((int)bytes >> 2);
if (bytes < 128) return 8 + ((int)bytes >> 3);
if (bytes < 256) return 16 + ((int)bytes >> 4);
if (bytes < 512) return 28 + ((int)bytes >> 6);
if (bytes < LARGE_OBJECT_THRESHOLD) return 32 + ((int)bytes >> 7);
throw new Exception("GetSizeClass called on large object size");
}
/// <summary>
/// Returns the cell size for a given size class.
/// </summary>
private static UIntPtr GetCellSize(int sizeClass) {
VTable.Assert(sizeClass < SIZE_CLASSES,
"Attempt cellSize for invalid sizeClass");
uint sc = (uint)sizeClass + 1;
uint bytes;
if (sc <= 16) bytes = (sc - 0) << 2;
else if (sc <= 24) bytes = (sc - 8) << 3;
else if (sc <= 32) bytes = (sc - 16) << 4;
else if (sc <= 36) bytes = (sc - 28) << 6;
else bytes = (sc - 32) << 7;
return new UIntPtr(bytes);
}
#endregion
#region Memory Accounting
/// <summary>
/// A count of the bytes allocated for small objects.
/// </summary>
internal static UIntPtr SmallBytes;
/// <summary>
/// A count of the bytes for objects the process of being freed.
/// </summary>
internal static UIntPtr SmallFreedBytes;
/// <summary>
/// A count of the small pages in the process of being freed.
/// </summary>
internal static UIntPtr SmallFreedPages;
/// <summary>
/// A count of the large pages in the process of being freed.
/// </summary>
internal static UIntPtr LargeFreedPages;
/// <summary>
/// A count of the bytes allocated for large objects.
/// </summary>
internal static UIntPtr LargeBytes {
get {
return PageTable.RegionSize(LargePages);
}
}
/// <summary>
/// The number of pages reserved for large objects.
/// </summary>
internal static UIntPtr LargePages;
/// <summary>
/// The number of pages reserved for small objects.
/// </summary>
internal static UIntPtr SmallPages;
/// <summary>
/// This is the total size of data (including object headers)
/// </summary>
internal static UIntPtr TotalBytes {
get {
return SmallBytes + LargeBytes;
}
}
/// <summary>
/// The number of pages managed by the alloc heap, including space
/// occupied by empty cells.
/// </summary>
internal static UIntPtr ReservedBytes {
get {
return PageTable.RegionSize(SmallPages) + LargeBytes;
}
}
/// <summary>
/// Increment the counter of the number of small bytes allocated.
/// </summary>
[Inline]
private static void AddSmallBytes(UIntPtr newBytes) {
SmallBytes += newBytes;
GC.newBytesSinceGC += newBytes;
}
/// <summary>
/// Increment the counter of the number of large bytes allocated.
/// </summary>
[Inline]
private static void AddLargePages(UIntPtr pageCount) {
LargePages += pageCount;
GC.newBytesSinceGC += PageTable.RegionSize(pageCount);
}
/// <summary>
/// Decrement the counter of the number of small bytes allocated.
/// </summary>
[Inline]
private static void SubSmallBytes(UIntPtr newBytes) {
SmallFreedBytes += newBytes;
}
/// <summary>
/// Decrement the counter of the number of large pages allocated.
/// </summary>
[Inline]
private static void SubLargePages(UIntPtr pageCount) {
LargeFreedPages += pageCount;
}
/// <summary>
/// Increment the counter of the number of small pages allocated.
/// </summary>
[Inline]
private static void AddSmallPage() {
SmallPages += new UIntPtr(1);
AddSmallBytes(PageTable.PageSize);
}
/// <summary>
/// Decrement the counter of the number of small pages allocated.
/// </summary>
[Inline]
private static void SubSmallPage(PageHeader *page) {
SmallFreedPages += new UIntPtr(1);
SubSmallBytes(PageTable.PageSize -
(page->cellSize * (UIntPtr) page->freeCount));
}
/// <summary>
/// Subtract and zero the freed data counts.
/// </summary>
internal static void CommitFreedData() {
SmallPages -= SmallFreedPages;
SmallFreedPages = new UIntPtr(0);
LargePages -= LargeFreedPages;
LargeFreedPages = new UIntPtr(0);
SmallBytes -= SmallFreedBytes;
SmallFreedBytes = new UIntPtr(0);
}
#endregion
/// <summary>
/// When notified of the creation of a new thread we initialize the
/// alloc heap in that thread.
/// </summary>
internal unsafe static void NewThreadNotification(Thread newThread,
bool initial)
{
SegregatedFreeListThread mixinThread = MixinThread(newThread);
if (initial) {
// Initialise the initial thread.
mixinThread.segregatedFreeList.localPages = (PageHeader*[])
BootstrapMemory.Allocate(typeof(PageHeader*[]),
SIZE_CLASSES);
mixinThread.segregatedFreeList.freeList = (UIntPtr[])
BootstrapMemory.Allocate(typeof(UIntPtr[]), SIZE_CLASSES);
} else {
// We have to create the thread-specific array of pages.
mixinThread.segregatedFreeList.freeList =
new UIntPtr[SIZE_CLASSES];
mixinThread.segregatedFreeList.localPages =
new PageHeader*[SIZE_CLASSES];
}
}
/// <summary>
/// A thread has finished, so we release any local pages.
/// </summary>
internal static void DeadThreadNotification(Thread deadThread)
{
SegregatedFreeListThread mixinThread = MixinThread(deadThread);
for(int i=0; i < SIZE_CLASSES; i++) {
if (mixinThread.segregatedFreeList.localPages[i] != null) {
mixinThread.segregatedFreeList.ReleaseLocalPage(i);
}
}
}
/// <summary>
/// Allocate a large object. Large objects don't share pages with
/// any other objects. Get memory for the object directly from the
/// PageManager.
/// </summary>
[ManualRefCounts]
private static unsafe UIntPtr AllocateLarge(uint alignment,
UIntPtr bytes,
Thread currentThread)
{
UIntPtr pageCount = PageTable.PageCount(PageTable.PagePad(bytes));
bool fCleanPages = true;
UIntPtr page = PageManager.EnsurePages(currentThread,
pageCount, INIT_PAGE,
ref fCleanPages);
UIntPtr regionSize = PageTable.RegionSize(pageCount);
AddLargePages(pageCount);
int unusedBytes = (int) (regionSize - bytes);
int unusedCacheLines = unusedBytes >> 5;
int offset;
if (unusedCacheLines != 0) {
offset = (largeOffset % unusedCacheLines) << 5;
largeOffset =
(largeOffset + 1) & ((int)PageTable.PageSize - 1);
} else {
offset = 0;
}
UIntPtr pageAddr = PageTable.PageAddr(page);
UIntPtr startAddr = pageAddr + offset + PreHeader.Size;
UIntPtr resultAddr =
Allocator.AlignedObjectPtr(startAddr, alignment);
short shortOff = (short) (resultAddr - pageAddr);
VTable.Assert(shortOff > 0, "offset not positive");
PageTable.SetExtra(page, shortOff);
// Ready to be visited
PageTable.SetType(page, pageCount, LARGE_OBJ_PAGE);
return resultAddr;
}
/// <summary>
/// Free the specified object. For large objects the page becomes
/// immediately available. For small objects it may require a call
/// to RecycleGlobalPages.
/// </summary>
[ManualRefCounts]
internal static void Free(Object obj) {
UIntPtr objectStart = Magic.addressOf(obj) - PreHeader.Size;
UIntPtr page = PageTable.Page(objectStart);
PageType pageType = PageTable.Type(page);
// Free the object
if (pageType == SMALL_OBJ_PAGE) {
FreeSmall(obj);
} else {
VTable.Assert(pageType == LARGE_OBJ_PAGE,
"Found GC Page not small or large");
FreeLarge(obj);
}
}
/// <summary>
/// Free a small object.
/// </summary>
[ManualRefCounts]
internal static void FreeSmall(Object obj) {
UIntPtr objectAddr = Magic.addressOf(obj);
// Put the object memory cell on the freelist for the page
UIntPtr pageAddr = PageTable.PageAlign(objectAddr);
PageHeader *pageHeader = (PageHeader*) pageAddr;
UIntPtr cellStart;
if (obj.vtable.baseAlignment > UIntPtr.Size) {
cellStart = FindSmallCell(objectAddr);
VTable.Assert(cellStart > (UIntPtr) pageHeader &&
cellStart <= objectAddr &&
cellStart+pageHeader->cellSize >= objectAddr,
"find small cell found invalid start");
} else {
cellStart = objectAddr - PreHeader.Size;
}
Util.MemClear(cellStart, pageHeader->cellSize);
SubSmallBytes(pageHeader->cellSize);
UIntPtr oldFreeList;
cellStart = (cellStart + PreHeader.Size);
do {
oldFreeList = pageHeader->freeList;
*(UIntPtr *)cellStart = oldFreeList;
} while (Interlocked.CompareExchange(ref pageHeader->freeList,
cellStart, oldFreeList) != oldFreeList);
Interlocked.Increment(ref pageHeader->freeCount);
}
internal unsafe struct TempList {
private UIntPtr next;
internal void Add(UIntPtr memAddr) {
*(UIntPtr *) memAddr = this.next;
this.next = memAddr;
}
internal UIntPtr GetList() {
UIntPtr result = this.next;
this.next = UIntPtr.Zero;
return result;
}
}
[ManualRefCounts]
internal static void FreeSmallList(ref TempList tempList)
{
UIntPtr cell = tempList.GetList();
if (cell != UIntPtr.Zero) {
PageHeader *pageHeader = (PageHeader *)
PageTable.PageAlign(cell);
UIntPtr newChain = UIntPtr.Zero;
UIntPtr newChainTail = cell + PreHeader.Size;
UIntPtr cellSize = pageHeader->cellSize;
int count = 0;
do {
count++;
UIntPtr next = *(UIntPtr *) cell;
Util.MemClear(cell, cellSize);
UIntPtr cellPtrAddr = cell + PreHeader.Size;
*(UIntPtr *) cellPtrAddr = newChain;
newChain = cellPtrAddr;
cell = next;
} while (cell != UIntPtr.Zero);
SubSmallBytes((UIntPtr) count * cellSize);
UIntPtr oldFreeList;
do {
oldFreeList = pageHeader->freeList;
*(UIntPtr*)newChainTail = oldFreeList;
} while (Interlocked.CompareExchange(ref pageHeader->freeList,
newChain, oldFreeList) !=
oldFreeList);
int oldFreeCount, newFreeCount;
do {
oldFreeCount = pageHeader->freeCount;
newFreeCount = oldFreeCount + count;
} while (Interlocked.CompareExchange(ref pageHeader->freeCount,
newFreeCount,
oldFreeCount) !=
oldFreeCount);
}
}
/// <summary>
/// Free a large object.
/// </summary>
[ManualRefCounts]
internal static void FreeLarge(Object obj) {
UIntPtr objectStart = Magic.addressOf(obj) - PreHeader.Size;
UIntPtr firstPage = PageTable.Page(objectStart);
// Release the page(s) that the object resides on
UIntPtr pageAddr = PageTable.PageAlign(objectStart);
UIntPtr objectSize = ObjectLayout.Sizeof(obj);
UIntPtr limitPage =
PageTable.Page(PageTable.PagePad(objectStart + objectSize));
UIntPtr pageCount = limitPage - firstPage;
PageTable.SetType(firstPage, pageCount, INIT_PAGE);
PageTable.SetExtra(PageTable.Page(pageAddr), 0);
PageManager.ReleaseUnusedPages(firstPage, pageCount, false);
SubLargePages(pageCount);
}
/// <summary>
/// Given a possibly interior pointer to an object, return the
/// real address of the object.
/// </summary>
internal static unsafe UIntPtr Find(UIntPtr addr) {
UIntPtr page = PageTable.Page(addr);
PageType pageType = PageTable.Type(page);
VTable.Assert(PageTable.IsGcPage(pageType),
"Attempt find on non-GC page");
if (pageType == SMALL_OBJ_PAGE) {
return FindSmall(addr);
} else {
return FindLarge(addr);
}
}
/// <summary>
/// Find a small object (after determining it is on a small page)
/// </summary>
private static UIntPtr FindSmall(UIntPtr cellAddr) {
UIntPtr objectAddr = FindSmallCell(cellAddr) + PreHeader.Size;
objectAddr = Allocator.SkipAlignment(objectAddr);
return objectAddr;
}
/// <summary>
/// Find the cell for a given object address
/// </summary>
private static unsafe UIntPtr FindSmallCell(UIntPtr addr) {
UIntPtr pageAddr = PageTable.PageAlign(addr - PreHeader.Size);
PageHeader *pageHeader = (PageHeader *) pageAddr;
UIntPtr firstAddr = pageAddr + PageHeader.SIZEOF;
return
addr - ((int)(addr - firstAddr) % (int)pageHeader->cellSize);
}
/// <summary>
/// Find a large object (after determining it is on a large page)
/// </summary>
private static UIntPtr FindLarge(UIntPtr addr) {
UIntPtr page = PageTable.Page(addr - PreHeader.Size);
short brickData = PageTable.Extra(page);
while (brickData == 0) {
page--;
brickData = PageTable.Extra(page);
VTable.Assert(PageTable.Type(page) == LARGE_OBJ_PAGE,
"page type invalid");
}
return (PageTable.PageAddr(page) + brickData);
}
internal abstract class ObjectVisitor : ObjectLayout.ObjectVisitor {
internal virtual void VisitSmall(Object obj, UIntPtr memAddr) {
this.Visit(obj);
}
internal virtual void VisitSmallPageEnd() { }
internal virtual UIntPtr VisitLarge(Object obj) {
return this.Visit(obj);
}
internal override UIntPtr Visit(Object obj) {
VTable.NotReached("Someone forgot an override method in a "+
"subclass of SegregatedFreeList.ObjectVisitor");
return UIntPtr.Zero;
}
}
// Wraps an SegregatedFreeList.ObjectVisitor around a
// ObjectLayout.ObjectVisitor.
// Both large and small objects are visited by the same
// ObjectLayout.ObjectVisitor.
internal class ObjectVisitorWrapper : ObjectVisitor {
private ObjectLayout.ObjectVisitor visitor;
internal ObjectVisitorWrapper(ObjectLayout.ObjectVisitor visitor) {
this.visitor = visitor;
}
internal override void VisitSmall(Object obj, UIntPtr memAddr) {
this.visitor.Visit(obj);
}
internal override UIntPtr VisitLarge(Object obj) {
return this.visitor.Visit(obj);
}
}
/// <summary>
/// Visit each object in the heap across all pages.
/// </summary>
[ManualRefCounts]
internal static void VisitAllObjects(ObjectVisitor visitor)
{
VisitObjects(UIntPtr.Zero, PageTable.pageTableCount, visitor);
}
/// <summary>
/// Visit each object in the heap across a range of pages.
///
/// This can be run concurrent to allocations, but not frees.
/// </summary>
[ManualRefCounts]
internal static void VisitObjects(UIntPtr lowPage,
UIntPtr highPage,
ObjectVisitor visitor)
{
for (UIntPtr i = lowPage; i < highPage; i++) {
PageType pageType = PageTable.MyType(i);
if (pageType == INIT_PAGE) {
// Not yet ready for allocation so we can't visit it...
} else if (pageType == SMALL_OBJ_PAGE) {
VisitSmallObjects(i, visitor);
} else if (pageType == LARGE_OBJ_PAGE) {
UIntPtr j = i + 1;
while (j < highPage &&
PageTable.MyType(j) == LARGE_OBJ_PAGE) {
j++;
}
UIntPtr largeObjectPageCount =
VisitLargeObject(i, visitor);
i += largeObjectPageCount - 1;
VTable.Assert(i <= j);
}
}
}
internal static void VisitObjects(UIntPtr lowPage,
UIntPtr highPage,
ObjectLayout.ObjectVisitor visitor)
{
ObjectVisitor myObjectVisitor = visitor as ObjectVisitor;
VTable.Assert(myObjectVisitor != null,
"SegregatedFreeList requires specialized ObjectVisitor");
VisitObjects(lowPage, highPage, myObjectVisitor);
}
/// <summary>
/// Visit small objects in a single page.
/// </summary>
[ManualRefCounts]
private static unsafe void VisitSmallObjects(UIntPtr page,
ObjectVisitor visitor)
{
VTable.Assert(PageTable.Type(page) == SMALL_OBJ_PAGE,
"Visiting small objects on invalid page");
UIntPtr pageAddr = PageTable.PageAddr(page);
PageHeader *pageHeader = (PageHeader *) pageAddr;
UIntPtr cellSize = pageHeader->cellSize;
VTable.Assert(cellSize != UIntPtr.Zero,
"zero cellSize visiting small");
UIntPtr lowAddr = pageAddr + PageHeader.SIZEOF;
UIntPtr highAddr = PageTable.PagePad(lowAddr);
while (lowAddr <= highAddr - cellSize) {
UIntPtr objectAddr = lowAddr + PreHeader.Size;
objectAddr = Allocator.SkipAlignment(objectAddr);
Object maybeObject = Magic.fromAddress(objectAddr);
UIntPtr vtablePtr = *maybeObject.VTableFieldAddr;
if (vtablePtr != 0) {
// In Singularity, it is not always allowed
// to compute PageTable.Page(UIntPtr.Zero);
UIntPtr vtablePage = PageTable.Page(vtablePtr);
if (vtablePage != page) {
// We have found a slot containing an object
visitor.VisitSmall(maybeObject, lowAddr);
}
}
lowAddr += cellSize;
}
visitor.VisitSmallPageEnd();
}
/// <summary>
/// Visit a large object on the specified page.
/// </summary>
[ManualRefCounts]
private static UIntPtr VisitLargeObject(UIntPtr page,
ObjectVisitor visitor)
{
VTable.Assert(PageTable.Type(page) == LARGE_OBJ_PAGE,
"Visiting large object on invalid page");
// Find the object
UIntPtr pageAddr = PageTable.PageAddr(page);
short brickData = PageTable.Extra(page);
if (brickData == 0) {
// Possibly in the process of being allocated.
return new UIntPtr(1);
}
UIntPtr objectAddr = pageAddr + brickData;
Object obj = Magic.fromAddress(objectAddr);
if (obj.vtable == null) {
// Memory has been allocated, but object is not initialized
return new UIntPtr(1);
}
// Visit the object
UIntPtr objectSize = visitor.VisitLarge(obj);
// Return the page count
UIntPtr objectEnd = objectAddr + objectSize - PreHeader.Size;
UIntPtr limitPage = PageTable.Page(PageTable.PagePad(objectEnd));
return limitPage - page;
}
/// <summary>
/// This is the the free list of pages to allocate from.
/// </summary>
private static UIntPtr[] globalFreePages;
/// <summary>
/// This is the list of pages released by threads. These pages must
/// be periodically processes to release them back for allocation if
/// necessary.
/// </summary>
private static UIntPtr[] globalPages;
// Used by RecycleGlobalPages to avoid the ABA problem of
// lock-free data structures.
private static UIntPtr[] stolenGlobalFreePages;
private static UIntPtr[] stolenGlobalFullPages;
/// <summary>
/// Initialize the alloc heap by setting up the heads for all the
/// linked lists.
/// </summary>
[PreInitRefCounts]
internal static unsafe void Initialize() {
// Global array of allocated pages
globalPages = (UIntPtr[])
BootstrapMemory.Allocate(typeof(UIntPtr[]), SIZE_CLASSES);
// Global array of pages with free elements
globalFreePages = (UIntPtr[])
BootstrapMemory.Allocate(typeof(UIntPtr[]), SIZE_CLASSES);
// Temporary list holders used by RecycleGlobalPages
stolenGlobalFullPages = (UIntPtr[])
BootstrapMemory.Allocate(typeof(UIntPtr[]), SIZE_CLASSES);
stolenGlobalFreePages = (UIntPtr[])
BootstrapMemory.Allocate(typeof(UIntPtr[]), SIZE_CLASSES);
}
/// <summary>
/// Take all global pages that have had elements freed and put them in
/// the allocation queues.
/// </summary>
internal static void RecycleGlobalPages() {
RecycleGlobalPagesPhase1();
RecycleGlobalPagesPhase2();
}
internal static void RecycleGlobalPagesPhase1() {
for (int i=0; i < SIZE_CLASSES; i++) {
// Steal chains
VTable.Assert(stolenGlobalFullPages[i] == UIntPtr.Zero);
VTable.Assert(stolenGlobalFreePages[i] == UIntPtr.Zero);
stolenGlobalFullPages[i] =
AtomicPopChain(ref globalPages[i]);
stolenGlobalFreePages[i] =
AtomicPopChain(ref globalFreePages[i]);
}
}
internal static void RecycleGlobalPagesPhase2() {
for (int i=0; i < SIZE_CLASSES; i++) {
UIntPtr globalFull = stolenGlobalFullPages[i];
stolenGlobalFullPages[i] = UIntPtr.Zero;
UIntPtr globalFree = stolenGlobalFreePages[i];
stolenGlobalFreePages[i] = UIntPtr.Zero;
// Start with free pages (they can not become full)
UIntPtr current = globalFree;
// New free and full chains
PageHeader *freeHead = null, freeTail = null;
PageHeader *fullHead = null, fullTail = null;
// Determine starting point
if (current == UIntPtr.Zero) {
current = globalFull;
globalFull = UIntPtr.Zero;
if (current == UIntPtr.Zero) {
continue;
}
}
// Number of cells of this size class in a block
int cells = BLOCK_SIZE / (int) GetCellSize(i);
VTable.Assert(cells > 0 && cells < (BLOCK_SIZE >> 1),
"invalid cell count");
// Iterate through list
while (current != UIntPtr.Zero) {
PageHeader *page = (PageHeader*) current;
current = page->nextPage;
if (page->freeCount == cells) {
// Completely Free Page
SubSmallPage(page);
UIntPtr pageNum = PageTable.Page(new UIntPtr(page));
PageTable.SetType(pageNum, INIT_PAGE);
PageTable.SetExtra(pageNum, 0);
PageManager.ReleaseUnusedPages(pageNum, new UIntPtr(1),
false);
} else if (page->freeCount > 0) {
// Partially Free Page
AddPageToList(page, ref freeHead, ref freeTail);
} else {
// Completely Full Page
AddPageToList(page, ref fullHead, ref fullTail);
}
if (current == UIntPtr.Zero) {
// Finished the free list, onto the full list.
current = globalFull;
globalFull = UIntPtr.Zero;
}
}
// Reinsert values onto the free and full chains
if (fullHead != null) {
AtomicPushChain(ref globalPages[i],
fullHead, fullTail);
}
if (freeHead != null) {
AtomicPushChain(ref globalFreePages[i],
freeHead, freeTail);
}
}
}
/// <summary>
/// Add a page onto a local linked list (possibly the first page)
/// </summary>
private static void AddPageToList(PageHeader *page,
ref PageHeader *head,
ref PageHeader *tail) {
if (head == null) {
page->nextPage = UIntPtr.Zero;
head = page;
tail = page;
return;
}
tail->nextPage = new UIntPtr(page);
tail = page;
}
/// <summary>
/// Atomically push a value onto the linked list.
/// </summary>
private static void AtomicPush(ref UIntPtr head, PageHeader *page) {
AtomicPushChain(ref head, page, page);
}
/// <summary>
/// Atomically remove a value from the linked list. Returns null if the
/// list is empty.
/// </summary>
private static UIntPtr AtomicPop(ref UIntPtr head) {
UIntPtr oldHead;
PageHeader *oldPage;
UIntPtr newHead;
do {
oldHead = head;
if (oldHead == UIntPtr.Zero) {
// Empty list.
return UIntPtr.Zero;
}
oldPage = (PageHeader*) oldHead;
newHead = oldPage->nextPage;
} while(oldHead != Interlocked.CompareExchange
(ref head, newHead, oldHead));
oldPage->nextPage = UIntPtr.Zero;
return oldHead;
}
/// <summary>
/// Steal an entire list.
/// </summary>
private static UIntPtr AtomicPopChain(ref UIntPtr head) {
return Interlocked.Exchange(ref head, UIntPtr.Zero);
}
/// <summary>
/// Push a whole chain onto a list.
/// </summary>
private static void AtomicPushChain(ref UIntPtr head,
PageHeader *chainHead,
PageHeader *chainTail) {
UIntPtr oldHead;
UIntPtr newHead = new UIntPtr(chainHead);
do {
oldHead = head;
chainTail->nextPage = oldHead;
} while (oldHead != Interlocked.CompareExchange
(ref head, newHead, oldHead));
}
/// <summary>
/// This struct represents the header data stored in each
/// small object page.
///
/// BUGBUG: Not space efficient.
/// </summary>
private struct PageHeader {
internal const int SIZEOF = 16;
/// <summary>
/// The next page in the linked list.
/// </summary>
internal UIntPtr nextPage;
/// <summary>
/// The head of the free list for this page. This is not
/// used when a page is assigned to a thread.
/// </summary>
internal UIntPtr freeList;
/// <summary>
/// The cell size for objects in this page.
/// </summary>
internal UIntPtr cellSize;
/// <summary>
/// The number of cells that have been freed. This is used
/// for accounting purposes.
/// </summary>
internal int freeCount;
}
#endregion
#region Local (Safe from the context of owner thread)
/// <summary>
/// This is a thread's local free list for each size class.
/// </summary>
[RequiredByBartok]
private UIntPtr[] freeList;
/// <summary>
/// This is a thread's local set of pages for each size class.
/// </summary>
private PageHeader*[] localPages;
[ManualRefCounts]
internal static UIntPtr Allocate(Thread thread,
UIntPtr bytes, uint alignment)
{
SegregatedFreeListThread mixinThread = MixinThread(thread);
return mixinThread.segregatedFreeList.Allocate(bytes, alignment,
thread);
}
[ManualRefCounts]
internal UIntPtr Allocate(UIntPtr bytes, uint alignment, Thread thread)
{
UIntPtr resultAddr = this.AllocateFast(bytes, alignment);
if (resultAddr == UIntPtr.Zero) {
resultAddr = this.AllocateSlow(bytes, alignment, thread);
}
return resultAddr;
}
[Inline]
[ManualRefCounts]
internal static UIntPtr AllocateFast(Thread thread,
UIntPtr bytes,
uint alignment)
{
SegregatedFreeListThread mixinThread = MixinThread(thread);
return mixinThread.segregatedFreeList.AllocateFast(bytes,
alignment);
}
[RequiredByBartok]
[Inline]
[DisableBoundsChecks]
public static unsafe Object CompilerAllocateMarkSweep
(VTable vtable, Thread currentThread, UIntPtr bytes, uint alignment)
{
VTable.Assert((alignment == 4)
|| ((alignment == 8) && (UIntPtr.Size == 8))
|| ((alignment == 8) && (PreHeader.Size == 4)),
"Unsupported object layout");
VTable.Assert(UIntPtr.Size == PreHeader.Size,
"Unsupported preheader size");
VTable.Assert
(Util.IsAligned((uint) PreHeader.Size + (uint)PostHeader.Size,
alignment),
"Unsupported header sizes");
VTable.Assert(bytes < LARGE_OBJECT_THRESHOLD,
"CompilerAllocate called for large object");
// Room to ensure alignment
bool alignRequired = (alignment > UIntPtr.Size);
if (alignRequired) {
bytes = bytes + alignment - UIntPtr.Size;
}
int sizeClass = GetSizeClass(bytes);
SegregatedFreeListThread mixinThread = MixinThread(currentThread);
UIntPtr region = mixinThread.segregatedFreeList.freeList[sizeClass];
if (region != UIntPtr.Zero) {
mixinThread.segregatedFreeList.freeList[sizeClass] =
*(UIntPtr *)region;
// Zero out the free list data structure. However, if the
// vtable is at offset zero, then we're going to overwrite it
// anyway. (and the optimizer does not see this through the
// unmanaged/managed conversion)
if (Magic.OffsetOfVTable != UIntPtr.Zero) {
*(UIntPtr *)region = UIntPtr.Zero;
}
UIntPtr objAddr = region;
if((alignment == 8) && (UIntPtr.Size == 4)) {
// Since 'objAddr' will be the actual object reference, we
// want to misalign it here so that the object payload will
// be aligned. (We know that PostHeader.Size & 8 == 4
// because the PreHeader is 4 and the sum of the PreHeader
// and PostHeader sizes is a multiple of alignment (8))
// Store alignment token at objAddr. This will be where an
// alignment token should go if it is required...
Allocator.WriteAlignment(objAddr);
// ... (align if necessary) ...
objAddr = Util.Align(objAddr, (UIntPtr) alignment);
// ... or where the object header will be if alignment was
// not necessary. This code zeroes the object header
// regardless and avoids a branch in this fast path.
*(UIntPtr *)objAddr = UIntPtr.Zero;
// Finally misalign 'objAddr'
objAddr += PreHeader.Size;
}
Object obj = Magic.fromAddress(objAddr);
obj.vtable = vtable;
return obj;
}
return GC.AllocateObjectNoInline(vtable, currentThread);
}
[Inline]
[ManualRefCounts]
private UIntPtr AllocateFast(UIntPtr bytes, uint alignment)
{
// Room to ensure alignment
bool alignRequired = (alignment > UIntPtr.Size);
if (alignRequired) {
bytes = bytes + alignment - UIntPtr.Size;
}
// Is this a large object?
if (!(bytes < LARGE_OBJECT_THRESHOLD)) {
return UIntPtr.Zero;
}
int sizeClass = GetSizeClass(bytes);
UIntPtr region = AllocateSmallFast(sizeClass);
if (region == UIntPtr.Zero) {
return UIntPtr.Zero;
} else {
UIntPtr resultAddr =
Allocator.AlignedObjectPtr(region, alignment);
return resultAddr;
}
}
[Inline]
[ManualRefCounts]
internal static UIntPtr AllocateSlow(Thread thread,
UIntPtr bytes, uint alignment)
{
SegregatedFreeListThread mixinThread = MixinThread(thread);
return mixinThread.segregatedFreeList.AllocateSlow(bytes,
alignment,
thread);
}
[NoInline]
[ManualRefCounts]
private UIntPtr AllocateSlow(UIntPtr bytes, uint alignment,
Thread currentThread)
{
// Room to ensure alignment
bool alignRequired = (alignment > UIntPtr.Size);
if (alignRequired) {
bytes = bytes + alignment - UIntPtr.Size;
}
// Is this a large object?
if (!(bytes < LARGE_OBJECT_THRESHOLD)) {
return AllocateLarge(alignment, bytes, currentThread);
}
int sizeClass = GetSizeClass(bytes);
UIntPtr region = AllocateSmall(sizeClass, currentThread);
UIntPtr resultAddr =
Allocator.AlignedObjectPtr(region, alignment);
return resultAddr;
}
/// <summary>
/// Used to attempt to spread large objects across pages to avoid
/// higher cache conflicts on low page addresses.
/// </summary>
private static int largeOffset;
/// <summary>
/// Allocate an object of a specified size class from the
/// thread's local block.
/// </summary>
[Inline]
[ManualRefCounts]
private UIntPtr AllocateSmall(int sizeClass, Thread currentThread) {
UIntPtr region = freeList[sizeClass];
if (region != UIntPtr.Zero) {
freeList[sizeClass] = *(UIntPtr*)region;
*(UIntPtr*)region = UIntPtr.Zero;
return region;
} else {
return AllocateSmallSlow(sizeClass, currentThread);
}
}
[Inline]
private UIntPtr AllocateSmallFast(int sizeClass) {
UIntPtr region = freeList[sizeClass];
if (region != UIntPtr.Zero) {
freeList[sizeClass] = *(UIntPtr*)region;
*(UIntPtr*)region = UIntPtr.Zero;
return region;
} else {
return UIntPtr.Zero;
}
}
/// <summary>
/// Get a new page and allocate into it.
/// </summary>
/// <param name="sizeClass"></param>
/// <returns></returns>
[ManualRefCounts]
private UIntPtr AllocateSmallSlow(int sizeClass, Thread currentThread)
{
// Return our old page
if (localPages[sizeClass] != null) {
ReleaseLocalPage(sizeClass);
}
// Get a new one.
localPages[sizeClass] = GetLocalPage(sizeClass, currentThread);
VTable.Assert(localPages[sizeClass] != null, "no local page");
// Read (and then zero) the free list.
freeList[sizeClass] = Interlocked.Exchange
(ref localPages[sizeClass]->freeList, UIntPtr.Zero);
VTable.Assert(freeList[sizeClass] != UIntPtr.Zero,
"GetLocalPage returned empty page");
// Allocate off the free list
return AllocateSmallFast(sizeClass);
}
/// <summary>
/// Release a local allocation page into the pool of consumed pages.
/// </summary>
[ManualRefCounts]
private void ReleaseLocalPage(int sizeClass) {
PageHeader *page = localPages[sizeClass];
// Prepare the page to be released
if (freeList[sizeClass] != UIntPtr.Zero) {
// We are releasing the page 'early'.
UIntPtr pageFreeList = freeList[sizeClass];
freeList[sizeClass] = UIntPtr.Zero;
// Save our local free list as the page's free list
UIntPtr oldFreeList = Interlocked.CompareExchange
(ref page->freeList, pageFreeList, UIntPtr.Zero);
// Count the reclaimed cells.
int freeCount = 0;
UIntPtr next = pageFreeList;
while (next != UIntPtr.Zero) {
next = *(UIntPtr*)next;
freeCount++;
}
SubSmallBytes(new UIntPtr((uint)freeCount *
(uint)page->cellSize));
if (oldFreeList != UIntPtr.Zero) {
// Page already had a free list, follow it to the end.
while(*(UIntPtr*)oldFreeList != UIntPtr.Zero) {
oldFreeList = *(UIntPtr*)oldFreeList;
}
// And stitch the free lists together
*(UIntPtr*)oldFreeList = pageFreeList;
}
// Set the reclaimed cells.
int old;
do {
old = page->freeCount;
} while (old != Interlocked.CompareExchange
(ref page->freeCount, old + freeCount, old));
}
// Atomically insert the page in the globalPages queue.
AtomicPush(ref globalPages[sizeClass], page);
}
/// <summary>
/// Either reuse an existing or make a new page to allocate into.
/// </summary>
[ManualRefCounts]
private PageHeader * GetLocalPage(int sizeClass, Thread currentThread)
{
GC.CheckForNeededGCWork(currentThread);
UIntPtr page = AtomicPop(ref globalFreePages[sizeClass]);
if (page != UIntPtr.Zero) {
// We got the page
PageHeader *pageHeader = (PageHeader*) page;
VTable.Assert(pageHeader->freeCount > 0, "empty FreePage");
AddSmallBytes(new UIntPtr((uint)pageHeader->freeCount *
(uint)pageHeader->cellSize));
pageHeader->freeCount = 0;
return pageHeader;
}
// Create a new local page.
return NewLocalPage(sizeClass, currentThread);
}
/// <summary>
/// Create a new page to allocate into
/// </summary>
[ManualRefCounts]
private PageHeader * NewLocalPage(int sizeClass, Thread currentThread)
{
VTable.Assert(sizeClass > 0, "non-positive sizeClass");
bool fCleanPages = true;
UIntPtr page = PageManager.EnsurePages(currentThread,
(UIntPtr) 1, INIT_PAGE,
ref fCleanPages);
AddSmallPage();
UIntPtr pageAddr = PageTable.PageAddr(page);
PageTable.SetExtra(page, (short) sizeClass);
PageHeader *pageHeader = (PageHeader *) pageAddr;
// Set up the free list of free slots
UIntPtr stride = GetCellSize(sizeClass);
VTable.Assert(stride != UIntPtr.Zero, "Zero Stride");
pageHeader->cellSize = stride;
UIntPtr cursor =
pageAddr + PageHeader.SIZEOF + PreHeader.Size;
pageHeader->freeList = cursor;
UIntPtr limit =
PageTable.PageAddr(page+1) - stride + PreHeader.Size;
UIntPtr nextAddr = cursor + stride;
while (nextAddr <= limit) {
*(UIntPtr*)cursor = nextAddr;
cursor = nextAddr;
nextAddr = cursor + stride;
}
PageTable.SetType(page, SMALL_OBJ_PAGE);
return pageHeader;
}
#endregion
}
}