1725 lines
64 KiB
C#
1725 lines
64 KiB
C#
//
|
|
// Microsoft Research Singularity
|
|
//
|
|
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
//
|
|
// File: SharedHeap.cs - Heap for memory passed between processes.
|
|
//
|
|
|
|
using System;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Runtime.InteropServices;
|
|
using System.Threading;
|
|
using System.GCs;
|
|
using System.Collections;
|
|
using System.Diagnostics;
|
|
|
|
using Microsoft.Singularity;
|
|
using Microsoft.Singularity.Channels;
|
|
|
|
namespace Microsoft.Singularity.Memory
|
|
{
|
|
// <summary>
|
|
// Non-garbage collected heap for memory transferred between processes.
|
|
//
|
|
// This heap supports explicit Allocate/Free operations of arbitrary sizes.
|
|
//
|
|
// An allocated region may be split into two separate regions at a later
|
|
// time. The split may occur at an arbitrary position inside the original
|
|
// region. These split regions may be further split. In order to support
|
|
// this feature, the heap meta-data for tracking allocated regions is kept
|
|
// on the side, instead of directly adjacent to the allocation.
|
|
//
|
|
// Each allocated region is owned by a process. All regions owned by a
|
|
// process may be quickly found and deleted upon process death.
|
|
//
|
|
// An allocated region's ownership may be transferred between owners.
|
|
// A region may also have multiple owners.
|
|
// </summary>
|
|
[NoCCtor]
|
|
[CLSCompliant(false)]
|
|
public class SharedHeap {
|
|
|
|
//
|
|
// The system always has at least one shared heap
|
|
//
|
|
private static SharedHeap kernelSharedHeap;
|
|
|
|
public static SharedHeap KernelSharedHeap {
|
|
get {
|
|
DebugStub.Assert(kernelSharedHeap != null);
|
|
return kernelSharedHeap;
|
|
}
|
|
}
|
|
|
|
public static SharedHeap CurrentProcessSharedHeap {
|
|
get {
|
|
return Thread.CurrentProcess.ProcessSharedHeap;
|
|
}
|
|
}
|
|
|
|
#if PAGING
|
|
internal static unsafe void Initialize()
|
|
{
|
|
kernelSharedHeap = new SharedHeap(ProtectionDomain.DefaultDomain, MemoryManager.GetKernelRange());
|
|
}
|
|
|
|
// Debug-only sanity check that we are in the right protection domain.
|
|
protected void CheckDomain()
|
|
{
|
|
//
|
|
// If this assertion fails, we are not operating within
|
|
// correct address space. This is likely a serious problem; check
|
|
// to make sure the caller is properly checking that this
|
|
// shared heap is, in fact, mapped before trying to use it!
|
|
//
|
|
if (!parentDomain.KernelMode) {
|
|
DebugStub.Assert(
|
|
Processor.GetCurrentAddressSpace() ==
|
|
parentDomain.AddressSpace);
|
|
}
|
|
}
|
|
#else
|
|
internal static void Initialize()
|
|
{
|
|
kernelSharedHeap = new SharedHeap();
|
|
}
|
|
|
|
protected void CheckDomain()
|
|
{
|
|
// Do nothing; there are no protection domains on a non-PAGING build
|
|
}
|
|
#endif
|
|
|
|
#region Small/Large Region Accounting
|
|
// <summary>
|
|
// If we can fit multiple regions of a certain size on a single page,
|
|
// then we consider them to be "small" regions. Large regions get
|
|
// sole usage of their page(s).
|
|
// </summary>
|
|
private const uint MaxSmallRegion = MemoryManager.PageSize / 2;
|
|
|
|
// <summary>
|
|
// Small regions fall into various raw bucket sizes:
|
|
//
|
|
// Bucket Raw Size 4K page example
|
|
// 0 0 to round(MaxSmallRegion / 128) 16
|
|
// 1 prior to round(MaxSmallRegion / 64) 32
|
|
// 2 prior to round(MaxSmallRegion / 32) 64
|
|
// 3 prior to round(MaxSmallRegion / 16) 128
|
|
// 4 prior to round(MaxSmallRegion / 8) 256
|
|
// 5 prior to round(MaxSmallRegion / 4) 512
|
|
// 6 prior to round(MaxSmallRegion / 2) 1024
|
|
// 7 prior to MaxSmallRegion 2048
|
|
//
|
|
// Where round() rounds down to UIntPtr-size boundaries.
|
|
// Actual space available to user is the raw size minus the
|
|
// size of the small region header.
|
|
// </summary>
|
|
private const byte TotalBuckets = 8;
|
|
private unsafe UIntPtr *freeSmallRegions;
|
|
|
|
#if PAGING
|
|
private VirtualMemoryRange memoryRange; // where we get our pages
|
|
private ProtectionDomain parentDomain;
|
|
#endif
|
|
|
|
// <summary>
|
|
// Given a bucket number, return the size of the regions
|
|
// stored in that bucket.
|
|
// </summary>
|
|
[Inline]
|
|
private static UIntPtr BucketSize(byte bucket)
|
|
{
|
|
DebugStub.Assert(bucket < TotalBuckets);
|
|
|
|
return (UIntPtr)(((MaxSmallRegion >> (TotalBuckets - 1)) &
|
|
~(UIntPtr.Size - 1)) << bucket);
|
|
}
|
|
|
|
// <summary>
|
|
// Given a desired size, return the number of the smallest bucket
|
|
// which can hold something of the desired size.
|
|
// </summary>
|
|
private static byte BucketForSize(UIntPtr size)
|
|
{
|
|
DebugStub.Assert(size <= MaxSmallRegion);
|
|
|
|
for (byte Loop = 0; Loop < TotalBuckets - 1; Loop++) {
|
|
if (size <= BucketSize(Loop)) {
|
|
return Loop;
|
|
}
|
|
}
|
|
|
|
return TotalBuckets - 1;
|
|
}
|
|
|
|
|
|
// <summary>
|
|
// Header for small regions.
|
|
// A small region is not returned to its bucket's free list until all
|
|
// of its shared and split-off pieces have been freed. The reference
|
|
// count tracks outstanding views into a region.
|
|
// </summary>
|
|
private struct SmallRegion {
|
|
internal uint referenceCount;
|
|
}
|
|
|
|
// <summary>
|
|
// Increment the reference count on a small region.
|
|
//
|
|
// BUGBUG: Only works where sizeof(UIntPtr) = sizeof(uint)
|
|
// Need another version of Interlocked.CompareExchange for uints.
|
|
//
|
|
// BUGBUG: Should handle ref count overflow more gracefully.
|
|
// </summary>
|
|
private static unsafe void AtomicReferenceSmallRegion(UIntPtr region)
|
|
{
|
|
uint *refCountLocation;
|
|
uint oldRefCount, newRefCount;
|
|
|
|
refCountLocation = &(((SmallRegion *)region)->referenceCount);
|
|
|
|
do {
|
|
oldRefCount = *refCountLocation;
|
|
DebugStub.Assert(oldRefCount != uint.MaxValue);
|
|
newRefCount = oldRefCount + 1;
|
|
} while (oldRefCount != Interlocked.CompareExchange(
|
|
(UIntPtr *)refCountLocation, newRefCount,
|
|
oldRefCount));
|
|
}
|
|
|
|
// <summary>
|
|
// Decrement the reference count on a small region.
|
|
//
|
|
// BUGBUG: Only works where sizeof(UIntPtr) = sizeof(uint)
|
|
// Need another version of Interlocked.CompareExchange for uints.
|
|
// </summary>
|
|
private static unsafe uint AtomicDereferenceSmallRegion(
|
|
UIntPtr region)
|
|
{
|
|
uint *refCountLocation;
|
|
uint oldRefCount, newRefCount;
|
|
|
|
refCountLocation = &(((SmallRegion *)region)->referenceCount);
|
|
|
|
do {
|
|
oldRefCount = *refCountLocation;
|
|
DebugStub.Assert(oldRefCount != 0);
|
|
newRefCount = oldRefCount - 1;
|
|
} while (oldRefCount != Interlocked.CompareExchange(
|
|
(UIntPtr *)refCountLocation, newRefCount,
|
|
oldRefCount));
|
|
|
|
return newRefCount;
|
|
}
|
|
|
|
|
|
// <summary>
|
|
// Masks to retrieve various interpretations of the "extra" 12 bits
|
|
// contained in a page descriptor. One bit differentiates pages
|
|
// broken into small regions from those comprising large regions.
|
|
// The remainder are used to indicate the bucket (for small regions)
|
|
// or the reference count on the page (for large regions).
|
|
// </summary>
|
|
private const uint SmallRegionBit = 0x00008000u;
|
|
private const uint SmallRegionBucketMask = 0x00000ff0u;
|
|
private const byte SmallRegionBucketShift = 4;
|
|
private const uint LargeRegionReferenceMask = 0x00007ff0u;
|
|
private const byte LargeRegionReferenceShift = 4;
|
|
|
|
[Inline]
|
|
private static uint BucketToExtra(byte bucket)
|
|
{
|
|
return (uint)(SmallRegionBit |
|
|
((uint)bucket << SmallRegionBucketShift));
|
|
}
|
|
|
|
[Inline]
|
|
private static uint LargeRefToExtra(uint refCount)
|
|
{
|
|
return (uint)(refCount << LargeRegionReferenceShift);
|
|
}
|
|
|
|
// <summary>
|
|
// Increment the reference count on a large region page.
|
|
// Note: This method relies upon page descriptor format.
|
|
//
|
|
// BUGBUG: Only works where sizeof(UIntPtr) = sizeof(uint)
|
|
// Need another version of Interlocked.CompareExchange for uints.
|
|
//
|
|
// BUGBUG: Should handle ref count overflow more gracefully.
|
|
// </summary>
|
|
private unsafe void AtomicReferencePage(UIntPtr page)
|
|
{
|
|
uint *descriptorLocation;
|
|
uint oldDescriptor, refCount, newDescriptor;
|
|
|
|
descriptorLocation = SharedHeapPageTable + page;
|
|
|
|
do {
|
|
oldDescriptor = *descriptorLocation;
|
|
refCount = (oldDescriptor & LargeRegionReferenceMask)
|
|
>> LargeRegionReferenceShift;
|
|
DebugStub.Assert(refCount != (LargeRegionReferenceMask >>
|
|
LargeRegionReferenceShift));
|
|
refCount++;
|
|
newDescriptor = (oldDescriptor & ~LargeRegionReferenceMask) |
|
|
(refCount << LargeRegionReferenceShift);
|
|
|
|
} while (oldDescriptor != Interlocked.CompareExchange(
|
|
(UIntPtr *)descriptorLocation, newDescriptor,
|
|
oldDescriptor));
|
|
}
|
|
|
|
// <summary>
|
|
// Decrement the reference count on a large region page.
|
|
// Note: This method relies upon page descriptor format.
|
|
//
|
|
// BUGBUG: Only works where sizeof(UIntPtr) = sizeof(uint)
|
|
// Need another version of Interlocked.CompareExchange for uints.
|
|
// </summary>
|
|
private unsafe uint AtomicDereferencePage(UIntPtr page)
|
|
{
|
|
uint *descriptorLocation;
|
|
uint oldDescriptor, refCount, newDescriptor;
|
|
|
|
descriptorLocation = SharedHeapPageTable + page;
|
|
|
|
do {
|
|
oldDescriptor = *descriptorLocation;
|
|
refCount = (oldDescriptor & LargeRegionReferenceMask)
|
|
>> LargeRegionReferenceShift;
|
|
DebugStub.Assert(refCount != 0);
|
|
refCount--;
|
|
newDescriptor = (oldDescriptor & ~LargeRegionReferenceMask) |
|
|
(refCount << LargeRegionReferenceShift);
|
|
|
|
} while (oldDescriptor != Interlocked.CompareExchange(
|
|
(UIntPtr *)descriptorLocation, newDescriptor,
|
|
oldDescriptor));
|
|
|
|
return refCount;
|
|
}
|
|
|
|
// <summary>
|
|
// Increment the reference count(s) on the page(s) or small region.
|
|
// </summary>
|
|
private unsafe void ReferenceRegion(
|
|
UIntPtr start, UIntPtr size)
|
|
{
|
|
UIntPtr page;
|
|
uint pageDescriptor;
|
|
|
|
if (size == 0) {
|
|
//
|
|
// Zero-sized allocations aren't backed by actual memory
|
|
// and thus have no reference count to maintain.
|
|
//
|
|
return;
|
|
}
|
|
|
|
page = PageFromAddr(start);
|
|
pageDescriptor = *(SharedHeapPageTable + page);
|
|
|
|
if ((pageDescriptor & SmallRegionBit) == SmallRegionBit) {
|
|
byte bucket;
|
|
UIntPtr region;
|
|
UIntPtr regionSize;
|
|
|
|
//
|
|
// This is a small region. Figure out which bucket it's from.
|
|
// Using the bucket size, we can find the start of the region
|
|
// and access the region header.
|
|
//
|
|
bucket = (byte)((pageDescriptor & SmallRegionBucketMask) >>
|
|
SmallRegionBucketShift);
|
|
regionSize = BucketSize(bucket);
|
|
region = start & ~(regionSize - 1);
|
|
|
|
AtomicReferenceSmallRegion(region);
|
|
|
|
} else {
|
|
//
|
|
// This is a large region. We need to walk all the pages
|
|
// it covers, and increment the reference count on each.
|
|
//
|
|
UIntPtr lastPage;
|
|
UIntPtr lastByteAddress;
|
|
|
|
lastByteAddress = start + size - 1;
|
|
lastPage = PageFromAddr(lastByteAddress);
|
|
|
|
while (page <= lastPage) {
|
|
AtomicReferencePage(page);
|
|
page++;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Decrement the reference count(s) on the page(s) or small region.
|
|
/// </summary>
|
|
/// <returns>true if last reference to this region was freed.</returns>
|
|
private unsafe bool DereferenceRegion(UIntPtr start, UIntPtr size)
|
|
{
|
|
// Dereference and release underlying memory
|
|
return DereferenceRegion(start, size, false);
|
|
}
|
|
|
|
// The dontFreePages argument indicates whether to actually release
|
|
// memory pages back to the operating system. If the caller
|
|
// specifies true, it acquires ownership of the memory pages if
|
|
// the return address is true (indicating that no more allocation
|
|
// records refer to the memory)
|
|
private unsafe bool DereferenceRegion(UIntPtr start, UIntPtr size,
|
|
bool dontFreePages)
|
|
{
|
|
UIntPtr page;
|
|
uint pageDescriptor;
|
|
bool lastReference = true;
|
|
|
|
if (size == 0) {
|
|
//
|
|
// Zero-sized allocations aren't backed by actual memory
|
|
// and thus have no reference count to maintain.
|
|
//
|
|
return lastReference;
|
|
}
|
|
|
|
page = PageFromAddr(start);
|
|
pageDescriptor = *(SharedHeapPageTable + page);
|
|
|
|
if ((pageDescriptor & SmallRegionBit) == SmallRegionBit) {
|
|
byte bucket;
|
|
UIntPtr region;
|
|
UIntPtr regionSize;
|
|
uint refCount;
|
|
|
|
//
|
|
// This is a small region. Figure out which bucket it's from.
|
|
// Using the bucket size, we can find the start of the region
|
|
// and access the region header.
|
|
//
|
|
bucket = (byte)((pageDescriptor & SmallRegionBucketMask) >>
|
|
SmallRegionBucketShift);
|
|
regionSize = BucketSize(bucket);
|
|
region = start & ~(regionSize - 1);
|
|
|
|
refCount = AtomicDereferenceSmallRegion(region);
|
|
if (refCount == 0) {
|
|
//
|
|
// Return region to the bucket's free list.
|
|
//
|
|
AtomicPushFreeList(ref freeSmallRegions[bucket], region);
|
|
}
|
|
else {
|
|
lastReference = false;
|
|
}
|
|
|
|
} else {
|
|
//
|
|
// This is a large region. We need to walk all the pages
|
|
// it covers, and decrement the reference count on each.
|
|
// Pages no longer referenced are returned to the page manager.
|
|
//
|
|
UIntPtr lastPage;
|
|
UIntPtr lastByteAddress;
|
|
|
|
lastByteAddress = start + size - 1;
|
|
lastPage = PageFromAddr(lastByteAddress);
|
|
|
|
while (page <= lastPage) {
|
|
|
|
if (AtomicDereferencePage(page) == 0) {
|
|
UIntPtr startAddr = AddrFromPage(page);
|
|
UIntPtr regionPages = 1;
|
|
page++;
|
|
|
|
while (page <= lastPage &&
|
|
AtomicDereferencePage(page) == 0) {
|
|
regionPages++;
|
|
page++;
|
|
}
|
|
|
|
if (!dontFreePages) {
|
|
FreePages(startAddr, regionPages);
|
|
}
|
|
}
|
|
else {
|
|
lastReference = false;
|
|
}
|
|
|
|
page++;
|
|
}
|
|
}
|
|
return lastReference;
|
|
}
|
|
#endregion
|
|
|
|
#region Allocation Owner Accounting
|
|
// <summary>
|
|
// Whether to use spinlocks (true) or disabling of interrupts (false)
|
|
// to protect the ownership lists.
|
|
// </summary>
|
|
private static bool useAllocationOwnerLocks;
|
|
|
|
// <summary>
|
|
// The free allocation owner descriptor list.
|
|
// Free descriptors are kept as unstructured memory, with the first
|
|
// UIntPtr bytes of each serving as the pointer to the next one.
|
|
// </summary>
|
|
private UIntPtr freeOwnerList;
|
|
|
|
//
|
|
// NOTE The shared heap has a concept of AllocationOwnerIds,
|
|
// but in our current system there is a single one of these for
|
|
// all data block and endpoint allocs in a given heap.
|
|
//
|
|
private AllocationOwnerId allocOwnerId;
|
|
private AllocationOwnerId endpointOwnerId;
|
|
private AllocationOwnerId endpointPeerOwnerId;
|
|
|
|
public AllocationOwnerId DataOwnerId {
|
|
get {
|
|
return allocOwnerId;
|
|
}
|
|
}
|
|
|
|
public AllocationOwnerId EndpointOwnerId {
|
|
get {
|
|
return endpointOwnerId;
|
|
}
|
|
}
|
|
|
|
public AllocationOwnerId EndpointPeerOwnerId {
|
|
get {
|
|
return endpointPeerOwnerId;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Call back for iterating over all elements of an allocation list.
|
|
/// </summary>
|
|
unsafe public delegate void AllocationVisitor(Allocation* alloc);
|
|
|
|
/// <summary>
|
|
/// Call back for filtering elements of an allocation list.
|
|
/// </summary>
|
|
unsafe public delegate bool AllocationMatches(Allocation* alloc);
|
|
|
|
// <summary>
|
|
// Per-process object structure used to track allocations 'owned'
|
|
// by a process. List integrity is protected by the spinlock.
|
|
// </summary>
|
|
internal struct AllocationOwner {
|
|
internal unsafe Allocation *sentinel;
|
|
private SpinLock listLock;
|
|
private bool lockInterrupt;
|
|
internal long outstandingBlocks;
|
|
|
|
internal static unsafe void Lock(AllocationOwner *owner)
|
|
{
|
|
if (useAllocationOwnerLocks) {
|
|
owner->listLock.Acquire(Thread.CurrentThread);
|
|
} else {
|
|
owner->lockInterrupt = Processor.DisableInterrupts();
|
|
}
|
|
}
|
|
|
|
internal static unsafe void Unlock(AllocationOwner *owner)
|
|
{
|
|
if (useAllocationOwnerLocks) {
|
|
owner->listLock.Release(Thread.CurrentThread);
|
|
} else {
|
|
Processor.RestoreInterrupts(owner->lockInterrupt);
|
|
}
|
|
}
|
|
|
|
internal static unsafe void ReuseOrCreateOwner(
|
|
SharedHeap sharedHeap,
|
|
out AllocationOwnerId ownerId)
|
|
{
|
|
UIntPtr descriptorAddress;
|
|
uint descriptorSize = (uint)sizeof(AllocationOwner);
|
|
|
|
//
|
|
// Attempt to get a descriptor from our free list.
|
|
//
|
|
descriptorAddress = AtomicPopFreeList(ref sharedHeap.freeOwnerList);
|
|
|
|
if (descriptorAddress == UIntPtr.Zero) {
|
|
//
|
|
// Free list is empty.
|
|
// Allocate new page and break it up into allocation owner
|
|
// descriptors. Use first descriptor to satisfy current
|
|
// request, put remainder on the free list.
|
|
//
|
|
UIntPtr newSpace;
|
|
uint spaceRemaining;
|
|
|
|
newSpace = sharedHeap.AllocatePages(1);
|
|
descriptorAddress = newSpace;
|
|
newSpace += descriptorSize;
|
|
spaceRemaining = MemoryManager.PageSize - descriptorSize;
|
|
while (spaceRemaining > descriptorSize) {
|
|
AtomicPushFreeList(ref sharedHeap.freeOwnerList, newSpace);
|
|
newSpace += descriptorSize;
|
|
spaceRemaining -= descriptorSize;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Initialize the owner struct before handing it out.
|
|
//
|
|
Buffer.ZeroMemory((byte *)descriptorAddress, descriptorSize);
|
|
|
|
((AllocationOwner*)descriptorAddress)->sentinel =
|
|
Allocation.CreateSentinel(sharedHeap);
|
|
ownerId.owner = (AllocationOwner*)descriptorAddress;
|
|
}
|
|
|
|
///
|
|
/// Diagnostic methods
|
|
///
|
|
public long OutstandingBlocks {
|
|
get { return this.outstandingBlocks; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calls the visitor with each allocation on the list.
|
|
/// Important: the visitor is not allowed to keep the element!
|
|
/// </summary>
|
|
unsafe public static void Iterate(AllocationOwner* owner,
|
|
AllocationVisitor visitor)
|
|
{
|
|
Lock(owner);
|
|
|
|
Allocation* sentinel = owner->sentinel;
|
|
//
|
|
// Walk the list.
|
|
//
|
|
Allocation* current = sentinel->next;
|
|
while (current != sentinel) {
|
|
if (Allocation.GetType(current) !=
|
|
typeof(RoverType).GetSystemType().id) {
|
|
visitor(current);
|
|
}
|
|
current = current->next;
|
|
}
|
|
Unlock(owner);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calls the visitor with each matching allocation on the list.
|
|
/// The matcher is not allowed to modify the list.
|
|
/// The visitor is allowed to modify the list (e.g. to delete
|
|
/// the visited element from the list), and the visitor
|
|
/// is called with the owner lock unacquired. Furthermore,
|
|
/// the iteration releases the owner lock periodically (even
|
|
/// if visitor is never called) so that it doesn't monopolize
|
|
/// the lock. The matcher should only return true if no other
|
|
/// thread can deallocate the matched element while the lock
|
|
/// is unheld; otherwise, the visitor will be in a race.
|
|
///
|
|
/// To ensure that iteration is still meaningful in the face
|
|
/// of concurrent mutation, the iterator adds a "rover"
|
|
/// node to the beginning of the list, then steps the rover
|
|
/// one node at a time through the list until it arrives at the
|
|
/// end.
|
|
/// </summary>
|
|
unsafe public static void IterateMatchingForModify(
|
|
AllocationOwnerId ownerId,
|
|
AllocationMatches matches,
|
|
AllocationVisitor visitor)
|
|
{
|
|
const int RELEASE_PERIOD = 128; // periodically release lock
|
|
int releaseCount = 0;
|
|
Allocation roverAllocation;
|
|
Allocation* rover = &roverAllocation;
|
|
UIntPtr roverType = typeof(RoverType).GetSystemType().id;
|
|
Allocation.Initialize(rover, 0, 0, roverType);
|
|
AllocationOwner* owner = ownerId.owner;
|
|
Allocation.AssignOwnership(rover, ownerId);
|
|
Lock(owner);
|
|
|
|
Allocation* sentinel = owner->sentinel;
|
|
//
|
|
// Walk the list.
|
|
//
|
|
while (rover->next != sentinel) {
|
|
releaseCount = (releaseCount + 1) % RELEASE_PERIOD;
|
|
Allocation* current = rover->next;
|
|
|
|
if (Allocation.GetType(current) != roverType
|
|
&& matches(current)) {
|
|
Unlock(owner);
|
|
visitor(current);
|
|
Lock(owner);
|
|
}
|
|
else if (releaseCount == 0) {
|
|
Unlock(owner);
|
|
// TODO: pause here? (prevent lock capture effects)
|
|
Lock(owner);
|
|
}
|
|
Allocation.MoveForward(rover);
|
|
}
|
|
Unlock(owner);
|
|
Allocation.RemoveOwnership(rover, ownerId);
|
|
}
|
|
}
|
|
|
|
// Used by IterateMatchingForModify
|
|
private struct RoverType {}
|
|
|
|
// <summary>
|
|
// Opaque type used to hide AllocationOwner structure
|
|
// from everything outside of the SharedHeap.
|
|
// </summary>
|
|
public struct AllocationOwnerId {
|
|
unsafe internal AllocationOwner* owner;
|
|
|
|
public void Iterate(AllocationVisitor visitor) {
|
|
unsafe {
|
|
AllocationOwner.Iterate(owner, visitor);
|
|
}
|
|
}
|
|
|
|
public void IterateMatchingForModify(
|
|
AllocationMatches matches,
|
|
AllocationVisitor visitor) {
|
|
AllocationOwner.IterateMatchingForModify(
|
|
this, matches, visitor);
|
|
}
|
|
}
|
|
#endregion
|
|
|
|
// <summary>
|
|
// Initialize the shared heap subsystem.
|
|
// Note: We trust the system to initialize statics to zero.
|
|
// </summary>
|
|
#if PAGING
|
|
internal unsafe SharedHeap(ProtectionDomain parentDomain,
|
|
VirtualMemoryRange range)
|
|
#else
|
|
internal unsafe SharedHeap()
|
|
#endif
|
|
{
|
|
useAllocationOwnerLocks = false;
|
|
|
|
#if PAGING
|
|
this.memoryRange = range;
|
|
this.parentDomain = parentDomain;
|
|
#endif
|
|
|
|
//
|
|
// We don't have another memory allocator handy, so we grab
|
|
// ourselves a page and carve it up.
|
|
//
|
|
UIntPtr newSpace = AllocatePages(1);
|
|
freeSmallRegions = (UIntPtr *)newSpace;
|
|
|
|
//
|
|
// REVIEW: Allocate doesn't currently zero-fill, so
|
|
// we need to initialize the free lists.
|
|
//
|
|
for (uint Loop = 0; Loop < TotalBuckets; Loop++) {
|
|
freeSmallRegions[Loop] = UIntPtr.Zero;
|
|
}
|
|
|
|
// Set up our owner IDs
|
|
allocOwnerId = OwnerInitialize();
|
|
endpointOwnerId = OwnerInitialize();
|
|
endpointPeerOwnerId = OwnerInitialize();
|
|
}
|
|
|
|
internal static void Finalize()
|
|
{
|
|
// Doesn't actually do anything.
|
|
}
|
|
|
|
private UIntPtr AllocatePages(UIntPtr numPages)
|
|
{
|
|
return AllocatePages(numPages, 0);
|
|
}
|
|
|
|
#if PAGING
|
|
private unsafe UIntPtr AllocatePages(UIntPtr numPages, uint extra)
|
|
{
|
|
return memoryRange.Allocate(numPages, null, extra, PageType.Shared);
|
|
}
|
|
|
|
private unsafe void FreePages(UIntPtr addr, UIntPtr numPages)
|
|
{
|
|
memoryRange.Free(addr, numPages, null);
|
|
}
|
|
|
|
private unsafe uint* SharedHeapPageTable {
|
|
get {
|
|
return memoryRange.PageTable;
|
|
}
|
|
}
|
|
|
|
private unsafe UIntPtr SharedHeapPageCount {
|
|
get {
|
|
return memoryRange.PageCount;
|
|
}
|
|
}
|
|
|
|
private unsafe UIntPtr PageFromAddr(UIntPtr addr)
|
|
{
|
|
return memoryRange.PageFromAddr(addr);
|
|
}
|
|
|
|
private unsafe UIntPtr AddrFromPage(UIntPtr pageIdx)
|
|
{
|
|
return memoryRange.AddrFromPage(pageIdx);
|
|
}
|
|
|
|
#else // PAGING
|
|
//
|
|
// When paging is not enabled, the (one and only) shared heap
|
|
// always uses general-purpose kernel memory.
|
|
//
|
|
private unsafe UIntPtr AllocatePages(UIntPtr numPages, uint extra)
|
|
{
|
|
return MemoryManager.KernelAllocate(numPages, null, extra, PageType.Shared);
|
|
}
|
|
|
|
private void FreePages(UIntPtr addr, UIntPtr numPages)
|
|
{
|
|
MemoryManager.KernelFree(addr, numPages, null);
|
|
}
|
|
|
|
private unsafe uint* SharedHeapPageTable {
|
|
get {
|
|
return MemoryManager.KernelPageTable;
|
|
}
|
|
}
|
|
|
|
private UIntPtr SharedHeapPageCount {
|
|
get {
|
|
return MemoryManager.KernelPageCount;
|
|
}
|
|
}
|
|
|
|
private UIntPtr PageFromAddr(UIntPtr addr)
|
|
{
|
|
return MemoryManager.PageFromAddr(addr);
|
|
}
|
|
|
|
private UIntPtr AddrFromPage(UIntPtr pageIdx)
|
|
{
|
|
return MemoryManager.AddrFromPage(pageIdx);
|
|
}
|
|
|
|
#endif // PAGING
|
|
|
|
|
|
// <summary>
|
|
// Initialize internal per-owner data structure.
|
|
// Returns an opaque identifier to the owner (usually a process)
|
|
// for it to use whenever it calls us in the future.
|
|
// </summary>
|
|
public AllocationOwnerId OwnerInitialize()
|
|
{
|
|
AllocationOwnerId ownerId;
|
|
AllocationOwner.ReuseOrCreateOwner(this, out ownerId);
|
|
return ownerId;
|
|
}
|
|
|
|
//
|
|
// Moves a block from another shared heap into this one!
|
|
// The source heap must be mapped and accessible.
|
|
//
|
|
public unsafe Allocation* Move(Allocation* source,
|
|
SharedHeap sourceHeap,
|
|
AllocationOwnerId sourceOwner,
|
|
AllocationOwnerId targetOwner)
|
|
{
|
|
//
|
|
// Note that since both the source and destination shared heaps must
|
|
// be accessible, one of them must be the kernel domain's heap!
|
|
//
|
|
sourceHeap.CheckDomain();
|
|
this.CheckDomain();
|
|
|
|
#if PAGING && (!DISABLE_PAGE_FLIPPING)
|
|
if (Allocation.GetSize(source) > MaxSmallRegion) {
|
|
// Large region. Move it by page-flipping.
|
|
//
|
|
// NOTE we currently ASSUME that the block
|
|
// being moved is not being shared, since the sharing
|
|
// feature is being deprecated.
|
|
UIntPtr rangeStart = (UIntPtr)Allocation.GetDataUnchecked(source);
|
|
UIntPtr size = Allocation.GetSize(source);
|
|
UIntPtr type = Allocation.GetType(source);
|
|
UIntPtr pagesStart = MemoryManager.PageTrunc(rangeStart);
|
|
UIntPtr pagesLimit = MemoryManager.PagePad(rangeStart + size);
|
|
UIntPtr pageCount = MemoryManager.PagesFromBytes(pagesLimit - pagesStart);
|
|
|
|
// Release the memory from the source heap.
|
|
bool lastFree = sourceHeap.DereferenceRegion(rangeStart, size, true /*don't free pages*/);
|
|
DebugStub.Assert(lastFree);
|
|
|
|
// Reserve room in our own memory range for the pages
|
|
UIntPtr mapPoint = this.memoryRange.Reserve(pageCount, null, LargeRefToExtra(1),
|
|
PageType.Shared);
|
|
|
|
DebugStub.Assert(mapPoint != UIntPtr.Zero);
|
|
|
|
UIntPtr stepSource = pagesStart;
|
|
UIntPtr stepTarget = mapPoint;
|
|
|
|
// Map each page into our reserved area of memory
|
|
while (stepSource < pagesLimit) {
|
|
PhysicalAddress phys = VMManager.GetPageMapping(stepSource);
|
|
DebugStub.Assert(phys != PhysicalAddress.Null);
|
|
VMManager.MapPage(phys, stepTarget, memoryRange.ParentDomain);
|
|
|
|
stepSource += MemoryManager.PageSize;
|
|
stepTarget += MemoryManager.PageSize;
|
|
}
|
|
|
|
// Unmap the pages from the source range; they're still marked as being used.
|
|
VMManager.UnmapRange(pagesStart, pageCount);
|
|
|
|
// Mark the pages as free. Now the source heap and range have completely
|
|
// forgotten about the memory.
|
|
sourceHeap.memoryRange.Unreserve(pagesStart, pageCount, null);
|
|
|
|
return AllocationFromRawMemory(mapPoint, size, type, targetOwner);
|
|
|
|
} else
|
|
#endif
|
|
{
|
|
// Small region. Move it by copying.
|
|
Allocation* retval = ShallowCopy(source, targetOwner);
|
|
sourceHeap.Free(source, sourceOwner);
|
|
return retval;
|
|
}
|
|
}
|
|
|
|
// Creates a new allocation record and copies the contents of
|
|
// the provided allocation.
|
|
internal unsafe Allocation *ShallowCopy(Allocation* source,
|
|
AllocationOwnerId targetOwner)
|
|
{
|
|
CheckDomain();
|
|
|
|
Allocation* copy = Allocate(Allocation.GetSize(source),
|
|
Allocation.GetType(source),
|
|
0,
|
|
targetOwner);
|
|
|
|
Buffer.MoveMemory((byte*)Allocation.GetDataUnchecked(copy),
|
|
(byte*)Allocation.GetDataUnchecked(source),
|
|
Allocation.GetSize(source));
|
|
|
|
return copy;
|
|
}
|
|
|
|
// <summary>
|
|
// Allocate an arbitrarily-sized region of memory.
|
|
// TODO: Handle memory exhaustion gracefully.
|
|
// </summary>
|
|
public unsafe Allocation *Allocate( // Returns new region.
|
|
UIntPtr bytes, // Number of bytes to allocate.
|
|
UIntPtr type, // Type information.
|
|
uint alignment, // Allocation alignment requirement.
|
|
AllocationOwnerId ownerId) // Id of owner to assign region to.
|
|
{
|
|
CheckDomain();
|
|
UIntPtr region;
|
|
|
|
// REVIEW: Pull out-of-bounds parameters into bounds instead?
|
|
DebugStub.Assert(bytes >= 0);
|
|
|
|
// TODO: deal with alignment.
|
|
|
|
//
|
|
// Allocate the memory region.
|
|
//
|
|
// Small regions impose an overhead of sizeof(SmallRegion),
|
|
// so we need to take that into account when determining
|
|
// whether or not we can fit the allocation into one.
|
|
//
|
|
// Zero-sized allocations are allowed, but we do not allocate
|
|
// a memory region for them.
|
|
//
|
|
if (bytes == 0) {
|
|
region = UIntPtr.Zero;
|
|
} else {
|
|
if (bytes + sizeof(SmallRegion) > MaxSmallRegion) {
|
|
region = AllocateLargeRegion(bytes);
|
|
} else {
|
|
region = AllocateSmallRegion(bytes);
|
|
}
|
|
// Allocation failed, request denied. Should
|
|
// have asked sooner.
|
|
if (region == UIntPtr.Zero) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// TODO: deal with alignment.
|
|
|
|
return AllocationFromRawMemory(region, bytes, type, ownerId);
|
|
}
|
|
|
|
//
|
|
// Wraps a range of raw memory in a new allocation record.
|
|
//
|
|
private unsafe Allocation* AllocationFromRawMemory(
|
|
UIntPtr region,
|
|
UIntPtr bytes,
|
|
UIntPtr type,
|
|
AllocationOwnerId ownerId)
|
|
{
|
|
CheckDomain();
|
|
|
|
// Get an allocation descriptor to keep track of the region.
|
|
// TODO: Handle failure here.
|
|
Allocation* allocation = Allocation.Create(this, region, bytes, type);
|
|
|
|
//
|
|
// Put descriptor on per-process list.
|
|
// all descriptors live on the kernel proc for now.
|
|
//
|
|
Allocation.AssignOwnership(allocation, ownerId);
|
|
|
|
// Figure out who our caller is.
|
|
UIntPtr pc1, pc2, pc3;
|
|
Processor.GetStackEips(out pc1, out pc2, out pc3);
|
|
Tracing.Log(Tracing.Debug, "SharedHeap.Allocate stack trace {0:x} {1:x} {2:x}",
|
|
pc1, pc2, pc3);
|
|
Tracing.Log(Tracing.Debug, "result = {0:x}", (UIntPtr)allocation);
|
|
|
|
return allocation;
|
|
}
|
|
|
|
// <summary>
|
|
// Allocate a large region of memory out of system pages.
|
|
// </summary>
|
|
private unsafe UIntPtr AllocateLargeRegion( // Returns new region.
|
|
UIntPtr bytes) // Number of bytes to allocate.
|
|
{
|
|
UIntPtr region;
|
|
|
|
//
|
|
// For large allocations, we just get pages from the system.
|
|
// REVIEW: We round up to a pagesize to keep Allocate happy.
|
|
// Note: Current implementation of Allocate can block.
|
|
//
|
|
bytes = MemoryManager.PagePad(bytes);
|
|
region = AllocatePages(MemoryManager.PagesFromBytes(bytes), LargeRefToExtra(1));
|
|
|
|
//
|
|
// Zero-fill the region before handing it off.
|
|
//
|
|
if (region != UIntPtr.Zero) {
|
|
Buffer.ZeroMemory((byte *)region, bytes);
|
|
}
|
|
|
|
return region;
|
|
}
|
|
|
|
// <summary>
|
|
// Allocate a small region of memory out of a heap page.
|
|
// </summary>
|
|
private unsafe UIntPtr AllocateSmallRegion(
|
|
UIntPtr bytes) // Number of bytes to allocate.
|
|
{
|
|
byte bucket;
|
|
UIntPtr region;
|
|
|
|
//
|
|
// We allocate the region from the bucket which contains the
|
|
// smallest sized regions which will satisfy the request.
|
|
// And we initialize the region's reference count to 1.
|
|
//
|
|
bucket = BucketForSize(bytes + sizeof(SmallRegion));
|
|
region = AtomicPopFreeList(ref freeSmallRegions[bucket]);
|
|
if (region == UIntPtr.Zero) {
|
|
//
|
|
// Free list is empty.
|
|
// Allocate new page and break it up into small regions
|
|
// of this bucket's size. Use first descriptor to satisfy
|
|
// current request, put remainder on the free list.
|
|
//
|
|
UIntPtr newSpace;
|
|
UIntPtr spaceRemaining;
|
|
UIntPtr regionSize;
|
|
|
|
newSpace = AllocatePages(1, BucketToExtra(bucket));
|
|
|
|
if (newSpace == UIntPtr.Zero) {
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
regionSize = BucketSize(bucket);
|
|
region = newSpace;
|
|
newSpace += regionSize;
|
|
spaceRemaining = MemoryManager.PageSize - regionSize;
|
|
while (spaceRemaining > regionSize) {
|
|
AtomicPushFreeList(ref freeSmallRegions[bucket], newSpace);
|
|
newSpace += regionSize;
|
|
spaceRemaining -= regionSize;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Set reference count on the region.
|
|
//
|
|
unsafe {
|
|
((SmallRegion *)region)->referenceCount = 1;
|
|
region += sizeof(SmallRegion);
|
|
}
|
|
|
|
//
|
|
// Zero-fill the region before handing it off.
|
|
//
|
|
Buffer.ZeroMemory((byte *)region, bytes);
|
|
|
|
return region;
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Free a previously-allocated region of memory.
|
|
/// </summary>
|
|
/// <returns>true if Free releases last reference to underlying memory</returns>
|
|
public unsafe bool Free(
|
|
Allocation *allocation, // Allocated region to be freed.
|
|
AllocationOwnerId ownerId) // Current owner of allocation.
|
|
{
|
|
CheckDomain();
|
|
DebugStub.Assert(Validate(allocation));
|
|
//
|
|
// Remove from owning process's allocation list.
|
|
//
|
|
Allocation.RemoveOwnership(allocation, ownerId);
|
|
|
|
//
|
|
// Release our allocation's claim on the underlying region.
|
|
//
|
|
bool freed = DereferenceRegion(Allocation.GetDataUnchecked(allocation),
|
|
Allocation.GetSize(allocation));
|
|
|
|
//
|
|
// Free our allocation descriptor.
|
|
//
|
|
AtomicPushFreeList(ref freeAllocationList, (UIntPtr)allocation);
|
|
|
|
//
|
|
// Tell caller whether last reference was freed
|
|
//
|
|
return freed;
|
|
}
|
|
|
|
// <summary>
|
|
// Free all of a given owner's allocated regions.
|
|
// </summary>
|
|
public unsafe void FreeAll(
|
|
AllocationOwnerId ownerId)
|
|
{
|
|
CheckDomain();
|
|
|
|
Allocation *first;
|
|
Allocation *current;
|
|
AllocationOwner *owner = ownerId.owner;
|
|
|
|
//
|
|
// REVIEW: Do we need this lock? This should be the last code
|
|
// for this process to ever call the SharedHeap.
|
|
//
|
|
AllocationOwner.Lock(owner);
|
|
|
|
//
|
|
// The FreeAll for a process is called by a kernel service
|
|
// thread and not by the process itself.
|
|
// Thus we use unchecked access to the data, since the process
|
|
// id's won't match
|
|
//
|
|
first = owner->sentinel;
|
|
current = Allocation.Next(first);
|
|
while(current != first) {
|
|
Allocation *victim = current;
|
|
current = Allocation.Next(current);
|
|
DereferenceRegion(Allocation.GetDataUnchecked(victim),
|
|
Allocation.GetSize(victim));
|
|
AtomicPushFreeList(ref freeAllocationList, (UIntPtr)victim);
|
|
}
|
|
|
|
DereferenceRegion(Allocation.GetDataUnchecked(first),
|
|
Allocation.GetSize(first));
|
|
AtomicPushFreeList(ref freeAllocationList, (UIntPtr)first);
|
|
owner->sentinel = null;
|
|
|
|
AllocationOwner.Unlock(owner);
|
|
|
|
//
|
|
// Free our owner descriptor.
|
|
//
|
|
AtomicPushFreeList(ref freeOwnerList, (UIntPtr)ownerId.owner);
|
|
}
|
|
|
|
// <summary>
|
|
// Create a second access handle for a previously allocated region.
|
|
//
|
|
// Sharing of a zero-sized region is tolerated, although weird.
|
|
// </summary>
|
|
public unsafe Allocation *Share( // Returns new handle.
|
|
Allocation *existing, // Allocated region to share.
|
|
AllocationOwnerId ownerId, // Owner of above region.
|
|
UIntPtr startOffset, // Offset at which shared copy starts.
|
|
UIntPtr endOffset) // Offset at which shared copy ends.
|
|
{
|
|
CheckDomain();
|
|
|
|
DebugStub.Assert(Validate(existing));
|
|
Allocation *copy;
|
|
UIntPtr size;
|
|
|
|
size = Allocation.GetSize(existing);
|
|
|
|
//
|
|
// We don't handle nonsensical parameters.
|
|
// REVIEW: Pull out-of-bounds parameters into bounds instead?
|
|
//
|
|
DebugStub.Assert(startOffset <= size && endOffset >= startOffset &&
|
|
endOffset <= size);
|
|
|
|
//
|
|
// Create a new descriptor for the allocation with (potentially
|
|
// a subset of) the same data and size as the original.
|
|
//
|
|
copy = Allocation.Create(
|
|
this,
|
|
Allocation.GetData(existing) + startOffset,
|
|
endOffset - startOffset,
|
|
Allocation.GetType(existing),
|
|
Allocation.GetOwnerProcessId(existing));
|
|
|
|
//
|
|
// Increment the reference count on affected region/page(s).
|
|
//
|
|
ReferenceRegion(Allocation.GetData(copy),
|
|
Allocation.GetSize(copy));
|
|
|
|
//
|
|
// Put new descriptor on owning process's list.
|
|
//
|
|
Allocation.AssignOwnership(copy, ownerId);
|
|
|
|
return copy;
|
|
}
|
|
|
|
// <summary>
|
|
// Truncate an allocation to a length.
|
|
// </summary>
|
|
public unsafe void Truncate(Allocation *allocation,
|
|
UIntPtr newSize)
|
|
{
|
|
DebugStub.Assert(Validate(allocation));
|
|
UIntPtr size = Allocation.GetSize(allocation);
|
|
DebugStub.Assert(newSize >= 0 && newSize <= size);
|
|
Allocation.TruncateTo(allocation, newSize);
|
|
}
|
|
|
|
// <summary>
|
|
// Split a previously allocated region into two smaller regions.
|
|
// The original region is shrunk by the size of the carved-off
|
|
// second region.
|
|
//
|
|
// Splitting off a zero-sized region is tolerated, although weird.
|
|
// </summary>
|
|
public unsafe Allocation *Split( // Returns latter region.
|
|
Allocation *existing, // Region to split.
|
|
AllocationOwnerId ownerId, // Owner of above region.
|
|
UIntPtr offset) // Offset at which latter region starts.
|
|
{
|
|
CheckDomain();
|
|
|
|
DebugStub.Assert(Validate(existing));
|
|
Allocation *copy;
|
|
UIntPtr size;
|
|
UIntPtr splitPoint;
|
|
|
|
size = Allocation.GetSize(existing);
|
|
|
|
//
|
|
// We don't handle nonsensical parameters.
|
|
// REVIEW: Pull out-of-bounds parameters into bounds instead?
|
|
//
|
|
DebugStub.Assert(offset >= 0 && offset <= size);
|
|
|
|
//
|
|
// Create a new descriptor for the allocation using a suffix
|
|
// of the original data.
|
|
//
|
|
splitPoint = Allocation.GetData(existing) + offset;
|
|
copy = Allocation.Create(
|
|
this,
|
|
splitPoint,
|
|
size - offset,
|
|
Allocation.GetType(existing));
|
|
|
|
Allocation.TruncateTo(existing, offset);
|
|
|
|
//
|
|
// Increment the reference count on any small region or
|
|
// page which ends up getting shared as a result of this split.
|
|
// (Note that any zero-sized allocations created as a result of
|
|
// this split don't share any memory with other allocations)
|
|
//
|
|
// In other words, unless one of the resulting allocations is
|
|
// zero-sized, or the split point occurs at a page boundary,
|
|
// increment the ref count on the page or small region the
|
|
// split occurs in.
|
|
//
|
|
if ((offset != 0) && (offset != size) &&
|
|
(splitPoint != (splitPoint & ~(MemoryManager.PageSize - 1))))
|
|
{
|
|
ReferenceRegion(splitPoint, 1);
|
|
}
|
|
|
|
//
|
|
// Put new descriptor on owning process's list.
|
|
//
|
|
Allocation.AssignOwnership(copy, ownerId);
|
|
|
|
return copy;
|
|
}
|
|
|
|
|
|
// <summary>
|
|
// Transfer ownership of an allocated region from one process
|
|
// to another.
|
|
// </summary>
|
|
public static unsafe void TransferOwnership(
|
|
Allocation *allocation,
|
|
AllocationOwnerId oldOwnerId,
|
|
AllocationOwnerId newOwnerId)
|
|
{
|
|
Allocation.RemoveOwnership(allocation, oldOwnerId);
|
|
Allocation.AssignOwnership(allocation, newOwnerId);
|
|
}
|
|
|
|
// Returns true if a provided allocation record appears to be
|
|
// a valid record in our heap.
|
|
public unsafe bool Validate(Allocation *allocation)
|
|
{
|
|
#if PAGING
|
|
CheckDomain();
|
|
|
|
UIntPtr pData = (UIntPtr)allocation;
|
|
|
|
if (pData != UIntPtr.Zero) {
|
|
if ((pData < memoryRange.DataStartAddr) ||
|
|
(pData >= memoryRange.DataLimitAddr)) {
|
|
// This can't be a pointer into our heap;
|
|
// it's outside the range of pages we use
|
|
return false;
|
|
}
|
|
|
|
UIntPtr dataPage = MemoryManager.PageAlign(pData);
|
|
if (!VMManager.IsPageMapped(dataPage)) {
|
|
// It would not be a good idea to try to
|
|
// use this pointer
|
|
return false;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
return true;
|
|
}
|
|
|
|
// <summary>
|
|
// The free allocation descriptor list.
|
|
// Free descriptors are kept as unstructured memory, with the first
|
|
// UIntPtr bytes of each serving as the pointer to the next one.
|
|
// </summary>
|
|
internal UIntPtr freeAllocationList;
|
|
|
|
// <summary>
|
|
// Structure for tracking allocated regions.
|
|
// </summary>
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
public struct Allocation {
|
|
private UIntPtr data; // Address of the allocated memory region.
|
|
private UIntPtr size; // Size of above in bytes.
|
|
private unsafe UIntPtr type; // Type information.
|
|
internal unsafe Allocation *next; // Next on owner's list.
|
|
private unsafe Allocation *prev; // Previous on owner's list.
|
|
private int owner;
|
|
|
|
[Conditional("DEBUG")]
|
|
[NoHeapAllocation]
|
|
private static unsafe void AssertCorrectOwner(Allocation* allocation)
|
|
{
|
|
/*
|
|
if (Thread.CurrentProcess.ProcessId != allocation->owner) {
|
|
DebugStub.Break();
|
|
}
|
|
*/
|
|
}
|
|
|
|
[NoHeapAllocation]
|
|
public static unsafe UIntPtr GetData(Allocation *allocation)
|
|
{
|
|
AssertCorrectOwner(allocation);
|
|
return allocation->data;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Only to be used from
|
|
/// - TransferToProcess
|
|
/// - Freeing the Sentinel on FreeAll
|
|
/// - SharedHeap walker
|
|
///
|
|
/// All other uses should be checked.
|
|
/// </summary>
|
|
[NoHeapAllocation]
|
|
public static unsafe UIntPtr GetDataUnchecked(Allocation *allocation)
|
|
{
|
|
return allocation->data;
|
|
}
|
|
|
|
[NoHeapAllocation]
|
|
public static unsafe UIntPtr GetSize(Allocation *allocation)
|
|
{
|
|
return allocation->size;
|
|
}
|
|
|
|
[NoHeapAllocation]
|
|
public static unsafe UIntPtr GetType(Allocation *allocation)
|
|
{
|
|
return allocation->type;
|
|
}
|
|
|
|
[NoHeapAllocation]
|
|
public static unsafe int GetOwnerProcessId(Allocation *allocation)
|
|
{
|
|
return allocation->owner;
|
|
}
|
|
|
|
[NoHeapAllocation]
|
|
internal static unsafe Allocation *Next(Allocation *allocation)
|
|
{
|
|
return allocation->next;
|
|
}
|
|
|
|
[NoHeapAllocation]
|
|
internal static unsafe void SetType(Allocation *allocation, UIntPtr type)
|
|
{
|
|
allocation->type = type;
|
|
}
|
|
|
|
internal static unsafe void SetOwnerProcessId(Allocation *allocation,
|
|
int processId)
|
|
{
|
|
unchecked{
|
|
Tracing.Log(Tracing.Debug, "SetOwnerProcessId old:{0} new:{1} who:{2}",
|
|
(UIntPtr)allocation->owner,
|
|
(UIntPtr)processId,
|
|
(UIntPtr)Thread.CurrentProcess.ProcessId);
|
|
}
|
|
// Figure out who our caller is.
|
|
/*
|
|
UIntPtr[] stack = new UIntPtr[15];
|
|
Processor.GetStackEips(stack);
|
|
for(int i=0; i<stack.Length; i++) {
|
|
UIntPtr eip = stack[i];
|
|
if (eip == UIntPtr.Zero) break;
|
|
Tracing.Log(Tracing.Debug, "SetOwnerProcessId stack trace {0:x}", eip);
|
|
}
|
|
*/
|
|
allocation->owner = processId;
|
|
}
|
|
|
|
// <summary>
|
|
// Move the allocation one element forward in the list.
|
|
// requires owner lock held.
|
|
// </summary>
|
|
internal static unsafe void MoveForward(Allocation *allocation)
|
|
{
|
|
Allocation *next = allocation->next;
|
|
Allocation *prev = allocation->prev;
|
|
if (next == allocation) return; // only one element; can't move
|
|
// remove allocation from list
|
|
prev->next = next;
|
|
next->prev = prev;
|
|
// insert allocation into next location in list
|
|
Allocation *nextnext = next->next;
|
|
next->next = allocation;
|
|
allocation->prev = next;
|
|
allocation->next = nextnext;
|
|
nextnext->prev = allocation;
|
|
}
|
|
|
|
// <summary>
|
|
// Allocate an allocation tracking descriptor.
|
|
// </summary>
|
|
internal static unsafe Allocation *Create(SharedHeap sharedHeap,
|
|
UIntPtr address, UIntPtr size, UIntPtr type)
|
|
{
|
|
Allocation *allocation = ReuseOrCreateAllocation(sharedHeap);
|
|
Initialize(allocation, address, size, type);
|
|
DebugStub.Assert(sharedHeap.Validate(allocation));
|
|
return allocation;
|
|
}
|
|
|
|
internal static unsafe Allocation *Create(SharedHeap sharedHeap,
|
|
UIntPtr address, UIntPtr size, UIntPtr type, int ownerprocessId)
|
|
{
|
|
Allocation* result = Create(sharedHeap, address, size, type);
|
|
result->owner = ownerprocessId;
|
|
return result;
|
|
}
|
|
|
|
// <summary>
|
|
// Allocate an allocation tracking descriptor.
|
|
// </summary>
|
|
internal static unsafe Allocation *CreateSentinel(SharedHeap sharedHeap)
|
|
{
|
|
Allocation *allocation = ReuseOrCreateAllocation(sharedHeap);
|
|
allocation->data = 0;
|
|
allocation->size = 0;
|
|
allocation->type = 0;
|
|
allocation->next = allocation;
|
|
allocation->prev = allocation;
|
|
allocation->owner = 0;
|
|
return allocation;
|
|
}
|
|
|
|
// <summary>
|
|
// Allocate an allocation tracking descriptor.
|
|
// </summary>
|
|
internal static unsafe Allocation *Initialize(
|
|
Allocation* allocation,
|
|
UIntPtr address, UIntPtr size, UIntPtr type)
|
|
{
|
|
allocation->data = address;
|
|
allocation->size = size;
|
|
allocation->type = type;
|
|
allocation->next = allocation;
|
|
allocation->prev = allocation;
|
|
|
|
allocation->owner = Thread.CurrentProcess.ProcessId;
|
|
|
|
return allocation;
|
|
}
|
|
|
|
// <summary>
|
|
// Truncate an existing allocation at a given offset.
|
|
// Note that this merely affects the reported size, the entire
|
|
// region remains allocated, etc.
|
|
// </summary>
|
|
internal static unsafe void TruncateTo(Allocation *allocation,
|
|
UIntPtr offset)
|
|
{
|
|
allocation->size = offset;
|
|
}
|
|
|
|
// <summary>
|
|
// Assign ownership of an allocated region to a process.
|
|
// Note: this operation can block.
|
|
// </summary>
|
|
internal static unsafe void AssignOwnership(
|
|
Allocation *allocation,
|
|
AllocationOwnerId ownerId)
|
|
{
|
|
AllocationOwner *owner = ownerId.owner;
|
|
|
|
#if PAGING
|
|
// Sanity: we shouldn't be putting user blocks onto
|
|
// the kernel's shared heap lists, and vice versa.
|
|
if ((UIntPtr)owner >= BootInfo.KERNEL_BOUNDARY) {
|
|
DebugStub.Assert((UIntPtr)allocation >= BootInfo.KERNEL_BOUNDARY);
|
|
} else {
|
|
DebugStub.Assert((UIntPtr)allocation < BootInfo.KERNEL_BOUNDARY);
|
|
}
|
|
#endif
|
|
AllocationOwner.Lock(owner);
|
|
|
|
//
|
|
// Put allocation at beginning of owner's list.
|
|
//
|
|
allocation->next = owner->sentinel->next;
|
|
allocation->prev = owner->sentinel;
|
|
allocation->next->prev = allocation;
|
|
owner->sentinel->next = allocation;
|
|
owner->outstandingBlocks++;
|
|
AllocationOwner.Unlock(owner);
|
|
}
|
|
|
|
// <summary>
|
|
// Remove ownership of an allocated region from a process.
|
|
// </summary>
|
|
internal static unsafe void RemoveOwnership(
|
|
Allocation *allocation,
|
|
AllocationOwnerId ownerId)
|
|
{
|
|
AllocationOwner *owner = ownerId.owner;
|
|
|
|
AllocationOwner.Lock(owner);
|
|
|
|
//
|
|
// Remove allocation from owner's list.
|
|
//
|
|
allocation->next->prev = allocation->prev;
|
|
allocation->prev->next = allocation->next;
|
|
|
|
owner->outstandingBlocks--;
|
|
AllocationOwner.Unlock(owner);
|
|
|
|
//
|
|
// Sanitize this element.
|
|
//
|
|
allocation->next = allocation->prev = null;
|
|
}
|
|
|
|
private static unsafe Allocation *ReuseOrCreateAllocation(SharedHeap sharedHeap)
|
|
{
|
|
sharedHeap.CheckDomain();
|
|
UIntPtr descriptorAddress;
|
|
|
|
//
|
|
// Attempt to get a descriptor from our free list.
|
|
//
|
|
descriptorAddress = AtomicPopFreeList(ref sharedHeap.freeAllocationList);
|
|
|
|
if (descriptorAddress == UIntPtr.Zero) {
|
|
//
|
|
// Free list is empty.
|
|
// Allocate new page and break it up into allocation
|
|
// descriptors. Use first descriptor to satisfy current
|
|
// request, put remainder on the free list.
|
|
//
|
|
UIntPtr newSpace;
|
|
uint spaceRemaining;
|
|
uint descriptorSize;
|
|
|
|
newSpace = sharedHeap.AllocatePages(1);
|
|
descriptorSize = (uint)sizeof(Allocation);
|
|
|
|
descriptorAddress = newSpace;
|
|
newSpace += descriptorSize;
|
|
spaceRemaining = MemoryManager.PageSize - descriptorSize;
|
|
while (spaceRemaining > descriptorSize) {
|
|
AtomicPushFreeList(ref sharedHeap.freeAllocationList, newSpace);
|
|
newSpace += descriptorSize;
|
|
spaceRemaining -= descriptorSize;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Turn this chunk of memory into an Allocation.
|
|
//
|
|
return (Allocation *)descriptorAddress;
|
|
}
|
|
}
|
|
|
|
|
|
// <summary>
|
|
// Pop an unstructured memory region off a singly-linked list.
|
|
// The unstructured memory is expected to hold the address of the
|
|
// next element on the list in its first UIntPtr.
|
|
//
|
|
// Note: requires that the passed-in list head be in a fixed
|
|
// location in memory.
|
|
// </summary>
|
|
internal static unsafe UIntPtr AtomicPopFreeList( // Returns address.
|
|
ref UIntPtr head) // List head.
|
|
{
|
|
UIntPtr firstElementLocation;
|
|
UIntPtr secondElementLocation;
|
|
|
|
do {
|
|
//
|
|
// Cache the contents of head pointer location (i.e. the
|
|
// address of the first element on the list).
|
|
//
|
|
firstElementLocation = head;
|
|
|
|
if (firstElementLocation == UIntPtr.Zero) {
|
|
//
|
|
// No first element. List is empty.
|
|
//
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
//
|
|
// The first element contains the address of the second.
|
|
//
|
|
secondElementLocation = *(UIntPtr *)firstElementLocation;
|
|
|
|
//
|
|
// Called this way, Interlocked.CompareExchange will only
|
|
// replace the contents of the head pointer location with
|
|
// the address of the second element if the contents of the
|
|
// the head pointer location (i.e. the first element's address)
|
|
// hasn't changed since we cached it. If the contents of
|
|
// the head pointer have changed in the meantime, it means
|
|
// some other thread has popped that element off the stack.
|
|
// So we loop back and try it all over again.
|
|
//
|
|
|
|
} while(firstElementLocation != Interlocked.CompareExchange(
|
|
ref head, secondElementLocation,
|
|
firstElementLocation));
|
|
|
|
//
|
|
// We successfully popped an element off the list.
|
|
// Sanitize it before returning it.
|
|
//
|
|
*(UIntPtr *)firstElementLocation = UIntPtr.Zero;
|
|
return firstElementLocation;
|
|
}
|
|
|
|
|
|
// <summary>
|
|
// Push an unstructured memory region onto a singly-linked list.
|
|
// The unstructured memory must be large enough to hold the address
|
|
// of the next element on the list in its first UIntPtr.
|
|
//
|
|
// Note: requires that the passed-in list head be in a fixed
|
|
// location in memory.
|
|
// </summary>
|
|
internal static unsafe void AtomicPushFreeList( // Returns address.
|
|
ref UIntPtr head, // List head.
|
|
UIntPtr newElementLocation) // Address of element to add to list.
|
|
{
|
|
UIntPtr firstElementLocation;
|
|
|
|
do {
|
|
//
|
|
// Cache the contents of head pointer location (i.e. the
|
|
// address of the first element on the list).
|
|
//
|
|
firstElementLocation = head;
|
|
|
|
//
|
|
// Put the address of the current first element into
|
|
// our new first element.
|
|
//
|
|
*(UIntPtr *)newElementLocation = firstElementLocation;
|
|
|
|
//
|
|
// Called this way, Interlocked.CompareExchange will only swap
|
|
// the contents of the head pointer with that of the new first
|
|
// element's pointer if the contents of the head pointer
|
|
// haven't changed since we cached it. If the contents of
|
|
// the head pointer have changed in the meantime, it means
|
|
// some other thread has popped that element off the stack.
|
|
// So we loop back and try it all over again.
|
|
//
|
|
|
|
} while(firstElementLocation != Interlocked.CompareExchange(
|
|
ref head, newElementLocation,
|
|
firstElementLocation));
|
|
}
|
|
}
|
|
}
|