1795 lines
68 KiB
C#
1795 lines
68 KiB
C#
////////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// Microsoft Research Singularity
|
|
//
|
|
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
//
|
|
// File: Legacy.cs - Primitive memory manager
|
|
//
|
|
// Note:
|
|
//
|
|
|
|
//#define TEST
|
|
//#define VERBOSE
|
|
//#define MP_VERBOSE
|
|
//#define COMPLEX_TEST
|
|
|
|
#if MARKSWEEPCOLLECTOR
|
|
#define ALLOW_BOOT_ARGLIST // Cannot be used during boot for GCs w/ write barrier
|
|
#endif
|
|
#define NO__TRACE_PAGES
|
|
|
|
using System;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Runtime.InteropServices;
|
|
using System.Threading;
|
|
using System.GCs;
|
|
|
|
using Microsoft.Singularity;
|
|
using Microsoft.Singularity.Hal;
|
|
using Microsoft.Bartok.Runtime;
|
|
using Microsoft.Singularity.Eventing;
|
|
|
|
namespace Microsoft.Singularity.Memory
|
|
{
|
|
[NoCCtor]
|
|
[CLSCompliant(false)]
|
|
public class FlatPages {
|
|
|
|
// WARNING: don't initialize any static fields in this class
|
|
// without manually running the class constructor at startup!
|
|
//private const uint PageMask = MemoryManager.PageSize - 1;
|
|
private const uint SmallSize = MemoryManager.PageSize;
|
|
|
|
private static UIntPtr lowerLimit;
|
|
private static UIntPtr upperLimit;
|
|
private static UIntPtr pageBase; // Lowest page.
|
|
private static UIntPtr pageCount; // Number of pages.
|
|
private static SpinLock pageLock;
|
|
private static UIntPtr baseAddress;
|
|
private static ulong size;
|
|
|
|
private static unsafe uint *pageTable;
|
|
|
|
// We keep two lists of free pages:
|
|
// The freeList has pages that can be used at any moment.
|
|
// The saveList has pages that were reserved for growing a region.
|
|
|
|
private static FreeNode freeListTail;
|
|
private static FreeNode saveListTail;
|
|
private static unsafe FreeNode *freeList;
|
|
private static unsafe FreeNode *saveList;
|
|
|
|
//
|
|
// This represents the real time count of available memory
|
|
//
|
|
private static UIntPtr availableMemory;
|
|
|
|
[NoHeapAllocation]
|
|
[Inline]
|
|
private unsafe static bool RangeWithinLimits(ulong addr, ulong size, UIntPtr upperLimit, UIntPtr lowerLimit,
|
|
out UIntPtr baseAddress, out ulong newSize)
|
|
{
|
|
|
|
baseAddress = addr;
|
|
newSize = size;
|
|
|
|
if(addr + size < lowerLimit) {
|
|
return false;
|
|
}
|
|
if(addr > upperLimit) {
|
|
return false;
|
|
}
|
|
|
|
if((addr > lowerLimit) && ((addr + size) < upperLimit)) {
|
|
return true;
|
|
}
|
|
//trim the limits to fit into the valid address range
|
|
if(addr < lowerLimit) {
|
|
baseAddress = lowerLimit;
|
|
newSize = newSize - ((ulong)lowerLimit - addr);
|
|
}
|
|
if((addr + size) > upperLimit) {
|
|
newSize = newSize - (((ulong)baseAddress + size) - (ulong)upperLimit);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
internal static unsafe void Initialize()
|
|
{
|
|
Tracing.Log(Tracing.Debug, "FlatPages.Initialize() called");
|
|
|
|
// Initialize spinlock info
|
|
pageLock = new SpinLock(SpinLock.Types.FlatPages);
|
|
|
|
Platform p = Platform.ThePlatform;
|
|
HistogramSize = 64;
|
|
HistogramGranularityShift = 20;
|
|
FreeMemoryCounters = null;
|
|
CurrentSystemLogger = null;
|
|
|
|
// Retrieve the highest RAM address
|
|
MemoryManager.PhysicalMemoryLimits(out lowerLimit, out upperLimit);
|
|
|
|
DebugStub.WriteLine("MemoryManager Physical limits {0:x8}...{1:x8}\n",
|
|
__arglist(lowerLimit, upperLimit));
|
|
|
|
// Compute page count
|
|
|
|
pageCount = Pad((upperLimit >> MemoryManager.PageBits) + 1,
|
|
MemoryManager.PageSize / sizeof(uint));
|
|
|
|
// Find a spot for the page table.
|
|
pageTable = null;
|
|
|
|
// Figure out the lowest possible location for positioning the pageMap
|
|
UIntPtr limit = Pad(lowerLimit, 0x00200000);
|
|
|
|
// Make sure "limit" doesn't straddle some of the other non-SMAP regions
|
|
if (p.BootAllocatedMemorySize != 0) {
|
|
if ((p.BootAllocatedMemory < limit) &&
|
|
(limit < p.BootAllocatedMemory + p.BootAllocatedMemorySize)) {
|
|
limit = MemoryManager.PagePad(p.BootAllocatedMemory + p.BootAllocatedMemorySize);
|
|
}
|
|
}
|
|
if (p.KernelDllSize != 0) {
|
|
if ((p.KernelDllBase < limit) &&
|
|
(limit < p.KernelDllBase + p.KernelDllSize)) {
|
|
limit = MemoryManager.PagePad(p.KernelDllBase + p.KernelDllSize);
|
|
}
|
|
}
|
|
DebugStub.WriteLine("KernelDllBase {0,8:x} size {1,8:x} limit {2,8:x} \n", __arglist(p.KernelDllBase, p.KernelDllSize, limit));
|
|
|
|
SMAPINFO *smap = (SMAPINFO*)p.Smap;
|
|
for (uint i = 0; i < p.SmapCount; i++) {
|
|
if (smap[i].type == (ulong)SMAPINFO.AddressType.Free) {
|
|
if (RangeWithinLimits(smap[i].addr, smap[i].size, upperLimit, lowerLimit, out baseAddress, out size)){
|
|
if(baseAddress + size >= limit) {
|
|
UIntPtr start = baseAddress;
|
|
UIntPtr end = baseAddress + size;
|
|
|
|
if (start < limit) {
|
|
start = limit;
|
|
}
|
|
if ((end - start) >= pageCount * sizeof(uint)) {
|
|
pageTable = (uint *) start;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (pageTable == null) {
|
|
DebugStub.WriteLine("pageTable == null, un-able to find page table memory\n");
|
|
DebugStub.Break();
|
|
}
|
|
|
|
DebugStub.WriteLine("p.PhysicalBase 0x{0,8:x8} upperLimit: 0x{1,8:x8}, pageTable: 0x{2,8:x8}",
|
|
__arglist(p.PhysicalBase, upperLimit, (UIntPtr)pageTable));
|
|
|
|
Tracing.Log(Tracing.Debug,
|
|
"Limit of RAM={0,8:x}, entries={1:x}, table={2:x}",
|
|
upperLimit, pageCount, (UIntPtr) pageTable);
|
|
|
|
// Initialize all page descriptors to Unknown.
|
|
SetPages(0, pageCount, MemoryManager.PageUnknown);
|
|
|
|
// Second pass over SMAP, mark known RAM.
|
|
// we use to bootloader knowledge to map our address space
|
|
for (uint i = 0; i < p.SmapCount; i++) {
|
|
if (RangeWithinLimits(smap[i].addr, smap[i].size, upperLimit, lowerLimit, out baseAddress, out size)) {
|
|
if (smap[i].type == (ulong)SMAPINFO.AddressType.Free) {
|
|
SetRange(baseAddress, size, MemoryManager.PageFree);
|
|
}
|
|
else if(smap[i].type == (ulong)SMAPINFO.AddressType.KernelNonGc) {
|
|
SetRange(baseAddress, size, MemoryManager.KernelPageNonGC);
|
|
}
|
|
else if(smap[i].type == (ulong)SMAPINFO.AddressType.KernelStack) {
|
|
SetRange(baseAddress, size, MemoryManager.KernelPageStack);
|
|
}
|
|
else {
|
|
;
|
|
}
|
|
}
|
|
else {
|
|
DebugStub.WriteLine("Ignoring Entry: Smap range {0:x8}...{1:x8} memory range {2:x8}...{3:x8}",
|
|
__arglist(
|
|
smap[i].addr,
|
|
smap[i].addr + smap[i].size,
|
|
lowerLimit,
|
|
upperLimit)
|
|
);
|
|
}
|
|
}
|
|
|
|
// Set non-physical pages as unknown
|
|
SetRange(0, Platform.ThePlatform.PhysicalBase, MemoryManager.PageUnknown);
|
|
|
|
// Record the page table memory.
|
|
SetRange((UIntPtr) pageTable, pageCount * sizeof(uint), MemoryManager.KernelPageNonGC);
|
|
|
|
|
|
// Initialize the free and save lists.
|
|
fixed (FreeNode *tail = &freeListTail) {
|
|
freeList = tail;
|
|
FreeNode.Init(freeList, false);
|
|
}
|
|
fixed (FreeNode *tail = &saveListTail) {
|
|
saveList = tail;
|
|
FreeNode.Init(saveList, true);
|
|
}
|
|
|
|
availableMemory = UIntPtr.Zero;
|
|
|
|
// Initialize the memory reservations
|
|
MemoryReservations.Initialize();
|
|
|
|
uint *desc = pageTable;
|
|
uint last = *desc & MemoryManager.SystemPageMask;
|
|
UIntPtr begin = UIntPtr.Zero;
|
|
|
|
for (UIntPtr i = UIntPtr.Zero; i < pageCount; i++) {
|
|
uint val = *desc++ & MemoryManager.SystemPageMask;
|
|
|
|
if (val != last) {
|
|
if (last == MemoryManager.PageFree) {
|
|
FreeNode.CreateAndInsert(freeList,
|
|
AddrFromPage(begin),
|
|
AddrFromPage(i) - AddrFromPage(begin));
|
|
}
|
|
begin = i;
|
|
last = val;
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static void Finalize()
|
|
{
|
|
// Doesn't actually do anything.
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
private static bool Lock()
|
|
{
|
|
bool enabled = Processor.DisableInterrupts();
|
|
#if SINGULARITY_MP
|
|
pageLock.Acquire(Thread.CurrentThread);
|
|
#endif // SINGULARITY_MP
|
|
return enabled;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
private static void Unlock(bool iflag)
|
|
{
|
|
#if SINGULARITY_MP
|
|
pageLock.Release(Thread.CurrentThread);
|
|
#endif // SINGULARITY_MP
|
|
Processor.RestoreInterrupts(iflag);
|
|
}
|
|
|
|
// Currently, we just return the BSP free list. In the
|
|
// future, this should consult the ProcessorMemoryMap
|
|
private static unsafe FreeNode* GetFreeList()
|
|
{
|
|
return freeList;
|
|
}
|
|
|
|
private static unsafe FreeNode* GetSaveList()
|
|
{
|
|
return saveList;
|
|
}
|
|
|
|
|
|
|
|
//////////////////////////////////////////////////////////////////////
|
|
//
|
|
internal static UIntPtr PageCount
|
|
{
|
|
get { return pageCount; }
|
|
}
|
|
|
|
internal static unsafe uint * PageTable
|
|
{
|
|
get { return pageTable; }
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
[FlatPagesLock]
|
|
internal static UIntPtr StackAllocate(UIntPtr bytes,
|
|
UIntPtr reserve,
|
|
UIntPtr alignment,
|
|
Process process,
|
|
uint extra,
|
|
bool kernelAllocation,
|
|
bool initialStack)
|
|
{
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(960);
|
|
#endif
|
|
UIntPtr got = new UIntPtr();
|
|
|
|
bool iflag = Lock();
|
|
try {
|
|
got = RawAllocate(bytes, reserve, alignment,
|
|
(process != null ? process.ProcessTag : MemoryManager.KernelPage)
|
|
| (extra & MemoryManager.ExtraMask)
|
|
| (uint)PageType.Stack, kernelAllocation);
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, "{0:x8} Allocate({1:x},{2:x},{3:x}",
|
|
Kernel.AddressOf(process), bytes, reserve,
|
|
alignment);
|
|
#endif
|
|
if (got != UIntPtr.Zero) {
|
|
|
|
if (initialStack) {
|
|
// increase our kernel stack reservation
|
|
MemoryReservations.ThreadCreateNotification();
|
|
}
|
|
|
|
if (process != null) {
|
|
process.Allocated(bytes);
|
|
}
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(961);
|
|
#endif
|
|
return got;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
[FlatPagesLock]
|
|
internal static void StackFree(UIntPtr addr,
|
|
UIntPtr bytes,
|
|
Process process,
|
|
bool kernelAllocation,
|
|
bool initialStack)
|
|
{
|
|
bool iflag = Lock();
|
|
try {
|
|
RawFree(addr, bytes, process != null ? process.ProcessTag : MemoryManager.KernelPage, true, kernelAllocation);
|
|
if (process != null) {
|
|
process.Freed(bytes);
|
|
}
|
|
|
|
if (initialStack) {
|
|
// release our kernel stack reservation
|
|
MemoryReservations.ThreadDestroyNotification();
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
return;
|
|
}
|
|
|
|
//
|
|
// If process == null, its a kernel allocation, otherwise its
|
|
// a user (SIP) allocation.
|
|
//
|
|
[NoStackLinkCheckTrans]
|
|
[FlatPagesLock]
|
|
internal static UIntPtr Allocate(UIntPtr bytes,
|
|
UIntPtr reserve,
|
|
UIntPtr alignment,
|
|
Process process,
|
|
uint extra,
|
|
PageType type)
|
|
{
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(960);
|
|
#endif
|
|
UIntPtr got = new UIntPtr();
|
|
|
|
bool iflag = Lock();
|
|
try {
|
|
got = RawAllocate(bytes, reserve, alignment,
|
|
(process != null ? process.ProcessTag : MemoryManager.KernelPage)
|
|
| (extra & MemoryManager.ExtraMask)
|
|
| (uint)type, true);
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, "{0:x8} Allocate({1:x},{2:x},{3:x}",
|
|
Kernel.AddressOf(process), bytes, reserve,
|
|
alignment);
|
|
#endif
|
|
if (process != null) {
|
|
process.Allocated(bytes);
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(961);
|
|
#endif
|
|
return got;
|
|
}
|
|
|
|
[FlatPagesLock]
|
|
internal static UIntPtr AllocateBelow(UIntPtr limit,
|
|
UIntPtr bytes,
|
|
UIntPtr alignment,
|
|
Process process,
|
|
uint extra,
|
|
PageType type)
|
|
{
|
|
UIntPtr got = new UIntPtr();
|
|
|
|
bool iflag = Lock();
|
|
try {
|
|
got = RawAllocateBelow(limit, bytes, alignment,
|
|
(process != null ? process.ProcessTag : MemoryManager.KernelPage)
|
|
| (extra & MemoryManager.ExtraMask)
|
|
| (uint)type);
|
|
if (process != null) {
|
|
process.Allocated(bytes);
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
return got;
|
|
}
|
|
|
|
[FlatPagesLock]
|
|
internal static UIntPtr AllocateExtend(UIntPtr addr,
|
|
UIntPtr bytes,
|
|
Process process,
|
|
uint extra,
|
|
PageType type)
|
|
{
|
|
UIntPtr got = new UIntPtr();
|
|
|
|
bool iflag = Lock();
|
|
try {
|
|
uint tag =
|
|
(process != null ?
|
|
process.ProcessTag :
|
|
MemoryManager.KernelPage)
|
|
| (extra & MemoryManager.ExtraMask)
|
|
| (uint)type;
|
|
got = RawAllocateExtend(addr, bytes, tag);
|
|
if (got != UIntPtr.Zero && process != null) {
|
|
process.Allocated(bytes);
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
return got;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
[FlatPagesLock]
|
|
internal static void Free(UIntPtr addr,
|
|
UIntPtr bytes,
|
|
Process process)
|
|
{
|
|
bool iflag = Lock();
|
|
try {
|
|
RawFree(addr, bytes, process != null ? process.ProcessTag : MemoryManager.KernelPage, false, true);
|
|
if (process != null) {
|
|
process.Freed(bytes);
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
}
|
|
|
|
[FlatPagesLock]
|
|
internal static unsafe UIntPtr FreeAll(Process process)
|
|
{
|
|
DebugStub.Assert(process != null,
|
|
"FlatPages.FreeAll null process");
|
|
DebugStub.Assert(process.ProcessTag != MemoryManager.KernelPage,
|
|
"FlatPages.FreeAll ProcessTag={0}",
|
|
__arglist(process.ProcessTag));
|
|
|
|
uint tag = process.ProcessTag & MemoryManager.ProcessPageMask;
|
|
uint *pageLimit = pageTable + pageCount;
|
|
UIntPtr bytes = 0;
|
|
|
|
Tracing.Log(Tracing.Debug, "FreeAll({0,8:x})", tag);
|
|
|
|
for (uint *begin = pageTable; begin < pageLimit;) {
|
|
uint *limit = begin;
|
|
uint val = (*limit++) & MemoryManager.ProcessPageMask;
|
|
#if VERBOSE
|
|
unchecked {
|
|
Tracing.Log(Tracing.Debug, " {0,8:x}: {1,8:x}",
|
|
AddrFromPage((UIntPtr)(begin - pageTable)),
|
|
val);
|
|
}
|
|
#endif
|
|
|
|
if (val == tag) {
|
|
while ((((*limit) & MemoryManager.ProcessPageMask) == tag) && (limit < pageLimit)) {
|
|
limit++;
|
|
}
|
|
|
|
UIntPtr page = (UIntPtr)(begin - pageTable);
|
|
UIntPtr size = (UIntPtr)(limit - begin);
|
|
|
|
Tracing.Log(Tracing.Debug,
|
|
" {0,8:x}..{1,8:x} : {2,8:x} [will free]",
|
|
page << MemoryManager.PageBits, (page + size) << MemoryManager.PageBits,
|
|
*begin);
|
|
|
|
bool iflag = Lock();
|
|
UIntPtr sizeInBytes = (size << MemoryManager.PageBits);
|
|
try {
|
|
RawFree(AddrFromPage(page), sizeInBytes, tag, false, false);
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
|
|
bytes += size;
|
|
}
|
|
else {
|
|
while ((((*limit) & MemoryManager.ProcessPageMask) != tag) && (limit < pageLimit)) {
|
|
limit++;
|
|
}
|
|
|
|
UIntPtr page = (UIntPtr)(begin - pageTable);
|
|
UIntPtr size = (UIntPtr)(limit - begin);
|
|
|
|
Tracing.Log(Tracing.Debug,
|
|
"- {0,8:x}..{1,8:x} : {2,8:x} [will free]",
|
|
page << MemoryManager.PageBits, (page + size) << MemoryManager.PageBits,
|
|
*begin);
|
|
}
|
|
begin = limit;
|
|
}
|
|
if (process != null) {
|
|
process.Freed(bytes * MemoryManager.PageSize);
|
|
}
|
|
return bytes * MemoryManager.PageSize;
|
|
}
|
|
|
|
[FlatPagesLock]
|
|
internal static PageType Query(UIntPtr queryAddr,
|
|
Process process,
|
|
out UIntPtr regionAddr,
|
|
out UIntPtr regionSize)
|
|
{
|
|
PageType type = new PageType();
|
|
|
|
bool iflag = Lock();
|
|
try {
|
|
type = RawQuery(queryAddr,
|
|
process != null ? process.ProcessTag : 0,
|
|
out regionAddr, out regionSize);
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
return type;
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////
|
|
//
|
|
[NoStackLinkCheckTrans]
|
|
private static unsafe UIntPtr RawAllocate(UIntPtr bytes,
|
|
UIntPtr reserve,
|
|
UIntPtr alignment,
|
|
uint tag,
|
|
bool kernelAllocation)
|
|
{
|
|
VTable.Assert(Processor.InterruptsDisabled());
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(970);
|
|
#endif
|
|
if (alignment < MemoryManager.PageSize) {
|
|
alignment = MemoryManager.PageSize;
|
|
}
|
|
if (reserve < bytes) {
|
|
reserve = bytes;
|
|
}
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug,
|
|
" size={0:x}, res={1:x}, aln={2:x}, tag={3:x}",
|
|
bytes, reserve, alignment, tag);
|
|
#endif
|
|
// Check the allocation against memory reservations
|
|
if (MemoryReservations.MemoryReservationExceeded((ulong)availableMemory, tag, kernelAllocation, bytes + reserve + alignment)) {
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
FreeNode * node = FreeNode.FindGoodFit(GetFreeList(), reserve, alignment);
|
|
if (node == null) {
|
|
node = FreeNode.FindGoodFit(GetFreeList(), bytes, alignment);
|
|
if (node == null) {
|
|
node = FreeNode.FindGoodFit(GetSaveList(), reserve, alignment);
|
|
if (node == null) {
|
|
node = FreeNode.FindGoodFit(GetSaveList(), bytes, alignment);
|
|
if (node == null) {
|
|
// TODO: We should try to combine free and save pages...
|
|
// But for now, we just fail.
|
|
DebugStub.WriteLine("Failed to allocate on the heap!!! {0} bytes kernelalloc {1}\n",
|
|
__arglist(bytes, kernelAllocation));
|
|
return UIntPtr.Zero;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
UIntPtr addr = (UIntPtr)node;
|
|
UIntPtr adjust = SpaceNotAligned(addr + node->bytes, alignment);
|
|
UIntPtr found = node->bytes;
|
|
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, " 0. {0:x8}..{1:x8}: res={2:x}, adj={3:x}",
|
|
addr, addr + found, reserve, adjust);
|
|
#endif
|
|
if (found > reserve + adjust) {
|
|
// Put the extraneous pages in the free list.
|
|
FreeNode.ReturnExtraBelow(GetFreeList(), ref addr, ref found, reserve + adjust);
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, " 1. {0:x8}..{1:x8}",
|
|
addr, addr + found);
|
|
#endif
|
|
}
|
|
#if ALLOW_BOOT_ARGLIST
|
|
DebugStub.Assert
|
|
(SpaceNotAligned(addr, alignment) == UIntPtr.Zero,
|
|
"FlatPages.RawAllocate not aligned addr={0} alignment={1}",
|
|
__arglist(addr, alignment));
|
|
#endif
|
|
if (found > bytes) {
|
|
// Put extra pages in the save list.
|
|
FreeNode.ReturnExtraAbove(GetSaveList(), addr, ref found, bytes);
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, " 2. {0:x8}..{1:x8}",
|
|
addr, addr + found);
|
|
#endif
|
|
}
|
|
#if ALLOW_BOOT_ARGLIST
|
|
DebugStub.Assert
|
|
(found == bytes,
|
|
"FlatPages.RawAllocate wrong amount found={0} bytes={1}",
|
|
__arglist(found, bytes));
|
|
#endif
|
|
SetRange(addr, found, tag);
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(971);
|
|
#endif
|
|
//
|
|
// The owner information is not reliable for determining kernel
|
|
// or SIP allocation when the page type is stack, so the logic
|
|
// is a little extra complicated here.
|
|
//
|
|
if ((tag & MemoryManager.TypeMask) == (uint)PageType.Stack) {
|
|
|
|
// Stack pages use the kernelAllocation flag
|
|
if (kernelAllocation) {
|
|
MemoryReservations.AllocationForStack((ulong)bytes);
|
|
}
|
|
else {
|
|
MemoryReservations.SIPAllocationForStack((ulong)bytes);
|
|
}
|
|
}
|
|
else if ((tag & MemoryManager.ProcessPageMask) != MemoryManager.KernelPage) {
|
|
|
|
// process tag is reliable for non-stack allocations
|
|
MemoryReservations.SIPAllocationForHeap((ulong)bytes);
|
|
}
|
|
else {
|
|
MemoryReservations.AllocationForHeap((ulong)bytes);
|
|
}
|
|
LogMemoryOperation(kernelAllocation, tag, addr, bytes);
|
|
return addr;
|
|
}
|
|
|
|
private static unsafe UIntPtr RawAllocateBelow(UIntPtr limit,
|
|
UIntPtr bytes,
|
|
UIntPtr alignment,
|
|
uint tag)
|
|
{
|
|
VTable.Assert(Processor.InterruptsDisabled());
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(972);
|
|
#endif
|
|
if (alignment < MemoryManager.PageSize) {
|
|
alignment = MemoryManager.PageSize;
|
|
}
|
|
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug,
|
|
"lim={0:x8}, size={1:x8}, align={2}, tag={3:x}",
|
|
limit, bytes, alignment, tag);
|
|
#endif
|
|
// Check the allocation against memory reservations
|
|
if (MemoryReservations.MemoryReservationExceeded((ulong)availableMemory, tag, false, bytes + alignment)) {
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
FreeNode * node = FreeNode.FindBelow(limit, GetFreeList(), bytes, alignment);
|
|
if (node == null) {
|
|
node = FreeNode.FindBelow(limit, GetSaveList(), bytes, alignment);
|
|
if (node == null) {
|
|
// TODO: We should try to combine free and save pages...
|
|
// But for now, we just fail.
|
|
return UIntPtr.Zero;
|
|
}
|
|
}
|
|
|
|
UIntPtr addr = (UIntPtr)node;
|
|
UIntPtr adjust = SpaceToAlign(addr, alignment);
|
|
UIntPtr found = node->bytes;
|
|
|
|
if (adjust != UIntPtr.Zero) {
|
|
// Put the alignment pages in free list.
|
|
FreeNode.ReturnExtraBelow(GetFreeList(), ref addr, ref found, found - adjust);
|
|
}
|
|
DebugStub.Assert
|
|
(SpaceNotAligned(addr, alignment) == UIntPtr.Zero,
|
|
"FlatPages.RawAllocateBelow not aligned addr={0} alignment={1}",
|
|
__arglist(addr, alignment));
|
|
|
|
if (found > bytes) {
|
|
// Put the extra pages in free list.
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug,
|
|
"found {0:x8}..{1:x8}, found={3:x8}, keep={4:x8}",
|
|
addr, addr + found, found, bytes);
|
|
#endif
|
|
|
|
FreeNode.ReturnExtraAbove(GetFreeList(), addr, ref found, bytes);
|
|
}
|
|
|
|
DebugStub.Assert
|
|
(found == bytes,
|
|
"FlatPages.RawAllocateBelow wrong amount found={0} bytes={1}",
|
|
__arglist(found, bytes));
|
|
|
|
SetRange(addr, found, tag);
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(973);
|
|
#endif
|
|
// Stacks are only allocated through RawAllocate()
|
|
VTable.Assert((tag & MemoryManager.TypeMask) != (uint)PageType.Stack);
|
|
|
|
if ((tag & MemoryManager.ProcessPageMask) != MemoryManager.KernelPage) {
|
|
MemoryReservations.SIPAllocationForHeap((ulong)bytes);
|
|
}
|
|
else {
|
|
MemoryReservations.AllocationForHeap((ulong)bytes);
|
|
}
|
|
|
|
LogMemoryOperation(false, tag, addr, bytes);
|
|
|
|
return addr;
|
|
}
|
|
|
|
private static unsafe UIntPtr RawAllocateExtend(UIntPtr addr,
|
|
UIntPtr bytes,
|
|
uint tag)
|
|
{
|
|
VTable.Assert(Processor.InterruptsDisabled());
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(974);
|
|
#endif
|
|
UIntPtr page = MemoryManager.PageFromAddr(addr);
|
|
if (*(pageTable + page) != MemoryManager.PageFreeFirst) {
|
|
Tracing.Log(Tracing.Error,
|
|
"{0:x} is not first free page {1:x}.",
|
|
addr, *(pageTable + page));
|
|
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
// Check the allocation against memory reservations
|
|
if (MemoryReservations.MemoryReservationExceeded((ulong)availableMemory, tag, false, bytes)) {
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
FreeNode *node = (FreeNode *)addr;
|
|
if (node->bytes < bytes) {
|
|
Tracing.Log(Tracing.Error,
|
|
"Only {0:x} free bytes, not {1:x} as requested.",
|
|
node->bytes, bytes);
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, "addr={0:x8}, size={1:x8}, tag={2:x}",
|
|
addr, bytes, tag);
|
|
#endif
|
|
|
|
// Remove the node from the list.
|
|
FreeNode.Remove(node);
|
|
|
|
UIntPtr found = node->bytes;
|
|
|
|
if (found > bytes) {
|
|
// Save the extra pages in the save list.
|
|
FreeNode.ReturnExtraAbove(GetSaveList(), addr, ref found, bytes);
|
|
}
|
|
|
|
DebugStub.Assert
|
|
(found == bytes,
|
|
"FlatPages.RawAllocateExtend wrong amount found={0} bytes{1}",
|
|
__arglist(found, bytes));
|
|
|
|
SetRange(addr, found, tag);
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(975);
|
|
#endif
|
|
// Stacks are only allocated through RawAllocate()
|
|
VTable.Assert((tag & MemoryManager.TypeMask) != (uint)PageType.Stack);
|
|
|
|
if ((tag & MemoryManager.ProcessPageMask) != MemoryManager.KernelPage) {
|
|
MemoryReservations.SIPAllocationForHeap((ulong)bytes);
|
|
}
|
|
else {
|
|
MemoryReservations.AllocationForHeap((ulong)bytes);
|
|
}
|
|
LogMemoryOperation(false, tag, addr, bytes);
|
|
|
|
return addr;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
private static unsafe bool VerifyOwner(UIntPtr page, UIntPtr pages, uint tag)
|
|
{
|
|
tag &= MemoryManager.ProcessPageMask;
|
|
for (UIntPtr i = UIntPtr.Zero; i < pages; i++) {
|
|
if (((*(pageTable + page + i)) & MemoryManager.ProcessPageMask) != tag) {
|
|
DebugStub.WriteLine("FlatPages.VerifyOwner page={0} i={1} tag={2} address 0x{3,8:x}",
|
|
__arglist(page, i, tag, ((pageTable+page))));
|
|
return false;
|
|
}
|
|
#if false
|
|
DebugStub.Assert
|
|
(((*(pageTable + page + i)) & MemoryManager.ProcessPageMask) == tag,
|
|
"FlatPages.VerifyOwner page={0} i={1} tag={2}",
|
|
__arglist(page, i, tag));
|
|
#endif
|
|
}
|
|
return true;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
private static unsafe void RawFree(UIntPtr addr, UIntPtr bytes, uint tag, bool stack, bool kernelAllocation)
|
|
{
|
|
VTable.Assert(Processor.InterruptsDisabled());
|
|
UIntPtr bytesIn = bytes;
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(976);
|
|
#endif
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, "adr={0:x}, size={1:x}, tag={2:x}",
|
|
addr, bytes, tag);
|
|
#endif
|
|
|
|
bool result = VerifyOwner(MemoryManager.PageFromAddr(addr), MemoryManager.PagesFromBytes(bytes), tag);
|
|
if(false == result) {
|
|
DebugStub.WriteLine("VerifyOwner Failed for addr 0x{0,8:x} bytes {1} page {2} pages {3}\n",
|
|
__arglist(addr, bytes, MemoryManager.PageFromAddr(addr), MemoryManager.PagesFromBytes(bytes)));
|
|
DebugStub.Break();
|
|
}
|
|
|
|
FreeNode *node = FreeNode.GetNodeAt(addr + bytes);
|
|
FreeNode *prev = FreeNode.GetNodeFromLast(addr - MemoryManager.PageSize);
|
|
|
|
SetRange(addr, bytes, MemoryManager.PageFree);
|
|
// Try to combine with the previous region if it isn't a save region.
|
|
if (prev != null && prev->isSave == false) {
|
|
addr = (UIntPtr)prev;
|
|
bytes += prev->bytes;
|
|
|
|
FreeNode.Remove(prev);
|
|
}
|
|
// Try to combine with the next region even if it was a save region.
|
|
if (node != null) {
|
|
bytes += node->bytes;
|
|
FreeNode.Remove(node);
|
|
if (node->isSave) {
|
|
// If next was save, then try to combine with the follower.
|
|
node = FreeNode.GetNodeAt(addr + bytes);
|
|
if (node != null) {
|
|
bytes += node->bytes;
|
|
FreeNode.Remove(node);
|
|
}
|
|
}
|
|
}
|
|
// Create the free node.
|
|
FreeNode.CreateAndInsert(GetFreeList(), addr, bytes);
|
|
#if NO__TRACE_PAGES
|
|
#else
|
|
Kernel.Waypoint(977);
|
|
#endif
|
|
// Stack allocations are ambiguous and must be called out separately
|
|
if (stack) {
|
|
if (kernelAllocation) {
|
|
MemoryReservations.FreeForStack((ulong)bytesIn);
|
|
}
|
|
else {
|
|
MemoryReservations.SIPFreeForStack((ulong)bytesIn);
|
|
}
|
|
}
|
|
else if (tag != MemoryManager.KernelPage) {
|
|
|
|
// Heap pages are reliable as per kernel/SIP
|
|
MemoryReservations.SIPFreeForHeap((ulong)bytesIn);
|
|
}
|
|
else {
|
|
MemoryReservations.FreeForHeap((ulong)bytesIn);
|
|
}
|
|
LogMemoryOperation(false, tag, addr, bytesIn);
|
|
}
|
|
|
|
private static unsafe PageType RawQuery(UIntPtr queryAddr,
|
|
uint tag,
|
|
out UIntPtr regionAddr,
|
|
out UIntPtr regionSize)
|
|
{
|
|
VTable.Assert(Processor.InterruptsDisabled());
|
|
UIntPtr page = MemoryManager.PageFromAddr(queryAddr);
|
|
UIntPtr startPage = page;
|
|
UIntPtr limitPage = page + 1;
|
|
|
|
PageType type;
|
|
uint val = *(pageTable + startPage);
|
|
bool used = ((val & MemoryManager.ProcessPageMask) != MemoryManager.SystemPage);
|
|
|
|
if ((val & MemoryManager.ProcessPageMask) == MemoryManager.SystemPage) {
|
|
// Found a system page.
|
|
type = (tag == 0) ? (PageType)(val & MemoryManager.TypeMask) : PageType.Unknown;
|
|
|
|
// Find the start of the SystemPage region.
|
|
for (; startPage > UIntPtr.Zero; startPage--) {
|
|
val = *(pageTable + startPage - 1);
|
|
if ((val & MemoryManager.ProcessPageMask) != MemoryManager.SystemPage) {
|
|
break;
|
|
}
|
|
}
|
|
// Find the end of the SystemPage region
|
|
for (; limitPage < pageCount; limitPage++) {
|
|
val = *(pageTable + limitPage);
|
|
if ((val & MemoryManager.ProcessPageMask) != MemoryManager.SystemPage) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
// Found a process page.
|
|
uint ptag = val & MemoryManager.ProcessPageMask;
|
|
type = (tag == 0 || ptag == tag)
|
|
? (PageType)(val & MemoryManager.TypeMask) : PageType.Unknown;
|
|
|
|
if ((val & MemoryManager.TypeMask) == (uint)PageType.System) {
|
|
// Find the start of the process code region.
|
|
for (; startPage > UIntPtr.Zero; startPage--) {
|
|
val = *(pageTable + startPage - 1);
|
|
if ((val & MemoryManager.ProcessPageMask) != ptag ||
|
|
(val & MemoryManager.TypeMask) != (uint)PageType.System) {
|
|
break;
|
|
}
|
|
}
|
|
// Find the end of the process code region
|
|
for (; limitPage < pageCount; limitPage++) {
|
|
val = *(pageTable + limitPage);
|
|
if ((val & MemoryManager.ProcessPageMask) != ptag ||
|
|
(val & MemoryManager.TypeMask) != (uint)PageType.System) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
// Find the start of the process region.
|
|
for (; startPage > UIntPtr.Zero; startPage--) {
|
|
val = *(pageTable + startPage - 1);
|
|
if ((val & MemoryManager.ProcessPageMask) != ptag ||
|
|
(val & MemoryManager.TypeMask) == (uint)PageType.System) {
|
|
break;
|
|
}
|
|
}
|
|
// Find the end of the process region
|
|
for (; limitPage < pageCount; limitPage++) {
|
|
val = *(pageTable + limitPage);
|
|
if ((val & MemoryManager.ProcessPageMask) != ptag ||
|
|
(val & MemoryManager.TypeMask) == (uint)PageType.System) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, "[{0:x8}..{1:x8}]",
|
|
AddrFromPage(startPage), AddrFromPage(limitPage));
|
|
#endif
|
|
regionAddr = AddrFromPage(startPage);
|
|
regionSize = AddrFromPage(limitPage) - AddrFromPage(startPage);
|
|
|
|
return type;
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////////
|
|
//
|
|
private static unsafe void DumpQuery(UIntPtr addr)
|
|
{
|
|
UIntPtr regionAddr;
|
|
UIntPtr regionSize;
|
|
PageType type = RawQuery(addr, 0, out regionAddr, out regionSize);
|
|
|
|
Tracing.Log(Tracing.Debug, " {0:x8} => {1:x8}..{2:x8} [{3:x}]",
|
|
addr, regionAddr, regionAddr + regionSize, (uint)type);
|
|
}
|
|
|
|
private static unsafe void DumpFreeNodes(FreeNode *list)
|
|
{
|
|
DumpFreeNodes(list, list->isSave);
|
|
}
|
|
|
|
private static unsafe void DumpFreeNodes(FreeNode *list, bool isSave)
|
|
{
|
|
if (isSave) {
|
|
DebugStub.WriteLine(" SaveList:");
|
|
}
|
|
else {
|
|
DebugStub.WriteLine(" FreeList:");
|
|
}
|
|
|
|
for (FreeNode *node = list->next; node != list; node = node->next) {
|
|
string fmt = " {0:x8}..{1:x8} prev={2:x8}, next={3:x8}, last={4:x8} ";
|
|
if (node->isSave != isSave) {
|
|
if (node->isSave) {
|
|
fmt = " {0:x8}..{1:x8} prev={2:x8}, next={3:x8}, last={4:x8} [Save!]";
|
|
}
|
|
else {
|
|
fmt = " {0:x8}..{1:x8} prev={2:x8}, next={3:x8}, last={4:x8} [Free!]";
|
|
}
|
|
}
|
|
unchecked {
|
|
DebugStub.WriteLine( fmt,
|
|
__arglist(
|
|
(UIntPtr)node, (UIntPtr)node + node->bytes,
|
|
(UIntPtr)node->prev, (UIntPtr)node->next,
|
|
(UIntPtr)node->last));
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static unsafe void Dump(string where)
|
|
{
|
|
DebugStub.WriteLine( "FlatPages.Dump: {0}", __arglist(where));
|
|
|
|
uint *descriptors = pageTable;
|
|
uint last = *descriptors++ & MemoryManager.SystemPageMask;
|
|
UIntPtr begin = UIntPtr.Zero;
|
|
|
|
UIntPtr freePages = UIntPtr.Zero;
|
|
UIntPtr usedPages = UIntPtr.Zero;
|
|
UIntPtr unknownPages = UIntPtr.Zero;
|
|
UIntPtr sharedPages = UIntPtr.Zero;
|
|
|
|
for (UIntPtr i = (UIntPtr)1; i < pageCount; i++) {
|
|
uint dsc = *descriptors++;
|
|
uint val = dsc & MemoryManager.SystemPageMask;
|
|
|
|
switch (val) {
|
|
case MemoryManager.PageUnknown:
|
|
unknownPages++;
|
|
break;
|
|
case MemoryManager.PageShared:
|
|
sharedPages++;
|
|
break;
|
|
case MemoryManager.PageFree:
|
|
freePages++;
|
|
break;
|
|
default:
|
|
usedPages++;
|
|
break;
|
|
}
|
|
|
|
if (dsc != last) {
|
|
DebugStub.WriteLine( " {0:x8}..{1:x8} : {2:x8} : {3:x8}",
|
|
__arglist(
|
|
begin << MemoryManager.PageBits, i << MemoryManager.PageBits, last,
|
|
(i - begin) << MemoryManager.PageBits));
|
|
last = dsc;
|
|
begin = i;
|
|
}
|
|
}
|
|
|
|
DebugStub.WriteLine(" {0:x8}..{1:x8} : {2:x8} : {3:x8}",
|
|
__arglist(
|
|
begin << MemoryManager.PageBits, pageCount << MemoryManager.PageBits, last,
|
|
(pageCount - begin) << MemoryManager.PageBits));
|
|
|
|
DumpFreeNodes(GetFreeList(), false);
|
|
DumpFreeNodes(GetSaveList(), true);
|
|
|
|
DebugStub.WriteLine("Totals: free={0:x8}, used={1:x8}, unknown={2:x8}, reserved={3:x8}",
|
|
__arglist(
|
|
freePages << MemoryManager.PageBits,
|
|
usedPages << MemoryManager.PageBits,
|
|
unknownPages << MemoryManager.PageBits,
|
|
sharedPages << MemoryManager.PageBits));
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////
|
|
//
|
|
[NoStackLinkCheckTrans]
|
|
[NoHeapAllocation]
|
|
private static unsafe void SetPages(UIntPtr startPage, UIntPtr pageCount, uint tag)
|
|
{
|
|
uint * descriptor = pageTable + startPage;
|
|
|
|
#if VERY_VERBOSE
|
|
Tracing.Log(Tracing.Audit,
|
|
"SetPages(beg={0:x},num={1:x},val={2:x}",
|
|
startPage << MemoryManager.PageBits,
|
|
pageCount << MemoryManager.PageBits,
|
|
tag);
|
|
|
|
#endif
|
|
|
|
while (pageCount > UIntPtr.Zero) {
|
|
*descriptor++ = tag;
|
|
pageCount--;
|
|
}
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
[NoHeapAllocation]
|
|
internal static void SetRange(UIntPtr start, UIntPtr bytes, uint tag)
|
|
{
|
|
if (start > upperLimit) {
|
|
return;
|
|
}
|
|
if (start + bytes > upperLimit) {
|
|
bytes = upperLimit - start;
|
|
}
|
|
SetPages(MemoryManager.PageFromAddr(start), MemoryManager.PagesFromBytes(bytes), tag);
|
|
}
|
|
//////////////////////////////////////////////////////////////////////////
|
|
//
|
|
|
|
public static UIntPtr GetMaxMemory()
|
|
{
|
|
return upperLimit;
|
|
}
|
|
|
|
[FlatPagesLock]
|
|
public static unsafe UIntPtr GetFreeMemory()
|
|
{
|
|
#if TEST_MEMORY_TRACKING
|
|
UIntPtr retval = 0;
|
|
bool iflag = Lock();
|
|
|
|
try {
|
|
uint *descriptors = pageTable;
|
|
|
|
// Count free pages
|
|
for (UIntPtr i = (UIntPtr)1; i < pageCount; i++) {
|
|
uint dsc = *descriptors++;
|
|
uint val = dsc & MemoryManager.SystemPageMask;
|
|
|
|
if (val == MemoryManager.PageFree) {
|
|
retval++;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Make sure everything reports the same info
|
|
//
|
|
UIntPtr freeListSize = FreeNode.GetFreeListTotalSize(freeList);
|
|
UIntPtr saveListSize = FreeNode.GetFreeListTotalSize(saveList);
|
|
UIntPtr totalFreeList = freeListSize + saveListSize;
|
|
|
|
if (((retval * MemoryManager.PageSize) != totalFreeList) ||
|
|
(totalFreeList != availableMemory)) {
|
|
|
|
DebugStub.WriteLine("GetFreeMemory: Internal inconsistency");
|
|
|
|
DebugStub.WriteLine("Pages total 0x{0:x8}", __arglist(retval * MemoryManager.PageSize));
|
|
|
|
DebugStub.WriteLine("FreeList 0x{0:x8}, SaveList 0x{1:x8}, Total 0x{2:x8}",
|
|
__arglist(
|
|
(ulong)freeListSize, (ulong)saveListSize,
|
|
((ulong)freeListSize + (ulong)saveListSize)));
|
|
|
|
DebugStub.WriteLine("availableMemory: 0x{0:x8}\n", __arglist(availableMemory));
|
|
|
|
DebugStub.Break();
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
|
|
return retval * MemoryManager.PageSize;
|
|
#else
|
|
//
|
|
// We track it in real time so it may be used by other components
|
|
// at runtime.
|
|
//
|
|
return availableMemory;
|
|
#endif
|
|
}
|
|
|
|
public static unsafe UIntPtr GetUsedMemory()
|
|
{
|
|
uint *descriptors = pageTable;
|
|
UIntPtr retval = 0;
|
|
|
|
// Count non-free pages
|
|
for (UIntPtr i = (UIntPtr)1; i < pageCount; i++) {
|
|
uint dsc = *descriptors++;
|
|
uint val = dsc & MemoryManager.SystemPageMask;
|
|
|
|
if (val != MemoryManager.PageFree) {
|
|
retval++;
|
|
}
|
|
}
|
|
|
|
return retval * MemoryManager.PageSize;
|
|
}
|
|
|
|
//
|
|
// Memory monitoring support
|
|
//
|
|
|
|
[Inline]
|
|
private static void LogMemoryOperation(bool isKernelMemory, uint tag, UIntPtr address, UIntPtr size)
|
|
{
|
|
if (CurrentSystemLogger != null) {
|
|
|
|
ushort memoryType = (ushort)(tag & MemoryManager.TypeMask);
|
|
|
|
if (isKernelMemory && ((tag & MemoryManager.TypeMask) == (uint)PageType.Stack)) {
|
|
|
|
memoryType |= 0x8000;
|
|
}
|
|
|
|
CurrentSystemLogger.Log((ushort)((tag & MemoryManager.ProcessPageMask) >> 16),
|
|
memoryType,
|
|
address,
|
|
size);
|
|
}
|
|
}
|
|
|
|
public static void InitializeMemoryMonitoring()
|
|
{
|
|
CurrentSystemLogger = SystemAllocationLogger.Create("SystemMemoryLogger");
|
|
DebugStub.Assert(CurrentSystemLogger != null);
|
|
|
|
FreeMemoryCounters = FreeMemoryDistribution.Create("FreeMemoryCounters", HistogramSize);
|
|
DebugStub.Assert(FreeMemoryCounters != null);
|
|
|
|
InitializeMemoryCounters();
|
|
}
|
|
|
|
[FlatPagesLock]
|
|
// moved into separate function to localize lock auditing
|
|
private static void InitializeMemoryCounters()
|
|
{
|
|
bool iflag = Lock();
|
|
try {
|
|
|
|
for (int i = 0; i < HistogramSize; i++) {
|
|
FreeMemoryCounters.Buffer[i].BlockCount = 0;
|
|
FreeMemoryCounters.Buffer[i].TotalBucketSize = 0;
|
|
}
|
|
|
|
unsafe {
|
|
RegisterExitentFreeNodes(GetFreeList());
|
|
RegisterExitentFreeNodes(GetSaveList());
|
|
}
|
|
}
|
|
finally {
|
|
Unlock(iflag);
|
|
}
|
|
}
|
|
|
|
internal static int HistogramSize;
|
|
internal static int HistogramGranularityShift;
|
|
private static FreeMemoryDistribution FreeMemoryCounters;
|
|
private static SystemAllocationLogger CurrentSystemLogger;
|
|
|
|
[Inline]
|
|
internal static int GetBucketIndex(UIntPtr bytes)
|
|
{
|
|
UIntPtr bucket = bytes >> 20;
|
|
|
|
if (bucket < (UIntPtr)HistogramSize) {
|
|
|
|
return (int)bucket;
|
|
}
|
|
|
|
return (HistogramSize - 1);
|
|
}
|
|
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static void RegisterFreeNode(UIntPtr blockSize)
|
|
{
|
|
if (FreeMemoryCounters != null) {
|
|
|
|
int i = GetBucketIndex(blockSize);
|
|
|
|
FreeMemoryCounters.Buffer[i].BlockCount += 1;
|
|
FreeMemoryCounters.Buffer[i].TotalBucketSize += blockSize;
|
|
}
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static void DeRegisterFreeNode(UIntPtr blockSize)
|
|
{
|
|
if (FreeMemoryCounters != null) {
|
|
|
|
int i = GetBucketIndex(blockSize);
|
|
|
|
DebugStub.Assert((FreeMemoryCounters.Buffer[i].BlockCount > 0),
|
|
"ASSERT: FreeMemoryCounters.BlockCount > 0");
|
|
|
|
DebugStub.Assert((FreeMemoryCounters.Buffer[i].TotalBucketSize >= blockSize),
|
|
"ASSERT: FreeMemoryCounters.TotalBucketSize > blockSize");
|
|
|
|
FreeMemoryCounters.Buffer[i].BlockCount -= 1;
|
|
FreeMemoryCounters.Buffer[i].TotalBucketSize -= blockSize;
|
|
}
|
|
}
|
|
|
|
private static unsafe void RegisterExitentFreeNodes(FreeNode *list)
|
|
{
|
|
for (FreeNode *node = list->next; node != list; node = node->next) {
|
|
|
|
RegisterFreeNode(node->bytes);
|
|
}
|
|
}
|
|
|
|
|
|
//////////////////////////////////////////////////////////////////////
|
|
//
|
|
|
|
[Inline]
|
|
internal static UIntPtr AddrFromPage(UIntPtr page)
|
|
{
|
|
return (page << MemoryManager.PageBits);
|
|
}
|
|
|
|
[Inline]
|
|
private static UIntPtr Align(UIntPtr data, UIntPtr size)
|
|
{
|
|
return ((data) & ~(size - 1));
|
|
}
|
|
|
|
[Inline]
|
|
private static UIntPtr Pad(UIntPtr data, UIntPtr size)
|
|
{
|
|
return ((data + size - 1) & ~(size - 1));
|
|
}
|
|
|
|
[Inline]
|
|
private static UIntPtr SpaceToAlign(UIntPtr data, UIntPtr size)
|
|
{
|
|
return Pad(data, size) - data;
|
|
}
|
|
|
|
[Inline]
|
|
private static UIntPtr SpaceNotAligned(UIntPtr data, UIntPtr size)
|
|
{
|
|
return ((data) & (size - 1));
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////
|
|
//
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
private struct LastNode
|
|
{
|
|
internal const uint Signature = 0xaa2222aa;
|
|
internal const uint Removed = 0xee1111ee;
|
|
|
|
internal uint signature;
|
|
internal unsafe FreeNode * node;
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe LastNode * Create(UIntPtr addr, FreeNode *node)
|
|
{
|
|
LastNode *last = (LastNode *)addr;
|
|
last->signature = LastNode.Signature;
|
|
last->node = node;
|
|
node->last = last;
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug, "addr={0:x8}, node={1:x8}",
|
|
addr, (UIntPtr) last->node);
|
|
#endif
|
|
return last;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe void Remove(LastNode *last)
|
|
{
|
|
last->signature = Removed;
|
|
last->node = null;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe void PrintLastNode(UIntPtr addr)
|
|
{
|
|
LastNode *last = (LastNode *)addr;
|
|
DebugStub.WriteLine("ln.{1:x8} ", __arglist((UIntPtr)last->node));
|
|
}
|
|
|
|
}
|
|
|
|
|
|
//////////////////////////////////////////////////////////////////////
|
|
//
|
|
[StructLayout(LayoutKind.Sequential)]
|
|
private struct FreeNode
|
|
{
|
|
internal const uint Signature = 0x22aaaa22;
|
|
internal const uint Removed = 0x11eeee11;
|
|
|
|
internal uint signature;
|
|
internal unsafe FreeNode * prev;
|
|
internal unsafe FreeNode * next;
|
|
internal unsafe LastNode * last;
|
|
internal UIntPtr bytes;
|
|
internal bool isSave;
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe void Init(FreeNode *list, bool isSave)
|
|
{
|
|
list->signature = Signature;
|
|
list->prev = list;
|
|
list->next = list;
|
|
list->last = null;
|
|
list->bytes = 0;
|
|
list->isSave = isSave;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe bool Remove(FreeNode *node)
|
|
{
|
|
FreeNode * prev;
|
|
FreeNode * next;
|
|
|
|
availableMemory -= node->bytes;
|
|
DeRegisterFreeNode(node->bytes);
|
|
|
|
UIntPtr page = MemoryManager.PageFromAddr((UIntPtr)node);
|
|
*(pageTable + page) = MemoryManager.PageFree;
|
|
|
|
next = node->next;
|
|
prev = node->prev;
|
|
prev->next = next;
|
|
next->prev = prev;
|
|
|
|
if (node->last != null) {
|
|
LastNode.Remove(node->last);
|
|
}
|
|
node->signature = Removed;
|
|
|
|
return (next == prev);
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
private static unsafe void InsertAsPrev(FreeNode *list, FreeNode *node)
|
|
{
|
|
FreeNode * prev;
|
|
|
|
prev = list->prev;
|
|
node->next = list;
|
|
node->prev = prev;
|
|
prev->next = node;
|
|
list->prev = node;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
private static unsafe void InsertAsNext(FreeNode *list, FreeNode *node)
|
|
{
|
|
FreeNode * next;
|
|
|
|
next = list->next;
|
|
node->prev = list;
|
|
node->next = next;
|
|
next->prev = node;
|
|
list->next = node;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
private static unsafe void InsertBySize(FreeNode *list, FreeNode *node)
|
|
{
|
|
#if ALLOW_BOOT_ARGLIST
|
|
DebugStub.Assert(node->bytes > 0,
|
|
"FlatPages.InsertBySize node->bytes={0}",
|
|
__arglist(node->bytes));
|
|
#endif
|
|
if (node->bytes <= SmallSize) {
|
|
// If the size is pretty small, we insert from the back of the list...
|
|
for (FreeNode *step = list->prev; step != list; step = step->prev) {
|
|
if (step->bytes >= node->bytes) {
|
|
InsertAsNext(step, node);
|
|
return;
|
|
}
|
|
}
|
|
InsertAsNext(list, node);
|
|
}
|
|
else {
|
|
// Insert a region into the list by size.
|
|
for (FreeNode *step = list; step->next != list; step = step->next) {
|
|
if (step->next->bytes <= node->bytes) {
|
|
InsertAsNext(step, node);
|
|
return;
|
|
}
|
|
}
|
|
InsertAsPrev(list, node);
|
|
}
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////
|
|
// FreeNode's new routines start here
|
|
|
|
internal static unsafe void PrintFreeList(FreeNode *list)
|
|
{
|
|
int count = 0;
|
|
DebugStub.WriteLine
|
|
(" PRINT FREE LIST (tail.{0:x8} prev.{1:x8} next.{2:x8})",
|
|
__arglist((UIntPtr)(list),
|
|
(UIntPtr)list->prev,
|
|
(UIntPtr)list->next));
|
|
DebugStub.WriteLine(" ---------------------------------------------------");
|
|
|
|
for (FreeNode *node = list->next;
|
|
node != list; node = node->next) {
|
|
DebugStub.Print
|
|
(" [{0}] b.{1:x8} e.{2:x8} {3,8}KB p.{4:x8} n.{5:x8} l.{6:x8} -- ",
|
|
__arglist(
|
|
count,
|
|
(UIntPtr)node, (UIntPtr)node + node->bytes,
|
|
node->bytes/(1024),
|
|
(UIntPtr)node->prev,
|
|
(UIntPtr)node->next,
|
|
(UIntPtr)node->last));
|
|
if (node->last != null) {
|
|
LastNode.PrintLastNode((UIntPtr)(node->last));
|
|
}
|
|
else {
|
|
DebugStub.WriteLine();
|
|
}
|
|
if (count++ > 20) {
|
|
DebugStub.WriteLine("\n **** ERROR INFINITE LIST ****\n");
|
|
DebugStub.Break();
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static unsafe UIntPtr GetFreeListTotalSize(FreeNode *list)
|
|
{
|
|
UIntPtr size = 0;
|
|
for (FreeNode *node = list->next;
|
|
node != list; node = node->next) {
|
|
size += node->bytes;
|
|
}
|
|
return size;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe FreeNode* GetFreeNodeAtBreakAddr(FreeNode *list, UIntPtr breakAddr)
|
|
{
|
|
int count = 0;
|
|
|
|
for (FreeNode *node = list->next;
|
|
node != list; node = node->next) {
|
|
|
|
if ((UIntPtr)node <= breakAddr
|
|
&& breakAddr < ((UIntPtr)node + node->bytes)) {
|
|
return node;
|
|
}
|
|
if (count++ > 20) {
|
|
DebugStub.WriteLine(" WARNING: Can't GetFreeNode ListTail.{0:x8} at {1:x8} after 20 iterations",
|
|
__arglist((UIntPtr)list, breakAddr));
|
|
DebugStub.Break();
|
|
}
|
|
|
|
}
|
|
return null;
|
|
}
|
|
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe FreeNode * FindGoodFit(FreeNode *list,
|
|
UIntPtr bytes, UIntPtr alignment)
|
|
{
|
|
#if ALLOW_BOOT_ARGLIST
|
|
DebugStub.Assert(bytes > 0,
|
|
"FlatPages.FindGoodFit bytes={0}",
|
|
__arglist(bytes));
|
|
#endif
|
|
// If it is a small allocation, we try to accelerate the search.
|
|
if (bytes <= SmallSize && alignment <= MemoryManager.PageSize) {
|
|
for (FreeNode *node = list->prev; node != list; node = node->prev) {
|
|
if (node->bytes >= bytes) {
|
|
Remove(node);
|
|
return node;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
else {
|
|
// First try to find a region closest in size to bytes...
|
|
FreeNode *best = null;
|
|
for (FreeNode *node = list->next; node != list; node = node->next) {
|
|
if (bytes <= node->bytes) {
|
|
UIntPtr full = SpaceToAlign((UIntPtr)node, alignment) + bytes;
|
|
if (full <= node->bytes) {
|
|
// If we find a candidate, remember it.
|
|
best = node;
|
|
if (full == node->bytes) {
|
|
// Stop if it is the ideal region.
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
// Stop if we have a candidate and we've reach smaller regions.
|
|
if (best != null) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (best != null) {
|
|
Remove(best);
|
|
}
|
|
return best;
|
|
}
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe FreeNode * FindBelow(UIntPtr limit, FreeNode *list,
|
|
UIntPtr bytes, UIntPtr alignment)
|
|
{
|
|
DebugStub.Assert(bytes > 0,
|
|
"FlatPages.FindBelow bytes={0}",
|
|
__arglist(bytes));
|
|
|
|
// Try to find the first region below the limit address.
|
|
for (FreeNode *node = list->next; node != list; node = node->next) {
|
|
if ((UIntPtr)node + bytes < limit && node->bytes >= bytes) {
|
|
UIntPtr full = SpaceToAlign((UIntPtr)node, alignment) + bytes;
|
|
if ((UIntPtr)node + full < limit && node->bytes >= full) {
|
|
Remove(node);
|
|
return node;
|
|
}
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe FreeNode * GetNodeAt(UIntPtr addr)
|
|
{
|
|
UIntPtr page = MemoryManager.PageFromAddr(addr);
|
|
|
|
if (*(pageTable + page) == MemoryManager.PageFreeFirst) {
|
|
return (FreeNode *)addr;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe FreeNode * GetNodeFromLast(UIntPtr addr)
|
|
{
|
|
UIntPtr page = MemoryManager.PageFromAddr(addr);
|
|
|
|
if (*(pageTable + page) == MemoryManager.PageFree &&
|
|
*(pageTable + page + 1) != MemoryManager.PageFree) {
|
|
|
|
return ((LastNode *)addr)->node;
|
|
}
|
|
if (*(pageTable + page) == MemoryManager.PageFreeFirst) {
|
|
return (FreeNode *)addr;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe FreeNode * Create(UIntPtr addr, UIntPtr bytes, bool isSave)
|
|
{
|
|
// Mark a page as a node in the free list, initialize the node struct.
|
|
FreeNode * node = (FreeNode *)addr;
|
|
|
|
#if VERY_VERBOSE
|
|
Tracing.Log(Tracing.Debug,
|
|
isSave ?
|
|
"{0:x8}..{1:x8}, last={4:x8}" :
|
|
"{0:x8}..{1:x8}, last={4:x8}",
|
|
addr, addr+bytes, addr + bytes - MemoryManager.PageSize);
|
|
#endif
|
|
|
|
UIntPtr page = MemoryManager.PageFromAddr(addr);
|
|
*(pageTable + page) = MemoryManager.PageFreeFirst;
|
|
|
|
node->signature = FreeNode.Signature;
|
|
node->bytes = bytes;
|
|
node->isSave = isSave;
|
|
node->prev = null;
|
|
node->next = null;
|
|
node->last = null;
|
|
|
|
if (bytes > MemoryManager.PageSize) {
|
|
LastNode.Create(addr + bytes - MemoryManager.PageSize, node);
|
|
}
|
|
return node;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe void CreateAndInsert(FreeNode *list,
|
|
UIntPtr addr,
|
|
UIntPtr bytes)
|
|
{
|
|
FreeNode * node = Create(addr, bytes, list->isSave);
|
|
//
|
|
// This memory is available on the freeList, or the
|
|
// SaveList.
|
|
//
|
|
// Upper level memory lock protects access to this field.
|
|
//
|
|
availableMemory += bytes;
|
|
RegisterFreeNode(bytes);
|
|
#if VERBOSE
|
|
Tracing.Log(Tracing.Debug,
|
|
list->isSave ?
|
|
"({0:x8}, {1:x8}, true), prev={3:x8}, next={4:x8}, last={5:x8}" :
|
|
"({0:x8}, {1:x8}, false), prev={3:x8}, next={4:x8}, last={5:x8}",
|
|
addr, bytes, (UIntPtr) node->prev,
|
|
(UIntPtr) node->next, (UIntPtr) node->last);
|
|
#endif
|
|
|
|
|
|
#if ALLOW_BOOT_ARGLIST
|
|
DebugStub.Assert((bytes & MemoryManager.PageMask) == 0,
|
|
"FlatPages.CreateAndInsert bytes={0}",
|
|
__arglist(bytes));
|
|
DebugStub.Assert((node->bytes & MemoryManager.PageMask) == 0,
|
|
"FlatPages.CreateAndInsert node->bytes={0}",
|
|
__arglist(node->bytes));
|
|
#endif
|
|
InsertBySize(list, node);
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe void ReturnExtraAbove(FreeNode *list,
|
|
UIntPtr addr,
|
|
ref UIntPtr found,
|
|
UIntPtr keep)
|
|
{
|
|
CreateAndInsert(list, addr + keep, found - keep);
|
|
found = keep;
|
|
}
|
|
|
|
[NoStackLinkCheckTrans]
|
|
internal static unsafe void ReturnExtraBelow(FreeNode *list,
|
|
ref UIntPtr addr,
|
|
ref UIntPtr found,
|
|
UIntPtr keep)
|
|
{
|
|
CreateAndInsert(list, addr, found - keep);
|
|
addr = addr + found - keep;
|
|
found = keep;
|
|
}
|
|
}
|
|
}
|
|
}
|