1554 lines
66 KiB
C#
1554 lines
66 KiB
C#
////////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// Microsoft Research Singularity
|
|
//
|
|
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
//
|
|
// File: VMManager.cs -- Virtual Memory Manager
|
|
//
|
|
// Notes:
|
|
//
|
|
// This module manages page-translation tables for virtual memory
|
|
//
|
|
|
|
using System;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Runtime.InteropServices;
|
|
using System.Threading;
|
|
|
|
using Microsoft.Singularity;
|
|
using Microsoft.Singularity.Hal;
|
|
using Microsoft.Singularity.Isal.IX;
|
|
|
|
using PageType = System.GCs.PageType;
|
|
|
|
namespace Microsoft.Singularity.Memory
|
|
{
|
|
[NoCCtor]
|
|
[CLSCompliant(false)]
|
|
[AccessedByRuntime("referenced from halkd.cpp")]
|
|
public class VMManager
|
|
{
|
|
//
|
|
// NOTES
|
|
//
|
|
// Currently, this class is hand-written to deal with 4KB paging
|
|
// on the AMD x64 in 32-bit mode, with PAE enabled. The
|
|
// three-tier structure of the descriptor tables is baked into
|
|
// the code; we will need to factor and expand this module to
|
|
// deal with 64-bit mode, or with a different processor.
|
|
//
|
|
// This code is also written on the assumption that the user/kernel
|
|
// boundary is 1GB-aligned, such that one, two or three of the
|
|
// top-level Page-Directory-Pointer-Table (PDPE) entries fully
|
|
// describe the kernel space.
|
|
//
|
|
// Obviously, memory pages are used to describe the mapping of
|
|
// virtual memory to physical memory. This gets a little confusing.
|
|
// We want there to be a mapping for the pages that contain
|
|
// the page tables (!) so we have virtual addresses we can
|
|
// use to manipulate the page tables!
|
|
//
|
|
// The kernel's page table pages must always be mapped, but the
|
|
// page tables for the user space must get switched when
|
|
// we change address spaces. Therefore, the pages used to
|
|
// describe the mappings for the kernel's space live at a
|
|
// certain virtual address (kernelPageTableBase), and the
|
|
// pages that describe the mapping for user-space live at
|
|
// an address in the user-space (userPageTableBase). This way,
|
|
// the user's page table mappings get swapped along with the
|
|
// rest of the user address space.
|
|
//
|
|
// A minimally usable address space must include a PDPT, and
|
|
// PDTs and PTs sufficient to be able to map in all page-
|
|
// table structures necessary to describe the space, including
|
|
// user-space if the user-space range will be used.
|
|
//
|
|
// Additionally, it is a tricky stunt to set up a new address
|
|
// space when paging is already turned on, since it is difficult
|
|
// to "talk about" the page-table pages for the new space.
|
|
// There are two pages of reserved VM space at scratchAddress1/2
|
|
// that are used to temporarily map in single pages of
|
|
// "foreign" memory so we can build a new address space without
|
|
// having to turn off paging.
|
|
//
|
|
|
|
// These are the PDPT entries for the kernel space. Whether we
|
|
// are using one, two or three of them is determined by
|
|
// Platform.KERNEL_BOUNDARY
|
|
private static PhysicalAddress kernelPDPE1;
|
|
private static PhysicalAddress kernelPDPE2;
|
|
private static PhysicalAddress kernelPDPE3;
|
|
|
|
// How many of the kernelPDPEs are we using?
|
|
private static uint numKernelPDPEs;
|
|
|
|
// The address space we create during boot
|
|
private static AddressSpace BootstrapSpace;
|
|
|
|
// Here is the limit below which memory is identity-mapped
|
|
private static UIntPtr identityMapLimit;
|
|
|
|
// Here is the virtual address where the page-table pages that
|
|
// describe the kernel's address space are always accessible
|
|
private static UIntPtr kernelPageTableBase;
|
|
private static UIntPtr kernelPageTableLimit;
|
|
private static UIntPtr kernelPDTOffset;
|
|
|
|
// Here is the same thing for the user space. This address is
|
|
// above KERNEL_BOUNDARY.
|
|
private static UIntPtr userPageTableBase;
|
|
private static UIntPtr userPageTableLimit;
|
|
private static UIntPtr userPDTOffset;
|
|
|
|
// The PDPT always lives in the user space, so its mapping gets
|
|
// switched naturally when we change spaces
|
|
private unsafe static PDPT* pdptAddr;
|
|
|
|
// Here are two page of VM space we keep available for scratch use.
|
|
// We must take care to use them atomically!
|
|
private static UIntPtr scratchAddress1;
|
|
private static UIntPtr scratchAddress2;
|
|
private static SpinLock scratchSpin;
|
|
|
|
// Here is where we map the uncached-memory window that
|
|
// corresponds to the high-memory range that we use to
|
|
// communicate with hardware (like the APIC)
|
|
private static UIntPtr uncachedWindowBase;
|
|
private static UIntPtr uncachedWindowSize;
|
|
|
|
// Here is where we have situated the bootRecord
|
|
private static UIntPtr bootRecordBase;
|
|
private static UIntPtr bootRecordLimit;
|
|
|
|
// Here is where we have situated the executable image that
|
|
// contains the kernel (where it gets put by the hal loader)
|
|
private static UIntPtr kernelImageBase;
|
|
private static UIntPtr kernelImageLimit;
|
|
|
|
// Here is where we have situated the executable images that
|
|
// are actually in high physical memory (where they get put
|
|
// by the boot loader)
|
|
private static UIntPtr miniDumpBase;
|
|
private static UIntPtr miniDumpLimit;
|
|
|
|
// Here is where we remap areas of physical memory that are
|
|
// reported as reserved by the SMAP (in the order they appear
|
|
// in the SMAP)
|
|
private static UIntPtr smapRemapBase;
|
|
private static UIntPtr smapRemapSize;
|
|
|
|
// Here are the bounds of the VM space that the kernel should use
|
|
// for its GC heap
|
|
private static UIntPtr kernelHeapBase;
|
|
private static UIntPtr kernelHeapLimit;
|
|
|
|
// Same for user-land
|
|
private static UIntPtr userHeapBase;
|
|
private static UIntPtr userHeapLimit;
|
|
|
|
// A protective lock for the kernel's page mappings
|
|
private static SmartSpinlock KernelMappingLock;
|
|
|
|
/////////////////////////////////////
|
|
// CONSTANTS
|
|
/////////////////////////////////////
|
|
|
|
// Each PT describes 512 4K pages
|
|
private const uint SpacePerPT = 4 * 1024 * 512; // 2MB
|
|
|
|
// Each PDT describes 512 PTs
|
|
private const uint SpacePerPDT = 512 * SpacePerPT; // 1GB
|
|
|
|
/////////////////////////////////////
|
|
// PROPERTIES
|
|
/////////////////////////////////////
|
|
|
|
internal static UIntPtr KernelHeapBase {
|
|
get {
|
|
return kernelHeapBase;
|
|
}
|
|
}
|
|
|
|
internal static UIntPtr KernelHeapLimit {
|
|
get {
|
|
return kernelHeapLimit;
|
|
}
|
|
}
|
|
|
|
internal static UIntPtr UserHeapBase {
|
|
get {
|
|
return userHeapBase;
|
|
}
|
|
}
|
|
|
|
internal static UIntPtr UserHeapLimit {
|
|
get {
|
|
return userHeapLimit;
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////////
|
|
// PUBLIC METHODS
|
|
/////////////////////////////////////
|
|
|
|
//
|
|
// This must be called with paging TURNED OFF, and it has the
|
|
// side effect of TURNING PAGING ON!
|
|
//
|
|
|
|
//
|
|
// The virtual address map used by VMManager is set up in Initialize
|
|
//
|
|
// Part of the layout is inheirated from MemoryManager who (ignoring SMAP
|
|
// complications) has set up a range for the pageBlocks and a physicalHeap
|
|
// for I/O memory below the 0xFFFFFF threshold. This function picks up
|
|
// from there and lays out the rest of the virtual address space.
|
|
//
|
|
// TODO: Create region structure and use it here to interleave fixed address
|
|
// blocks and SMAP blocks
|
|
// TODO: Better checks for overlapping range requests
|
|
|
|
internal static unsafe void Initialize()
|
|
{
|
|
Platform p = Platform.ThePlatform;
|
|
uint numKernelPTs = Platform.KERNEL_BOUNDARY / SpacePerPT;
|
|
UIntPtr userSpaceSize = Platform.MAX_VIRTUAL_ADDR - Platform.KERNEL_BOUNDARY + 1;
|
|
UIntPtr nextVirtualAddr = MemoryManager.IOMemoryBase.Value + Platform.IO_MEMORY_SIZE;
|
|
|
|
kernelPDTOffset = MemoryManager.BytesFromPages(numKernelPTs);
|
|
numKernelPDPEs = (uint)(Platform.KERNEL_BOUNDARY / SpacePerPDT);
|
|
scratchSpin = new SpinLock(SpinLock.Types.VMManager);
|
|
|
|
// How big is the kernel space?
|
|
DebugStub.Assert(MemoryManager.BytesNotAligned(Platform.KERNEL_BOUNDARY,SpacePerPDT) == 0);
|
|
DebugStub.Assert(numKernelPDPEs <= 3);
|
|
|
|
// First off locate the kernel and bootRecord which must live
|
|
// in identity mapped memory
|
|
if (p.KernelDllSize != 0) {
|
|
kernelImageBase = MemoryManager.PagePad(p.KernelDllBase);
|
|
kernelImageLimit = MemoryManager.PagePad(p.KernelDllBase + p.KernelDllSize);
|
|
|
|
DebugStub.WriteLine("Executable Image at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(kernelImageBase,
|
|
kernelImageLimit));
|
|
|
|
if (nextVirtualAddr < kernelImageLimit) {
|
|
nextVirtualAddr = kernelImageLimit;
|
|
}
|
|
}
|
|
|
|
if (p.BootAllocatedMemorySize != 0) {
|
|
bootRecordBase = MemoryManager.PagePad(p.BootAllocatedMemory);
|
|
bootRecordLimit = MemoryManager.PagePad(
|
|
p.BootAllocatedMemory + p.BootAllocatedMemorySize);
|
|
|
|
DebugStub.WriteLine("BootRecord at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(bootRecordBase,
|
|
bootRecordLimit));
|
|
|
|
if (nextVirtualAddr < bootRecordLimit) {
|
|
nextVirtualAddr = bootRecordLimit;
|
|
}
|
|
}
|
|
|
|
// Now figure out where the kernel page-table pages will live.
|
|
// We'll stick it immediately above the identity mapped range, on
|
|
// a page boundary.
|
|
identityMapLimit = MemoryManager.PagePad(nextVirtualAddr);
|
|
|
|
// numKernelPDPEs == the number of PDTs, by definition
|
|
kernelPageTableBase = nextVirtualAddr;
|
|
nextVirtualAddr =
|
|
kernelPageTableLimit = MemoryManager.PagePad(kernelPageTableBase
|
|
+ MemoryManager.BytesFromPages(numKernelPTs + numKernelPDPEs));
|
|
|
|
DebugStub.WriteLine("KernelPageTable initialized at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(kernelPageTableBase,
|
|
kernelPageTableLimit));
|
|
|
|
// Locate the uncached window immediately after the kernel's page-table space
|
|
uncachedWindowBase = nextVirtualAddr;
|
|
uncachedWindowSize = Platform.UNCACHED_MAPPED;
|
|
nextVirtualAddr = MemoryManager.PagePad(uncachedWindowBase
|
|
+ uncachedWindowSize);
|
|
|
|
DebugStub.WriteLine("Uncached Window at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(uncachedWindowBase,
|
|
uncachedWindowBase + uncachedWindowSize));
|
|
|
|
// Now accomidate the minidump containing the executable cache.
|
|
if (p.MiniDumpBase != p.MiniDumpLimit) {
|
|
miniDumpBase = nextVirtualAddr;
|
|
nextVirtualAddr =
|
|
miniDumpLimit = MemoryManager.PagePad(miniDumpBase + p.MiniDumpLimit - p.MiniDumpBase);
|
|
|
|
DebugStub.WriteLine("MiniDump at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(miniDumpBase,
|
|
miniDumpLimit));
|
|
}
|
|
|
|
// Locate the SMAP window next
|
|
smapRemapBase = nextVirtualAddr;
|
|
smapRemapSize = 0;
|
|
|
|
// Calculate its size
|
|
SMAPINFO* smap = (SMAPINFO*)p.Smap32;
|
|
for (uint i = 0; i < p.SmapCount; i++) {
|
|
if ((smap[i].type != (ulong)SMAPINFO.AddressType.Free) &&
|
|
(smap[i].addr >= identityMapLimit)) {
|
|
smapRemapSize += smap[i].size;
|
|
}
|
|
}
|
|
if (smapRemapSize != 0) {
|
|
DebugStub.WriteLine("SMAP reserve at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(smapRemapBase,
|
|
smapRemapBase+smapRemapSize));
|
|
|
|
nextVirtualAddr = MemoryManager.PagePad(smapRemapBase
|
|
+ smapRemapSize);
|
|
}
|
|
|
|
// Locate the one-page scratch window immediately above the smap area
|
|
scratchAddress1 = nextVirtualAddr;
|
|
scratchAddress2 = scratchAddress1 + MemoryManager.PageSize;
|
|
nextVirtualAddr = scratchAddress2 + MemoryManager.PageSize;
|
|
|
|
DebugStub.WriteLine("scratch#1 at 0x{0:x8}, scratch#2 at 0x{1:x8}.",
|
|
__arglist(scratchAddress1,
|
|
scratchAddress2));
|
|
|
|
// That's all for our fixed low-memory structures
|
|
kernelHeapBase = nextVirtualAddr;
|
|
kernelHeapLimit = Platform.KERNEL_BOUNDARY;
|
|
DebugStub.Assert(kernelHeapLimit > kernelHeapBase);
|
|
|
|
DebugStub.WriteLine("KernelHeap initialized at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(kernelHeapBase,
|
|
kernelHeapLimit));
|
|
|
|
// Stick the user-space page-table mapping right at the
|
|
// beginning of the user's address space
|
|
userPageTableBase = Platform.KERNEL_BOUNDARY;
|
|
uint numPTs = (uint)(userSpaceSize / SpacePerPT);
|
|
uint numPDTs = (uint)(userSpaceSize / SpacePerPDT);
|
|
userPDTOffset = MemoryManager.BytesFromPages(numPTs);
|
|
pdptAddr = (PDPT*)(userPageTableBase + MemoryManager.BytesFromPages(numPTs + numPDTs));
|
|
|
|
// Count the PDPT as part of the user's page table so we don't
|
|
// step on the PDPT mapping later
|
|
userPageTableLimit = (UIntPtr)pdptAddr + MemoryManager.PageSize;
|
|
|
|
DebugStub.WriteLine("UserPageTable initialized at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(userPageTableBase,
|
|
userPageTableLimit));
|
|
|
|
// Define the limits of general-purpose user space
|
|
userHeapBase = userPageTableLimit;
|
|
userHeapLimit = Platform.MAX_VIRTUAL_ADDR;
|
|
userHeapLimit = MemoryManager.Trunc(userHeapLimit, MemoryManager.PageSize);
|
|
|
|
DebugStub.WriteLine("UserHeap initialized at 0x{0:x8} - 0x{1:x8}.",
|
|
__arglist(userHeapBase,
|
|
userHeapLimit));
|
|
|
|
// Now we can create the kernel's address space!
|
|
BootstrapSpace = CreateBootstrapSpace();
|
|
|
|
// TURN ON PAGING!
|
|
//
|
|
// Save the kernel's operating Pdpt value so that non-boot CPU's
|
|
// may have the proper map set by the boot loader in undump.cpp
|
|
//
|
|
Platform nativePlatform = Platform.ThePlatform;
|
|
// temporary workaround for Bartok bug
|
|
AddressSpace b = BootstrapSpace;
|
|
nativePlatform.KernelPdpt64 = b.PdptPage.Value;
|
|
|
|
Processor.EnablePaging(BootstrapSpace);
|
|
|
|
#if DEBUG
|
|
// Sanity checks
|
|
DebugStub.Assert(IsUserDescriptorPageMapped((UIntPtr)pdptAddr));
|
|
DebugStub.Assert(((*pdptAddr)[0])->Valid);
|
|
|
|
// We expect there to be sufficient PTs to describe the entire
|
|
// kernel page table
|
|
for (UIntPtr step = MemoryManager.Trunc(kernelPageTableBase, SpacePerPT);
|
|
step < MemoryManager.Pad(kernelPageTableLimit, SpacePerPT);
|
|
step += SpacePerPT) {
|
|
|
|
PT* pt = GetPT(step);
|
|
DebugStub.Assert(IsKernelDescriptorPageMapped((UIntPtr)pt));
|
|
}
|
|
|
|
// Obviously, the PDT for this space should be mapped too
|
|
DebugStub.Assert(
|
|
IsKernelDescriptorPageMapped(
|
|
(UIntPtr)GetPDT(
|
|
MemoryManager.Trunc(kernelPageTableBase, SpacePerPDT))));
|
|
|
|
// Same checks for user-land
|
|
for (UIntPtr step = MemoryManager.Trunc(userPageTableBase, SpacePerPT);
|
|
step < MemoryManager.Pad(userPageTableLimit, SpacePerPT);
|
|
step += SpacePerPT) {
|
|
|
|
PT* pt = GetPT(step);
|
|
DebugStub.Assert(IsUserDescriptorPageMapped((UIntPtr)pt));
|
|
}
|
|
|
|
DebugStub.Assert(
|
|
IsUserDescriptorPageMapped(
|
|
(UIntPtr)GetPDT(
|
|
MemoryManager.Trunc(userPageTableBase, SpacePerPDT))));
|
|
#endif
|
|
|
|
DebugStub.WriteLine("VMManager initialized with {0} physical pages still free",
|
|
__arglist(PhysicalPages.GetFreePageCount()));
|
|
}
|
|
|
|
internal static void Finalize()
|
|
{
|
|
Platform nativePlatform = Platform.ThePlatform;
|
|
if (nativePlatform != null) {
|
|
// Restore the PDPT that the boot loader created
|
|
Processor.ChangeAddressSpace(new AddressSpace(new PhysicalAddress(nativePlatform.Pdpt32)));
|
|
}
|
|
}
|
|
|
|
internal static void PostGCInitialize()
|
|
{
|
|
// Create a spinlock to protect the kernel's page-mapping
|
|
// structures
|
|
KernelMappingLock = new SmartSpinlock(SpinLock.Types.VMKernelMapping);
|
|
}
|
|
|
|
//
|
|
// This gets called during the boot sequence when we're ready to
|
|
// start using user-space for the first time. We flesh out our
|
|
// bootstrap address space with a user range and hand it out for the
|
|
// first time.
|
|
//
|
|
internal static AddressSpace GetBootstrapSpace()
|
|
{
|
|
return BootstrapSpace;
|
|
}
|
|
|
|
//
|
|
// Call to create a whole new address space from scratch
|
|
//
|
|
public static AddressSpace CreateNewAddressSpace()
|
|
{
|
|
AddressSpace retval = CreateNewKernelSpace();
|
|
InitializeUserRange(retval);
|
|
return retval;
|
|
}
|
|
|
|
//
|
|
// Writes a new virtual -> physical address into the page table,
|
|
// ensuring all necessary PDTs and PTs are allocated and mapped
|
|
//
|
|
// The caller must specify the protection domain that owns the mapped
|
|
// pages, for locking purposes. When mapping pages into the kernel
|
|
// range this parameter can be null.
|
|
//
|
|
// What are the "locking purposes"? Routines that may *add* nodes
|
|
// to the page-mapping tree need to be protected against concurrency.
|
|
//
|
|
public static unsafe void MapPage(PhysicalAddress physAddr,
|
|
UIntPtr virtualAddr,
|
|
ProtectionDomain inDomain)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(virtualAddr));
|
|
DebugStub.Assert(!IsPageMapped(virtualAddr), "Attempt to map over a mapped page");
|
|
|
|
SmartSpinlock mappingLock = null;
|
|
|
|
if (virtualAddr < Platform.KERNEL_BOUNDARY) {
|
|
mappingLock = KernelMappingLock; // Might be null during early boot
|
|
}
|
|
else {
|
|
DebugStub.Assert(inDomain != null);
|
|
mappingLock = inDomain.UserMappingLock;
|
|
DebugStub.Assert(mappingLock != null);
|
|
DebugStub.Assert(inDomain.AddressSpace == Processor.GetCurrentAddressSpace());
|
|
}
|
|
|
|
bool iflag = false;
|
|
|
|
if (mappingLock != null) {
|
|
iflag = mappingLock.Lock();
|
|
}
|
|
|
|
try {
|
|
MapPageNoLock(physAddr, virtualAddr);
|
|
}
|
|
finally {
|
|
if (mappingLock != null) {
|
|
mappingLock.Unlock(iflag);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static unsafe void MapPageNoLock(PhysicalAddress physAddr,
|
|
UIntPtr virtualAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(virtualAddr));
|
|
DebugStub.Assert(!IsPageMapped(virtualAddr), "Attempt to map over a mapped page");
|
|
EnsurePTE(virtualAddr);
|
|
MapPageInternal(physAddr, virtualAddr);
|
|
}
|
|
|
|
//
|
|
// The caller must specify the protection domain that owns the mapped
|
|
// pages, for locking purposes. When mapping pages into the kernel
|
|
// range this parameter can be null.
|
|
//
|
|
// What are the "locking purposes"? Routines that may *add* nodes
|
|
// to the page-mapping tree need to be protected against concurrency.
|
|
//
|
|
public static unsafe void MapRange(PhysicalAddress physAddr,
|
|
UIntPtr virtualAddr,
|
|
UIntPtr numPages,
|
|
ProtectionDomain inDomain)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(virtualAddr));
|
|
ulong stepPhys = physAddr.Value;
|
|
UIntPtr stepVirt = virtualAddr;
|
|
UIntPtr limitAddr = virtualAddr + MemoryManager.BytesFromPages(numPages);
|
|
|
|
SmartSpinlock mappingLock = null;
|
|
|
|
if (limitAddr <= Platform.KERNEL_BOUNDARY) {
|
|
mappingLock = KernelMappingLock; // Might be null during early boot
|
|
}
|
|
else {
|
|
DebugStub.Assert(inDomain != null);
|
|
mappingLock = inDomain.UserMappingLock;
|
|
DebugStub.Assert(mappingLock != null);
|
|
DebugStub.Assert(inDomain.AddressSpace == Processor.GetCurrentAddressSpace());
|
|
}
|
|
|
|
bool iflag = false;
|
|
|
|
if (mappingLock != null) {
|
|
iflag = mappingLock.Lock();
|
|
}
|
|
|
|
try {
|
|
for (UIntPtr i = 0; i < numPages; i++) {
|
|
MapPageNoLock(new PhysicalAddress(stepPhys), stepVirt);
|
|
stepPhys += MemoryManager.PageSize;
|
|
stepVirt += MemoryManager.PageSize;
|
|
}
|
|
}
|
|
finally {
|
|
if (mappingLock != null) {
|
|
mappingLock.Unlock(iflag);
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Unmaps an existing virtual-memory page.
|
|
// Does NOT release the underlying physical page
|
|
// (whose address is returned)
|
|
//
|
|
public static unsafe PhysicalAddress UnmapPage(UIntPtr virtualAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(virtualAddr));
|
|
PTE* pte = GetPTE(virtualAddr);
|
|
|
|
if (pte->Valid) {
|
|
PhysicalAddress phys = new PhysicalAddress(pte->Address);
|
|
pte->Valid = false;
|
|
pte->Address = 0;
|
|
Processor.InvalidateTLBEntry(virtualAddr);
|
|
return phys;
|
|
}
|
|
else {
|
|
DebugStub.Assert(false, "Attempt to unmap an unmapped page");
|
|
return PhysicalAddress.Null;
|
|
}
|
|
}
|
|
|
|
public static unsafe void UnmapRange(UIntPtr virtualAddr,
|
|
UIntPtr numPages)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(virtualAddr));
|
|
UIntPtr stepVirt = virtualAddr;
|
|
|
|
for (UIntPtr i = 0; i < numPages; i++) {
|
|
UnmapPage(stepVirt);
|
|
stepVirt += MemoryManager.PageSize;
|
|
}
|
|
}
|
|
|
|
//
|
|
// Returns the physical address that a given virtual address is
|
|
// currently mapped to (or PhysicalAddress.Null if none)
|
|
//
|
|
public static unsafe PhysicalAddress GetPageMapping(UIntPtr virtualAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(virtualAddr));
|
|
PTE* pte = GetPTE(virtualAddr);
|
|
|
|
if (pte->Valid) {
|
|
return new PhysicalAddress(pte->Address);
|
|
}
|
|
else {
|
|
return PhysicalAddress.Null;
|
|
}
|
|
}
|
|
|
|
// Examines the page-table structures to determine whether the page
|
|
// at pageAddr is mapped or not.
|
|
//
|
|
// REVIEW: use a processor instruction instead to get a more definitive
|
|
// answer?
|
|
[AccessedByRuntime("referenced from halkd.cpp")]
|
|
public static unsafe bool IsPageMapped(UIntPtr pageAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(pageAddr));
|
|
|
|
if ((pageAddr >= kernelPageTableBase) &&
|
|
(pageAddr < kernelPageTableLimit)) {
|
|
// This is the address of a kernel page-table page.
|
|
// There should always be sufficient PTs to describe these.
|
|
return IsKernelDescriptorPageMapped(pageAddr);
|
|
}
|
|
else if ((pageAddr >= userPageTableBase) &&
|
|
(pageAddr < userPageTableLimit)) {
|
|
// User-land page-table page
|
|
return IsUserDescriptorPageMapped(pageAddr);
|
|
}
|
|
else {
|
|
// This page isn't part of the page table, so its
|
|
// PT may be missing. Check that.
|
|
PT* pt = GetPT(MemoryManager.Trunc(pageAddr, SpacePerPT));
|
|
|
|
if (pageAddr >= Platform.KERNEL_BOUNDARY) {
|
|
if (!IsUserDescriptorPageMapped((UIntPtr)pt)) {
|
|
// The PT for this address isn't even mapped,
|
|
// so the address itself can't be mapped!
|
|
return false;
|
|
}
|
|
}
|
|
else {
|
|
if (!IsKernelDescriptorPageMapped((UIntPtr)pt)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// There's a PT but it might say that this page isn't
|
|
// mapped.
|
|
PTE* pte = GetPTE(pageAddr);
|
|
return pte->Valid;
|
|
}
|
|
}
|
|
|
|
//
|
|
// A complement to TranslatePhysicalRange. Returns true if
|
|
// the virtual address provided falls into one of the
|
|
// virtual ranges we have reserved for permanent mappings
|
|
// of hardware.
|
|
//
|
|
internal static unsafe bool IsPermaMapped(UIntPtr virtualAddr,
|
|
UIntPtr limitAddr)
|
|
{
|
|
if (limitAddr <= identityMapLimit) {
|
|
return true;
|
|
}
|
|
|
|
if ((virtualAddr >= smapRemapBase) &&
|
|
(limitAddr <= smapRemapBase + smapRemapSize)) {
|
|
return true;
|
|
}
|
|
|
|
if ((virtualAddr >= uncachedWindowBase) &&
|
|
(limitAddr <= uncachedWindowBase + uncachedWindowSize)) {
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
//
|
|
// This call returns non-null if the physical address provided
|
|
// is already well-known and mapped into a stable area of
|
|
// memory.
|
|
//
|
|
internal static unsafe UIntPtr TranslatePhysicalRange(PhysicalAddress physAddress,
|
|
UIntPtr numBytes)
|
|
{
|
|
Platform p = Platform.ThePlatform;
|
|
ulong physAddr = physAddress.Value;
|
|
ulong physLimit = physAddr + (ulong)numBytes;
|
|
|
|
// Some physical pointers are identity-mapped
|
|
if (physLimit <= identityMapLimit) {
|
|
return physAddr;
|
|
}
|
|
|
|
// Is this an address into the executable-image range
|
|
// created by the loader?
|
|
if ((physAddr >= p.MiniDumpBase) && (physLimit <= p.MiniDumpLimit)) {
|
|
// Assume we move the exec image lower in memory
|
|
DebugStub.Assert(p.MiniDumpBase > miniDumpBase);
|
|
return (UIntPtr)(physAddr - (Platform.ThePlatform.MiniDumpBase - miniDumpBase));
|
|
}
|
|
|
|
// Is this a pointer into the uncached window at the
|
|
// top of physical memory?
|
|
ulong uncachedLimit = (ulong)Platform.UNCACHED_PHYSICAL +
|
|
(ulong)Platform.UNCACHED_MAPPED;
|
|
|
|
if ((physAddr >= Platform.UNCACHED_PHYSICAL) &&
|
|
(physLimit <= uncachedLimit)) {
|
|
return uncachedWindowBase + (UIntPtr)(physAddr - Platform.UNCACHED_PHYSICAL);
|
|
}
|
|
|
|
// Is this an address into an SMAP-reserved area?
|
|
SMAPINFO *smap = (SMAPINFO*)p.Smap32;
|
|
UIntPtr stepVirtual = smapRemapBase;
|
|
|
|
for (uint i = 0; i < p.SmapCount; i++) {
|
|
if ((smap[i].type != (ulong)SMAPINFO.AddressType.Free) &&
|
|
(smap[i].addr >= identityMapLimit)) {
|
|
|
|
if ((physAddr >= smap[i].addr) &&
|
|
(physLimit <= smap[i].addr + smap[i].size)) {
|
|
// Match!
|
|
return stepVirtual + (physAddr - smap[i].addr);
|
|
}
|
|
|
|
stepVirtual += smap[i].size;
|
|
}
|
|
}
|
|
|
|
// No match
|
|
return UIntPtr.Zero;
|
|
}
|
|
|
|
//
|
|
// Finds a range of virtual space in the provided address range
|
|
// and maps in the requested physical range on the fly
|
|
//
|
|
internal static UIntPtr MapPhysicalMemory(VirtualMemoryRange range,
|
|
Process process,
|
|
PhysicalAddress physStart,
|
|
UIntPtr numBytes)
|
|
{
|
|
ulong startPhysPage = MemoryManager.Trunc(physStart.Value, MemoryManager.PageSize);
|
|
ulong physLimit = MemoryManager.PagePad(physStart.Value + (ulong)numBytes);
|
|
UIntPtr numPages = MemoryManager.PagesFromBytes((UIntPtr)(physLimit - startPhysPage));
|
|
|
|
// Reserve an area to accomplish the mapping in
|
|
UIntPtr mapPoint = range.Reserve(numPages, process, 0, PageType.System);
|
|
|
|
DebugStub.Assert(mapPoint != UIntPtr.Zero,
|
|
"Out of virtual memory trying to map a physical range");
|
|
|
|
// Perform the mapping
|
|
MapRange(new PhysicalAddress(startPhysPage), mapPoint, numPages, range.ParentDomain);
|
|
|
|
// Take care to point into the mapped virtual memory properly
|
|
return mapPoint + (UIntPtr)MemoryManager.BytesNotAligned(
|
|
physStart.Value, MemoryManager.PageSize);
|
|
}
|
|
|
|
//
|
|
// Reverses the operation of MapPhysicalMemory
|
|
//
|
|
internal static void UnmapPhysicalMemory(VirtualMemoryRange range,
|
|
Process process,
|
|
UIntPtr startAddr,
|
|
UIntPtr limitAddr)
|
|
{
|
|
UIntPtr startPageAddr = MemoryManager.Trunc(startAddr, MemoryManager.PageSize);
|
|
UIntPtr limitPageAddr = MemoryManager.PagePad(limitAddr);
|
|
uint numPages = (uint)MemoryManager.PagesFromBytes(limitPageAddr - startPageAddr);
|
|
|
|
// First undo the mapping
|
|
UnmapRange(startPageAddr, numPages);
|
|
|
|
// Now mark all the pages as free
|
|
range.Unreserve(startPageAddr, numPages, process);
|
|
}
|
|
|
|
/////////////////////////////////////
|
|
// PRIVATE BOOTSTRAP METHODS
|
|
/////////////////////////////////////
|
|
|
|
// This function sets up the initial memory space from scratch.
|
|
// It assumes paging is NOT on.
|
|
private static unsafe AddressSpace CreateBootstrapSpace()
|
|
{
|
|
//
|
|
// The kernel's address space serves as the template for all other
|
|
// address spaces, because everything it maps must also be mapped
|
|
// in other spaces.
|
|
//
|
|
// We need mapping for a bunch of things before we can turn on
|
|
// paging. Here's the list:
|
|
//
|
|
// BELOW KERNEL_BOUNDARY:
|
|
//
|
|
// Zero through BootInfo.PHYSICAL_DISABLED is always kept unmapped
|
|
//
|
|
// Memory from PhysicalBase to the top of the I/O heap is
|
|
// identity-mapped (we could probably stand to punch some holes
|
|
// in this).
|
|
//
|
|
// The kernel image and the boot record need to be identity mapped.
|
|
//
|
|
// Next comes the page-table area, starting at kernelPageTableBase.
|
|
// It is large enough to cover all the pages that could be
|
|
// needed to describe the kernel's address space.
|
|
//
|
|
// Next comes a window that maps the executable images downloaded
|
|
// by the boot loader (these usually get stuck in high memory)
|
|
// into the kernel's space, which is a more convenient location.
|
|
//
|
|
// Mappings above KERNEL_BOUNDARY are free for use by processes,
|
|
// except that from UNCACHED_PHYSICAL to the top (4GB) is marked
|
|
// uncached.
|
|
//
|
|
Platform p = Platform.ThePlatform;
|
|
|
|
// To get started, we need a PDPT.
|
|
PhysicalAddress phys = PhysicalPages.AllocPage();
|
|
DebugStub.Assert(phys != PhysicalAddress.Null);
|
|
PDPT* pdpt = (PDPT*)phys.Value;
|
|
pdpt->Init();
|
|
|
|
DebugStub.WriteLine("Allocated PDPT at 0x{0:x8}.", __arglist(phys.Value));
|
|
|
|
// Start out mapping the entire identityMapped region as writable, not writethrough,
|
|
// with caching enabled
|
|
// TODO: Can't use p.PhysicalBase because hypervisor is currently using pages below
|
|
// that threshold. s/b fix hypervisor PhysicalBase reporting.
|
|
BootstrapMapRange(pdpt, 0,
|
|
identityMapLimit,
|
|
new PhysicalAddress(0),
|
|
true, // writeable
|
|
false, // not write-through
|
|
false); // don't disable caching
|
|
// Map the uncached area of high memory down into the kernel space
|
|
BootstrapMapRange(pdpt, uncachedWindowBase,
|
|
MemoryManager.PagePad(uncachedWindowBase + uncachedWindowSize),
|
|
new PhysicalAddress(Platform.UNCACHED_PHYSICAL),
|
|
true, // writeable
|
|
true, // writethrough
|
|
true); // disable caching
|
|
BootstrapMapRange(pdpt, miniDumpBase,
|
|
MemoryManager.PagePad(miniDumpLimit),
|
|
new PhysicalAddress(p.MiniDumpBase),
|
|
true, // writeable
|
|
false, // not writethrough
|
|
false); // don't disable caching
|
|
|
|
// Walk the SMAP and remap any reserved areas into the smap window
|
|
SMAPINFO* smap = (SMAPINFO*)p.Smap32;
|
|
UIntPtr stepVirtual = smapRemapBase;
|
|
|
|
for (uint i = 0; i < p.SmapCount; i++) {
|
|
if ((smap[i].type != (ulong)SMAPINFO.AddressType.Free) &&
|
|
(smap[i].addr >= identityMapLimit)) {
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(smap[i].size));
|
|
BootstrapMapRange(pdpt, stepVirtual, stepVirtual + smap[i].size,
|
|
new PhysicalAddress(smap[i].addr),
|
|
true, // writeable
|
|
true, // write-through,
|
|
true); // disable caching
|
|
stepVirtual += smap[i].size;
|
|
}
|
|
}
|
|
|
|
DebugStub.Assert((stepVirtual - smapRemapBase) == smapRemapSize);
|
|
|
|
// Make sure we have committed a PDT for each kernel PDPE slot, so
|
|
// other address spaces can alias the kernel range sensibly
|
|
BootstrapEnsurePDTs(pdpt, UIntPtr.Zero, Platform.KERNEL_BOUNDARY);
|
|
|
|
kernelPDPE1 = new PhysicalAddress(((*pdpt)[0])->Address);
|
|
DebugStub.Assert(kernelPDPE1 != PhysicalAddress.Null);
|
|
|
|
if (numKernelPDPEs >= 2) {
|
|
kernelPDPE2 = new PhysicalAddress(((*pdpt)[1])->Address);
|
|
DebugStub.Assert(kernelPDPE2 != PhysicalAddress.Null);
|
|
}
|
|
|
|
if (numKernelPDPEs >= 3) {
|
|
kernelPDPE3 = new PhysicalAddress(((*pdpt)[2])->Address);
|
|
DebugStub.Assert(kernelPDPE3 != PhysicalAddress.Null);
|
|
}
|
|
|
|
// Create the mappings for all the page-table structures
|
|
// we just set up so we can still manipulate them after
|
|
// we turn on paging!
|
|
BootstrapMapPageTable(pdpt);
|
|
|
|
return new AddressSpace(new PhysicalAddress((ulong)pdpt));
|
|
}
|
|
|
|
// For use during startup. Creates a mapping for all the physical
|
|
// pages currently being used to describe memory mappings (yes,
|
|
// this is confusing).
|
|
//
|
|
// Assumes paging is NOT on so that physical addresses can be
|
|
// used as-is
|
|
private static unsafe void BootstrapMapPageTable(PDPT* pdpt)
|
|
{
|
|
BootstrapEnsurePTs(pdpt, kernelPageTableBase, kernelPageTableLimit);
|
|
BootstrapEnsurePTs(pdpt, userPageTableBase, userPageTableLimit);
|
|
|
|
BootstrapMapPage(pdpt, (UIntPtr)pdptAddr, new PhysicalAddress((ulong)pdpt), true);
|
|
|
|
// First map the kernel page table pages
|
|
for (uint i = 0; i < numKernelPDPEs; i++) {
|
|
if (((*pdpt)[i])->Valid) {
|
|
PDT* pdt = (PDT*)(((*pdpt)[i])->Address);
|
|
|
|
BootstrapMapPage(pdpt,
|
|
kernelPageTableBase + kernelPDTOffset + MemoryManager.BytesFromPages(i),
|
|
new PhysicalAddress((ulong)pdt), true);
|
|
|
|
for (uint j = 0; j < 512; j++) {
|
|
if (((*pdt)[j])->Valid) {
|
|
PT* pt = (PT*)(((*pdt)[j])->Address);
|
|
|
|
BootstrapMapPage(pdpt,
|
|
kernelPageTableBase + MemoryManager.BytesFromPages((i * 512) + j),
|
|
new PhysicalAddress((ulong)pt), true);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Now map user-space page-table pages
|
|
for (uint i = 0; i < (4 - numKernelPDPEs); i++) {
|
|
if (((*pdpt)[numKernelPDPEs + i])->Valid) {
|
|
PDT* pdt = (PDT*)(((*pdpt)[numKernelPDPEs + i])->Address);
|
|
|
|
BootstrapMapPage(pdpt,
|
|
userPageTableBase + userPDTOffset + MemoryManager.BytesFromPages(i),
|
|
new PhysicalAddress((ulong)pdt), true);
|
|
|
|
for (uint j = 0; j < 512; j++) {
|
|
if (((*pdt)[j])->Valid) {
|
|
PT* pt = (PT*)(((*pdt)[j])->Address);
|
|
|
|
BootstrapMapPage(pdpt,
|
|
userPageTableBase + MemoryManager.BytesFromPages((i * 512) + j),
|
|
new PhysicalAddress((ulong)pt), true);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// For use during startup. Makes sure the given PDPT points to
|
|
// enough PDTs (which are allocated as necessary) to describe the
|
|
// address range specified.
|
|
//
|
|
// Assumes paging is NOT on.
|
|
private static unsafe void BootstrapEnsurePDTs(PDPT* pdpt,
|
|
UIntPtr startAddr,
|
|
UIntPtr limitAddr)
|
|
{
|
|
limitAddr = MemoryManager.Pad(limitAddr, SpacePerPDT);
|
|
uint stepPDTidx = (uint)(startAddr / SpacePerPDT);
|
|
UIntPtr stepAddr = startAddr;
|
|
|
|
while (stepAddr < limitAddr) {
|
|
if (!((*pdpt)[stepPDTidx])->Valid) {
|
|
PhysicalAddress newPDT = PhysicalPages.AllocPage();
|
|
((PDT*)newPDT.Value)->Init();
|
|
((*pdpt)[stepPDTidx])->Address = newPDT.Value;
|
|
((*pdpt)[stepPDTidx])->Valid = true;
|
|
}
|
|
|
|
stepPDTidx++;
|
|
stepAddr += SpacePerPDT;
|
|
}
|
|
}
|
|
|
|
|
|
// For use during startup. Makes sure there are enough PTs
|
|
// (which are allocated as necessary) to describe the address
|
|
// range specified.
|
|
//
|
|
// Assumes paging is NOT on.
|
|
private static unsafe void BootstrapEnsurePTs(PDPT* pdpt,
|
|
UIntPtr startAddr,
|
|
UIntPtr limitAddr)
|
|
{
|
|
limitAddr = MemoryManager.Pad(limitAddr, SpacePerPT);
|
|
BootstrapEnsurePDTs(pdpt, startAddr, limitAddr);
|
|
|
|
uint stepPDTidx = (uint)(startAddr / SpacePerPDT);
|
|
UIntPtr stepAddr = startAddr;
|
|
|
|
while (stepAddr < limitAddr) {
|
|
DebugStub.Assert(((*pdpt)[stepPDTidx])->Valid); // ensured above
|
|
PDT* pdt = (PDT*)(((*pdpt)[stepPDTidx])->Address);
|
|
UIntPtr offsetInPDT = MemoryManager.BytesNotAligned(stepAddr, SpacePerPDT);
|
|
|
|
uint stepPTidx = (uint)(offsetInPDT / SpacePerPT);
|
|
|
|
while ((stepAddr < limitAddr) && (stepPTidx < 512)) {
|
|
if (!((*pdt)[stepPTidx])->Valid) {
|
|
PhysicalAddress newPT = PhysicalPages.AllocPage();
|
|
((PT*)newPT.Value)->Init();
|
|
((*pdt)[stepPTidx])->Address = newPT.Value;
|
|
((*pdt)[stepPTidx])->User = true; // Ring3
|
|
((*pdt)[stepPTidx])->Valid = true;
|
|
// Up to each page to set its writability
|
|
((*pdt)[stepPTidx])->Writeable = true;
|
|
}
|
|
|
|
stepPTidx++;
|
|
stepAddr += SpacePerPT;
|
|
}
|
|
|
|
stepPDTidx++;
|
|
}
|
|
}
|
|
|
|
// For use during startup. Map a range of addresses.
|
|
//
|
|
// Assumes paging is NOT on.
|
|
private static unsafe void BootstrapMapRange(PDPT* pdpt,
|
|
UIntPtr startAddr,
|
|
UIntPtr limitAddr,
|
|
PhysicalAddress physicalStart,
|
|
bool writeable,
|
|
bool writeThrough,
|
|
bool cacheDisable)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(startAddr));
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(limitAddr));
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(physicalStart.Value));
|
|
|
|
BootstrapEnsurePTs(pdpt, startAddr, limitAddr);
|
|
|
|
uint stepPDTidx = (uint)(startAddr / SpacePerPDT);
|
|
UIntPtr stepAddr = startAddr;
|
|
ulong stepPhysical = physicalStart.Value;
|
|
|
|
while ((stepAddr < limitAddr) && (stepPDTidx < 4)) {
|
|
DebugStub.Assert(((*pdpt)[stepPDTidx])->Valid); // ensured above
|
|
PDT* pdt = (PDT*)((*pdpt)[stepPDTidx])->Address;
|
|
|
|
UIntPtr offsetInPDT = MemoryManager.BytesNotAligned(stepAddr, SpacePerPDT);
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(offsetInPDT));
|
|
uint stepPTidx = (uint)(offsetInPDT / SpacePerPT);
|
|
|
|
while ((stepAddr < limitAddr) && (stepPTidx < 512)) {
|
|
DebugStub.Assert(((*pdt)[stepPTidx])->Valid); // ensured above
|
|
PT* pt = (PT*)(((*pdt)[stepPTidx])->Address);
|
|
|
|
UIntPtr offsetInPT = MemoryManager.BytesNotAligned(stepAddr, SpacePerPT);
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(offsetInPT));
|
|
uint pageIndex = (uint)MemoryManager.PagesFromBytes(offsetInPT);
|
|
|
|
while ((stepAddr < limitAddr) && (pageIndex < 512)) {
|
|
// allow overlapping writes for convience...
|
|
//DebugStub.Assert(!((*pt)[pageIndex])->Valid);
|
|
((*pt)[pageIndex])->Address = stepPhysical;
|
|
((*pt)[pageIndex])->User = true; // Ring3
|
|
((*pt)[pageIndex])->Valid = true;
|
|
((*pt)[pageIndex])->Writeable = writeable;
|
|
((*pt)[pageIndex])->Global = stepAddr < Platform.KERNEL_BOUNDARY;
|
|
((*pt)[pageIndex])->WriteThrough = writeThrough;
|
|
((*pt)[pageIndex])->CacheDisable = cacheDisable;
|
|
|
|
stepPhysical += MemoryManager.PageSize;
|
|
pageIndex++;
|
|
stepAddr += MemoryManager.PageSize;
|
|
}
|
|
|
|
stepPTidx++;
|
|
}
|
|
|
|
stepPDTidx++;
|
|
}
|
|
|
|
DebugStub.Assert(stepAddr >= limitAddr);
|
|
}
|
|
|
|
// For use during startup. Maps a single page of memory.
|
|
//
|
|
// Assumes paging is NOT on.
|
|
private static unsafe void BootstrapMapPage(PDPT* pdpt,
|
|
UIntPtr pageAddr,
|
|
PhysicalAddress physPage,
|
|
bool writeable)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(pageAddr));
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(physPage.Value));
|
|
|
|
BootstrapEnsurePTs(pdpt, pageAddr, pageAddr + MemoryManager.PageSize);
|
|
|
|
uint pdtIndex = (uint)(pageAddr / SpacePerPDT);
|
|
UIntPtr offsetInPDT = MemoryManager.BytesNotAligned(pageAddr, SpacePerPDT);
|
|
DebugStub.Assert(((*pdpt)[pdtIndex])->Valid); // ensured above
|
|
PDT* pdt = (PDT*)((*pdpt)[pdtIndex])->Address;
|
|
|
|
uint ptIndex = (uint)(offsetInPDT / SpacePerPT);
|
|
UIntPtr offsetInPT = MemoryManager.BytesNotAligned(offsetInPDT, SpacePerPT);
|
|
DebugStub.Assert(((*pdt)[ptIndex])->Valid); // ensured above;
|
|
DebugStub.Assert(((*pdt)[ptIndex])->User); // Ring3
|
|
PT* pt = (PT*)(((*pdt)[ptIndex])->Address);
|
|
|
|
uint pageIndex = (uint)MemoryManager.PagesFromBytes(offsetInPT);
|
|
DebugStub.Assert(!((*pt)[pageIndex])->Valid);
|
|
|
|
((*pt)[pageIndex])->Address = physPage.Value;
|
|
((*pt)[pageIndex])->User = true; // Ring3
|
|
((*pt)[pageIndex])->Writeable = writeable;
|
|
((*pt)[pageIndex])->Global = pageAddr < Platform.KERNEL_BOUNDARY;
|
|
((*pt)[pageIndex])->Valid = true;
|
|
}
|
|
|
|
/////////////////////////////////////
|
|
// PRIVATE METHODS, POST-BOOTSTRAP
|
|
/////////////////////////////////////
|
|
|
|
//
|
|
// Create a new address space that just maps the (existing) kernel
|
|
// range.
|
|
//
|
|
private static unsafe AddressSpace CreateNewKernelSpace()
|
|
{
|
|
// REVIEW: it's regrettable that we have to lock
|
|
// for access to the scratch area. Maybe we should try to
|
|
// reserve two pages of kernel-space memory instead?
|
|
bool iflag = ScratchLock();
|
|
|
|
try {
|
|
// Get a page to use as a PDPT and map it
|
|
PhysicalAddress physPage = PhysicalPages.AllocPage();
|
|
DebugStub.Assert(physPage != PhysicalAddress.Null);
|
|
MapPageInternal(physPage, scratchAddress1);
|
|
PDPT* pdpt = (PDPT*)scratchAddress1;
|
|
|
|
pdpt->Init();
|
|
|
|
((*pdpt)[0])->Address = kernelPDPE1.Value;
|
|
((*pdpt)[0])->Valid = true;
|
|
|
|
// This is klunky but we can't have static arrays...
|
|
if (numKernelPDPEs >= 2) {
|
|
((*pdpt)[1])->Address = kernelPDPE2.Value;
|
|
((*pdpt)[1])->Valid = true;
|
|
}
|
|
|
|
if (numKernelPDPEs >= 3) {
|
|
((*pdpt)[2])->Address = kernelPDPE3.Value;
|
|
((*pdpt)[2])->Valid = true;
|
|
}
|
|
|
|
DebugStub.Assert(numKernelPDPEs < 4);
|
|
|
|
PhysicalAddress sanity = UnmapPage(scratchAddress1);
|
|
DebugStub.Assert(sanity == physPage);
|
|
|
|
return new AddressSpace(physPage);
|
|
}
|
|
finally {
|
|
ScratchUnlock(iflag);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Set up a kernel-only address space so its user range can be used.
|
|
//
|
|
private static unsafe void InitializeUserRange(AddressSpace space)
|
|
{
|
|
// Our task here is to set up enough PTs to cover the
|
|
// user range's page-table pages. This gets confusing because
|
|
// we also want to write mappings for the PDT and PTs we
|
|
// create.
|
|
|
|
// Assume the user-space page-table range doesn't straddle a PDT.
|
|
DebugStub.Assert((userPageTableBase / SpacePerPDT) ==
|
|
((userPageTableLimit - 1) / SpacePerPDT));
|
|
|
|
// REVIEW: it's regrettable that we have to lock
|
|
// for access to the scratch area. Maybe we should try to
|
|
// reserve two pages of kernel-space memory instead?
|
|
bool iflag = ScratchLock();
|
|
|
|
try {
|
|
PT* pt = null;
|
|
|
|
// First create the PDT
|
|
PhysicalAddress newPdt = PhysicalPages.AllocPage();
|
|
DebugStub.Assert(newPdt != PhysicalAddress.Null);
|
|
MapPageInternal(newPdt, scratchAddress1);
|
|
PDT* pdt = (PDT*)scratchAddress1;
|
|
pdt->Init();
|
|
|
|
// We're going to need to make a mapping for the PDT
|
|
// so it is addressable at userPageTableBase + userPDTOffset.
|
|
// Figure out what PT and PTE that lands in.
|
|
UIntPtr bytesIntoPDTRange = MemoryManager.BytesNotAligned(
|
|
userPageTableBase + userPDTOffset, SpacePerPDT);
|
|
uint pdtPTIdx = (uint)(bytesIntoPDTRange / SpacePerPT);
|
|
|
|
UIntPtr bytesIntoPTRange = MemoryManager.BytesNotAligned(
|
|
bytesIntoPDTRange, SpacePerPT);
|
|
uint pdtPTEIdx = (uint)(bytesIntoPTRange / MemoryManager.PageSize);
|
|
|
|
// Same story for the PDPT, which gets mapped to a user-space
|
|
// address (pdptAddr).
|
|
bytesIntoPDTRange = MemoryManager.BytesNotAligned((UIntPtr)pdptAddr, SpacePerPDT);
|
|
uint pdptPTIdx = (uint)(bytesIntoPDTRange / SpacePerPT);
|
|
|
|
bytesIntoPTRange = MemoryManager.BytesNotAligned(
|
|
bytesIntoPDTRange, SpacePerPT);
|
|
uint pdptPTEIdx = (uint)(bytesIntoPTRange / MemoryManager.PageSize);
|
|
|
|
// Now fill in as many PDT slots with new PTs as are
|
|
// necessary to cover the pagetable range
|
|
bytesIntoPDTRange = MemoryManager.BytesNotAligned(userPageTableBase, SpacePerPDT);
|
|
uint startPTIdx = (uint)(bytesIntoPDTRange / SpacePerPT);
|
|
bytesIntoPDTRange = MemoryManager.BytesNotAligned(userPageTableLimit, SpacePerPDT);
|
|
bytesIntoPDTRange = MemoryManager.Pad(bytesIntoPDTRange, SpacePerPT);
|
|
uint limitPTIdx = (uint)(bytesIntoPDTRange / SpacePerPT);
|
|
uint numPTs = limitPTIdx - startPTIdx;
|
|
|
|
for (uint i = startPTIdx; i < limitPTIdx; i++) {
|
|
PhysicalAddress newPT = PhysicalPages.AllocPage();
|
|
((*pdt)[i])->Address = newPT.Value;
|
|
((*pdt)[i])->User = true; // Ring3
|
|
((*pdt)[i])->Valid = true;
|
|
((*pdt)[i])->Writeable = true; // each page sets this individually
|
|
|
|
// Map in the new PT page
|
|
MapPageInternal(newPT, scratchAddress2, i > startPTIdx);
|
|
pt = (PT*)scratchAddress2;
|
|
pt->Init();
|
|
|
|
if (i == pdtPTIdx) {
|
|
// This is the PT that the PTE for our new PDT
|
|
// goes into (yes, that's confusing)
|
|
((*pt)[pdtPTEIdx])->Address = newPdt.Value;
|
|
((*pt)[pdtPTEIdx])->User = true; // Ring3
|
|
((*pt)[pdtPTEIdx])->Valid = true;
|
|
((*pt)[pdtPTEIdx])->Writeable = true;
|
|
}
|
|
|
|
if (i == pdptPTIdx) {
|
|
// Same drill for the PDPT
|
|
((*pt)[pdptPTEIdx])->Address = space.PdptPage.Value;
|
|
((*pt)[pdptPTEIdx])->User = true; // Ring3
|
|
((*pt)[pdptPTEIdx])->Valid = true;
|
|
((*pt)[pdptPTEIdx])->Writeable = true;
|
|
}
|
|
}
|
|
|
|
// Now, write mappings for all the new PT pages we just
|
|
// created. Assume here that all the PT page mappings
|
|
// fit into a single PT.
|
|
DebugStub.Assert(startPTIdx + numPTs < 512);
|
|
|
|
// Map the PT that will hold our PTEs
|
|
MapPageInternal(new PhysicalAddress(((*pdt)[startPTIdx])->Address),
|
|
scratchAddress2, true);
|
|
pt = (PT*)scratchAddress2;
|
|
|
|
bytesIntoPTRange = MemoryManager.BytesNotAligned(userPageTableBase, SpacePerPT);
|
|
uint startPTEIdx = (uint)(bytesIntoPTRange / MemoryManager.PageSize);
|
|
|
|
for (uint i = 0; i < numPTs; i++) {
|
|
((*pt)[startPTEIdx + i])->Address = ((*pdt)[startPTIdx + i])->Address;
|
|
DebugStub.Assert(((*pt)[startPTEIdx + i])->Address != 0);
|
|
((*pt)[startPTEIdx + i])->User = true; // Ring3
|
|
((*pt)[startPTEIdx + i])->Valid = true;
|
|
((*pt)[startPTEIdx + i])->Writeable = true;
|
|
}
|
|
|
|
PhysicalAddress sanity = UnmapPage(scratchAddress2);
|
|
DebugStub.Assert(sanity.Value == ((*pdt)[startPTIdx])->Address);
|
|
|
|
// Now flip scratchAddress1 to the PDPT for this space
|
|
MapPageInternal(space.PdptPage, scratchAddress1, true);
|
|
PDPT* pdpt = (PDPT*)scratchAddress1;
|
|
|
|
// Write our new PDT into the PDPT
|
|
uint pdtIdx = (uint)(userPageTableBase / SpacePerPDT);
|
|
DebugStub.Assert(pdtIdx >= numKernelPDPEs);
|
|
|
|
((*pdpt)[pdtIdx])->Address = newPdt.Value;
|
|
((*pdpt)[pdtIdx])->Valid = true;
|
|
|
|
sanity = UnmapPage(scratchAddress1);
|
|
DebugStub.Assert(sanity == space.PdptPage);
|
|
}
|
|
finally {
|
|
ScratchUnlock(iflag);
|
|
}
|
|
}
|
|
|
|
// Return a pointer to the PDT that describes the address
|
|
// range beginning at forAddr, allocating and mapping a
|
|
// new one if necessary.
|
|
private static unsafe PDT* EnsurePDT(UIntPtr forAddr)
|
|
{
|
|
PDPE* pdpe = GetPDPE(forAddr); // must succeed
|
|
PDT* retval = GetPDT(forAddr);
|
|
|
|
if (!IsPageMapped((UIntPtr)retval)) {
|
|
// Don't panic; grab a new page to use as a PDT
|
|
PhysicalAddress newPdt = PhysicalPages.AllocPage();
|
|
|
|
if (newPdt == PhysicalAddress.Null) {
|
|
Kernel.Panic("Out of physical pages while trying to allocate a PDT");
|
|
return null;
|
|
}
|
|
|
|
// Activate this page. If this faults, it means
|
|
// our address space wasn't set up properly with
|
|
// enough PTs to describe the page table pages.
|
|
MapPageInternal(newPdt, (UIntPtr)retval);
|
|
|
|
// Now we can address the new page
|
|
retval->Init();
|
|
|
|
// Point our parent PDPT here
|
|
DebugStub.Assert(!pdpe->Valid);
|
|
pdpe->Address = newPdt.Value;
|
|
pdpe->Valid = true;
|
|
}
|
|
else {
|
|
DebugStub.Assert(pdpe->Valid);
|
|
}
|
|
|
|
return retval;
|
|
}
|
|
|
|
// Return a pointer to the PDE for the PT describing the
|
|
// memory range beginning at forAddr, allocating and
|
|
// mapping descriptors as necessary.
|
|
private static unsafe PDE* EnsurePDE(UIntPtr forAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.BytesNotAligned(forAddr, SpacePerPT) == 0);
|
|
UIntPtr bytesIntoPDTRange = MemoryManager.BytesNotAligned(forAddr, SpacePerPDT);
|
|
EnsurePDT(forAddr - bytesIntoPDTRange);
|
|
return GetPDE(forAddr);
|
|
}
|
|
|
|
// Return a pointer to the PT that describes the address
|
|
// range beginning at forAddr, allocating and mapping a
|
|
// new one if necessary.
|
|
private static unsafe PT* EnsurePT(UIntPtr forAddr)
|
|
{
|
|
PDE* pde = EnsurePDE(forAddr);
|
|
PT* retval = GetPT(forAddr);
|
|
|
|
if (!IsPageMapped((UIntPtr)retval)) {
|
|
// Don't panic; grab a new page to use as a PT
|
|
PhysicalAddress newPt = PhysicalPages.AllocPage();
|
|
|
|
if (newPt == PhysicalAddress.Null) {
|
|
Kernel.Panic("Out of physical pages while trying to allocate a PT");
|
|
return null;
|
|
}
|
|
|
|
// Activate this page. If this faults, it means
|
|
// our address space wasn't set up properly with
|
|
// enough PTs to describe the page table pages.
|
|
MapPageInternal(newPt, (UIntPtr)retval);
|
|
|
|
// Now we can address the new page
|
|
retval->Init();
|
|
|
|
// Add ourselves to our parent PDT
|
|
DebugStub.Assert(!pde->Valid);
|
|
pde->Address = newPt.Value;
|
|
pde->User = true; // Ring3
|
|
pde->Valid = true;
|
|
pde->Writeable = true;
|
|
}
|
|
else {
|
|
DebugStub.Assert(pde->Valid);
|
|
}
|
|
|
|
return retval;
|
|
}
|
|
|
|
// Return a pointer to the PTE for the given page address,
|
|
// allocating and mapping descriptors as necessary.
|
|
private static unsafe PTE* EnsurePTE(UIntPtr pageAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(pageAddr));
|
|
|
|
UIntPtr bytesIntoPTRange = MemoryManager.BytesNotAligned(pageAddr, SpacePerPT);
|
|
EnsurePT(pageAddr - bytesIntoPTRange);
|
|
return GetPTE(pageAddr);
|
|
}
|
|
|
|
// Return a pointer to the PDT that describes the address
|
|
// range beginning at forAddr, without ensuring that it is
|
|
// mapped.
|
|
private static unsafe PDT* GetPDT(UIntPtr forAddr)
|
|
{
|
|
// Make sure forAddr specifies a range that
|
|
// makes sense
|
|
DebugStub.Assert(MemoryManager.BytesNotAligned(forAddr, SpacePerPDT) == 0);
|
|
|
|
UIntPtr pdtBase = kernelPageTableBase + kernelPDTOffset;
|
|
UIntPtr addr = forAddr;
|
|
|
|
if (forAddr >= Platform.KERNEL_BOUNDARY) {
|
|
// User-space address
|
|
pdtBase = userPageTableBase + userPDTOffset;
|
|
addr -= Platform.KERNEL_BOUNDARY;
|
|
}
|
|
|
|
// Index into the table of PDTs
|
|
uint pdtIndex = (uint)(addr / SpacePerPDT);
|
|
return (PDT*)(pdtBase + MemoryManager.BytesFromPages(pdtIndex));
|
|
}
|
|
|
|
// Return a pointer to the PT that describes the address
|
|
// range beginning at forAddr, without ensuring that it is
|
|
// mapped.
|
|
private static unsafe PT* GetPT(UIntPtr forAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.BytesNotAligned(forAddr, SpacePerPT) == 0);
|
|
|
|
UIntPtr ptBase = kernelPageTableBase;
|
|
UIntPtr addr = forAddr;
|
|
|
|
if (forAddr >= Platform.KERNEL_BOUNDARY) {
|
|
// User-space address
|
|
ptBase = userPageTableBase;
|
|
addr -= Platform.KERNEL_BOUNDARY;
|
|
}
|
|
|
|
// Index into the table of PTs
|
|
uint ptIndex = (uint)(addr / SpacePerPT);
|
|
return (PT*)(ptBase + MemoryManager.BytesFromPages(ptIndex));
|
|
}
|
|
|
|
// Return a pointer to the PDPE for the PDT that describes
|
|
// the memory range beginning at the provided address.
|
|
// We assume the PDPT is always mapped.
|
|
private static unsafe PDPE* GetPDPE(UIntPtr forAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.BytesNotAligned(forAddr, SpacePerPDT) == 0);
|
|
uint PDTidx = (uint)(forAddr / SpacePerPDT);
|
|
return (*pdptAddr)[(uint)PDTidx];
|
|
}
|
|
|
|
// Return a pointer to the PDE for the PT that describes
|
|
// the memory range beginning at the provided address,
|
|
// without ensuring that the PDT is mapped
|
|
private static unsafe PDE* GetPDE(UIntPtr forAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.BytesNotAligned(forAddr, SpacePerPT) == 0);
|
|
UIntPtr bytesIntoPDTRange = MemoryManager.BytesNotAligned(forAddr, SpacePerPDT);
|
|
uint ptWithinPDT = (uint)(bytesIntoPDTRange / SpacePerPT);
|
|
PDT* pdt = GetPDT(forAddr - bytesIntoPDTRange);
|
|
return (*pdt)[(uint)ptWithinPDT];
|
|
}
|
|
|
|
// Return a pointer to the PTE for the given page address,
|
|
// without ensuring that the PT is mapped.
|
|
private static unsafe PTE* GetPTE(UIntPtr pageAddr)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(pageAddr));
|
|
UIntPtr bytesIntoPTRange = MemoryManager.BytesNotAligned(pageAddr, SpacePerPT);
|
|
uint pageWithinPT = (uint)MemoryManager.PagesFromBytes(bytesIntoPTRange);
|
|
PT* pt = GetPT(pageAddr - bytesIntoPTRange);
|
|
return (*pt)[pageWithinPT];
|
|
}
|
|
|
|
// This function directly examines the PTE for the virtual
|
|
// address of a page-table structure (this address should
|
|
// fall between pageTableBaseAddr and
|
|
// pageTableBaseAddr + PageTableSize), on the assumption
|
|
// that sufficient PTs to describe all page-table structures
|
|
// are always mapped.
|
|
private static unsafe bool IsKernelDescriptorPageMapped(UIntPtr pageAddr)
|
|
{
|
|
// Make sure this address is for a page-table element
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(pageAddr));
|
|
DebugStub.Assert(pageAddr >= kernelPageTableBase);
|
|
DebugStub.Assert(pageAddr < kernelPageTableLimit);
|
|
PTE* pte = GetPTE(pageAddr);
|
|
|
|
// If this faults, our address space wasn't set up right
|
|
return pte->Valid;
|
|
}
|
|
|
|
// Same for userland
|
|
private static unsafe bool IsUserDescriptorPageMapped(UIntPtr pageAddr)
|
|
{
|
|
// Make sure this address is for a page-table element
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(pageAddr));
|
|
DebugStub.Assert(pageAddr >= userPageTableBase);
|
|
DebugStub.Assert(pageAddr < userPageTableLimit);
|
|
PTE* pte = GetPTE(pageAddr);
|
|
|
|
// If this faults, our address space wasn't set up right
|
|
return pte->Valid;
|
|
}
|
|
|
|
//
|
|
// Writes a new virtual -> physical address into the page table,
|
|
// ASSUMING the right PT is already allocated and mapped.
|
|
//
|
|
private static unsafe void MapPageInternal(PhysicalAddress physAddr,
|
|
UIntPtr virtualAddr,
|
|
bool expectExistingMapping)
|
|
{
|
|
DebugStub.Assert(MemoryManager.IsPageAligned(virtualAddr));
|
|
PTE* pte = GetPTE(virtualAddr);
|
|
|
|
DebugStub.Assert(pte->Valid == expectExistingMapping,
|
|
"Unexpected mapping state");
|
|
|
|
pte->Address = physAddr.Value;
|
|
pte->User = true; // Ring3
|
|
pte->Valid = true;
|
|
pte->Writeable = true;
|
|
pte->Global = virtualAddr < Platform.KERNEL_BOUNDARY;
|
|
Processor.InvalidateTLBEntry(virtualAddr);
|
|
}
|
|
|
|
private static unsafe void MapPageInternal(PhysicalAddress physAddr,
|
|
UIntPtr virtualAddr)
|
|
{
|
|
// Expect to not overwrite an existing mapping
|
|
MapPageInternal(physAddr, virtualAddr, false);
|
|
}
|
|
|
|
[Inline]
|
|
[NoStackLinkCheckTrans]
|
|
private static bool ScratchLock()
|
|
{
|
|
bool enabled = Processor.DisableInterrupts();
|
|
#if SINGULARITY_MP
|
|
scratchSpin.Acquire(Thread.CurrentThread);
|
|
#endif // SINGULARITY_MP
|
|
return enabled;
|
|
}
|
|
|
|
[Inline]
|
|
[NoStackLinkCheckTrans]
|
|
private static void ScratchUnlock(bool iflag)
|
|
{
|
|
#if SINGULARITY_MP
|
|
scratchSpin.Release(Thread.CurrentThread);
|
|
#endif // SINGULARITY_MP
|
|
Processor.RestoreInterrupts(iflag);
|
|
}
|
|
}
|
|
}
|