mirrors/ryujinx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rewrite kernel memory allocator (#3316)

Browse source 
* Rewrite kernel memory allocator

* Remove unused using

* Adjust private static field naming

* Change UlongBitSize to UInt64BitSize

* Fix unused argument, change argument order to be inline with official code and disable random allocation
This commit is contained in:
gdkchan 2022-06-22 12:28:14 -03:00 committed by GitHub
parent c881cd2d14
commit f2a41b7a1c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 676 additions and 480 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
Ryujinx.Common/Utilities/BitUtils.cs
View file

@ -22,7 +22,17 @@ namespace Ryujinx.Common
public static long AlignUp(long value, int size) public static long AlignUp(long value, int size)
{ {
return (value + (size - 1)) & -(long)size; return AlignUp(value, (long)size);
}
public static ulong AlignUp(ulong value, ulong size)
{
return (ulong)AlignUp((long)value, (long)size);
}
public static long AlignUp(long value, long size)
{
return (value + (size - 1)) & -size;
} }
public static uint AlignDown(uint value, int size) public static uint AlignDown(uint value, int size)
@ -42,7 +52,17 @@ namespace Ryujinx.Common
public static long AlignDown(long value, int size) public static long AlignDown(long value, int size)
{ {
return value & -(long)size; return AlignDown(value, (long)size);
}
public static ulong AlignDown(ulong value, ulong size)
{
return (ulong)AlignDown((long)value, (long)size);
}
public static long AlignDown(long value, long size)
{
return value & -size;
} }
public static int DivRoundUp(int value, int dividend) public static int DivRoundUp(int value, int dividend)

87
Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionBlock.cs
View file

@ -1,87 +0,0 @@
namespace Ryujinx.HLE.HOS.Kernel.Memory
{
class KMemoryRegionBlock
{
public long[][] Masks;
public ulong FreeCount;
public int MaxLevel;
public ulong StartAligned;
public ulong SizeInBlocksTruncated;
public ulong SizeInBlocksRounded;
public int Order;
public int NextOrder;
public bool TryCoalesce(int index, int count)
{
long mask = ((1L << count) - 1) << (index & 63);
index /= 64;
if (count >= 64)
{
int remaining = count;
int tempIdx = index;
do
{
if (Masks[MaxLevel - 1][tempIdx++] != -1L)
{
return false;
}
remaining -= 64;
}
while (remaining != 0);
remaining = count;
tempIdx = index;
do
{
Masks[MaxLevel - 1][tempIdx] = 0;
ClearMaskBit(MaxLevel - 2, tempIdx++);
remaining -= 64;
}
while (remaining != 0);
}
else
{
long value = Masks[MaxLevel - 1][index];
if ((mask & ~value) != 0)
{
return false;
}
value &= ~mask;
Masks[MaxLevel - 1][index] = value;
if (value == 0)
{
ClearMaskBit(MaxLevel - 2, index);
}
}
FreeCount -= (ulong)count;
return true;
}
public void ClearMaskBit(int startLevel, int index)
{
for (int level = startLevel; level >= 0; level--, index /= 64)
{
Masks[level][index / 64] &= ~(1L << (index & 63));
if (Masks[level][index / 64] != 0)
{
break;
}
}
}
}
}

456
Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs
View file

@ -1,102 +1,42 @@
using Ryujinx.Common;
using Ryujinx.HLE.HOS.Kernel.Common; using Ryujinx.HLE.HOS.Kernel.Common;
using System.Diagnostics; using System.Diagnostics;
using System.Numerics;
namespace Ryujinx.HLE.HOS.Kernel.Memory namespace Ryujinx.HLE.HOS.Kernel.Memory
{ {
class KMemoryRegionManager class KMemoryRegionManager
{ {
private static readonly int[] BlockOrders = new int[] { 12, 16, 21, 22, 25, 29, 30 }; private readonly KPageHeap _pageHeap;
public ulong Address { get; private set; } public ulong Address { get; }
public ulong EndAddr { get; private set; } public ulong Size { get; }
public ulong Size { get; private set; } public ulong EndAddr => Address + Size;
private int _blockOrdersCount;
private readonly KMemoryRegionBlock[] _blocks;
private readonly ushort[] _pageReferenceCounts; private readonly ushort[] _pageReferenceCounts;
public KMemoryRegionManager(ulong address, ulong size, ulong endAddr) public KMemoryRegionManager(ulong address, ulong size, ulong endAddr)
{ {
_blocks = new KMemoryRegionBlock[BlockOrders.Length];
Address = address; Address = address;
Size = size; Size = size;
EndAddr = endAddr;
_blockOrdersCount = BlockOrders.Length;
for (int blockIndex = 0; blockIndex < _blockOrdersCount; blockIndex++)
{
_blocks[blockIndex] = new KMemoryRegionBlock();
_blocks[blockIndex].Order = BlockOrders[blockIndex];
int nextOrder = blockIndex == _blockOrdersCount - 1 ? 0 : BlockOrders[blockIndex + 1];
_blocks[blockIndex].NextOrder = nextOrder;
int currBlockSize = 1 << BlockOrders[blockIndex];
int nextBlockSize = currBlockSize;
if (nextOrder != 0)
{
nextBlockSize = 1 << nextOrder;
}
ulong startAligned = BitUtils.AlignDown(address, nextBlockSize);
ulong endAddrAligned = BitUtils.AlignDown(endAddr, currBlockSize);
ulong sizeInBlocksTruncated = (endAddrAligned - startAligned) >> BlockOrders[blockIndex];
ulong endAddrRounded = BitUtils.AlignUp(address + size, nextBlockSize);
ulong sizeInBlocksRounded = (endAddrRounded - startAligned) >> BlockOrders[blockIndex];
_blocks[blockIndex].StartAligned = startAligned;
_blocks[blockIndex].SizeInBlocksTruncated = sizeInBlocksTruncated;
_blocks[blockIndex].SizeInBlocksRounded = sizeInBlocksRounded;
ulong currSizeInBlocks = sizeInBlocksRounded;
int maxLevel = 0;
do
{
maxLevel++;
}
while ((currSizeInBlocks /= 64) != 0);
_blocks[blockIndex].MaxLevel = maxLevel;
_blocks[blockIndex].Masks = new long[maxLevel][];
currSizeInBlocks = sizeInBlocksRounded;
for (int level = maxLevel - 1; level >= 0; level--)
{
currSizeInBlocks = (currSizeInBlocks + 63) / 64;
_blocks[blockIndex].Masks[level] = new long[currSizeInBlocks];
}
}
_pageReferenceCounts = new ushort[size / KPageTableBase.PageSize]; _pageReferenceCounts = new ushort[size / KPageTableBase.PageSize];
if (size != 0) _pageHeap = new KPageHeap(address, size);
{ _pageHeap.Free(address, size / KPageTableBase.PageSize);
FreePages(address, size / KPageTableBase.PageSize); _pageHeap.UpdateUsedSize();
}
} }
public KernelResult AllocatePages(ulong pagesCount, bool backwards, out KPageList pageList) public KernelResult AllocatePages(out KPageList pageList, ulong pagesCount)
{ {
lock (_blocks) if (pagesCount == 0)
{ {
KernelResult result = AllocatePagesImpl(pagesCount, backwards, out pageList); pageList = new KPageList();
return KernelResult.Success;
}
lock (_pageHeap)
{
KernelResult result = AllocatePagesImpl(out pageList, pagesCount, false);
if (result == KernelResult.Success) if (result == KernelResult.Success)
{ {
@ -112,9 +52,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
public ulong AllocatePagesContiguous(KernelContext context, ulong pagesCount, bool backwards) public ulong AllocatePagesContiguous(KernelContext context, ulong pagesCount, bool backwards)
{ {
lock (_blocks) if (pagesCount == 0)
{ {
ulong address = AllocatePagesContiguousImpl(pagesCount, backwards); return 0;
}
lock (_pageHeap)
{
ulong address = AllocatePagesContiguousImpl(pagesCount, 1, backwards);
if (address != 0) if (address != 0)
{ {
@ -126,373 +71,110 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
} }
} }
private KernelResult AllocatePagesImpl(ulong pagesCount, bool backwards, out KPageList pageList) private KernelResult AllocatePagesImpl(out KPageList pageList, ulong pagesCount, bool random)
{ {
pageList = new KPageList(); pageList = new KPageList();
if (_blockOrdersCount > 0) int heapIndex = KPageHeap.GetBlockIndex(pagesCount);
{
if (GetFreePagesImpl() < pagesCount) if (heapIndex < 0)
{
return KernelResult.OutOfMemory;
}
}
else if (pagesCount != 0)
{ {
return KernelResult.OutOfMemory; return KernelResult.OutOfMemory;
} }
for (int blockIndex = _blockOrdersCount - 1; blockIndex >= 0; blockIndex--) for (int index = heapIndex; index >= 0; index--)
{ {
KMemoryRegionBlock block = _blocks[blockIndex]; ulong pagesPerAlloc = KPageHeap.GetBlockPagesCount(index);
ulong bestFitBlockSize = 1UL << block.Order; while (pagesCount >= pagesPerAlloc)
ulong blockPagesCount = bestFitBlockSize / KPageTableBase.PageSize;
// Check if this is the best fit for this page size.
// If so, try allocating as much requested pages as possible.
while (blockPagesCount <= pagesCount)
{ {
ulong address = AllocatePagesForOrder(blockIndex, backwards, bestFitBlockSize); ulong allocatedBlock = _pageHeap.AllocateBlock(index, random);
// The address being zero means that no free space was found on that order, if (allocatedBlock == 0)
// just give up and try with the next one.
if (address == 0)
{ {
break; break;
} }
// Add new allocated page(s) to the pages list. KernelResult result = pageList.AddRange(allocatedBlock, pagesPerAlloc);
// If an error occurs, then free all allocated pages and fail.
KernelResult result = pageList.AddRange(address, blockPagesCount);
if (result != KernelResult.Success) if (result != KernelResult.Success)
{ {
FreePages(address, blockPagesCount); FreePages(pageList);
_pageHeap.Free(allocatedBlock, pagesPerAlloc);
foreach (KPageNode pageNode in pageList)
{
FreePages(pageNode.Address, pageNode.PagesCount);
}
return result; return result;
} }
pagesCount -= blockPagesCount; pagesCount -= pagesPerAlloc;
} }
} }
// Success case, all requested pages were allocated successfully. if (pagesCount != 0)
if (pagesCount == 0)
{ {
return KernelResult.Success; FreePages(pageList);
return KernelResult.OutOfMemory;
} }
// Error case, free allocated pages and return out of memory. return KernelResult.Success;
foreach (KPageNode pageNode in pageList)
{
FreePages(pageNode.Address, pageNode.PagesCount);
}
pageList = null;
return KernelResult.OutOfMemory;
} }
private ulong AllocatePagesContiguousImpl(ulong pagesCount, bool backwards) private ulong AllocatePagesContiguousImpl(ulong pagesCount, ulong alignPages, bool random)
{ {
if (pagesCount == 0 || _blocks.Length < 1) int heapIndex = KPageHeap.GetAlignedBlockIndex(pagesCount, alignPages);
ulong allocatedBlock = _pageHeap.AllocateBlock(heapIndex, random);
if (allocatedBlock == 0)
{ {
return 0; return 0;
} }
int blockIndex = 0; ulong allocatedPages = KPageHeap.GetBlockPagesCount(heapIndex);
while ((1UL << _blocks[blockIndex].Order) / KPageTableBase.PageSize < pagesCount) if (allocatedPages > pagesCount)
{ {
if (++blockIndex >= _blocks.Length) _pageHeap.Free(allocatedBlock + pagesCount * KPageTableBase.PageSize, allocatedPages - pagesCount);
{
return 0;
}
} }
ulong tightestFitBlockSize = 1UL << _blocks[blockIndex].Order; return allocatedBlock;
ulong address = AllocatePagesForOrder(blockIndex, backwards, tightestFitBlockSize);
ulong requiredSize = pagesCount * KPageTableBase.PageSize;
if (address != 0 && tightestFitBlockSize > requiredSize)
{
FreePages(address + requiredSize, (tightestFitBlockSize - requiredSize) / KPageTableBase.PageSize);
}
return address;
} }
private ulong AllocatePagesForOrder(int blockIndex, bool backwards, ulong bestFitBlockSize) public void FreePage(ulong address)
{ {
ulong address = 0; lock (_pageHeap)
KMemoryRegionBlock block = null;
for (int currBlockIndex = blockIndex;
currBlockIndex < _blockOrdersCount && address == 0;
currBlockIndex++)
{ {
block = _blocks[currBlockIndex]; _pageHeap.Free(address, 1);
int index = 0;
bool zeroMask = false;
for (int level = 0; level < block.MaxLevel; level++)
{
long mask = block.Masks[level][index];
if (mask == 0)
{
zeroMask = true;
break;
}
if (backwards)
{
index = (index * 64 + 63) - BitOperations.LeadingZeroCount((ulong)mask);
}
else
{
index = index * 64 + BitOperations.LeadingZeroCount((ulong)BitUtils.ReverseBits64(mask));
}
}
if (block.SizeInBlocksTruncated <= (ulong)index || zeroMask)
{
continue;
}
block.FreeCount--;
int tempIdx = index;
for (int level = block.MaxLevel - 1; level >= 0; level--, tempIdx /= 64)
{
block.Masks[level][tempIdx / 64] &= ~(1L << (tempIdx & 63));
if (block.Masks[level][tempIdx / 64] != 0)
{
break;
}
}
address = block.StartAligned + ((ulong)index << block.Order);
} }
for (int currBlockIndex = blockIndex;
currBlockIndex < _blockOrdersCount && address == 0;
currBlockIndex++)
{
block = _blocks[currBlockIndex];
int index = 0;
bool zeroMask = false;
for (int level = 0; level < block.MaxLevel; level++)
{
long mask = block.Masks[level][index];
if (mask == 0)
{
zeroMask = true;
break;
}
if (backwards)
{
index = index * 64 + BitOperations.LeadingZeroCount((ulong)BitUtils.ReverseBits64(mask));
}
else
{
index = (index * 64 + 63) - BitOperations.LeadingZeroCount((ulong)mask);
}
}
if (block.SizeInBlocksTruncated <= (ulong)index || zeroMask)
{
continue;
}
block.FreeCount--;
int tempIdx = index;
for (int level = block.MaxLevel - 1; level >= 0; level--, tempIdx /= 64)
{
block.Masks[level][tempIdx / 64] &= ~(1L << (tempIdx & 63));
if (block.Masks[level][tempIdx / 64] != 0)
{
break;
}
}
address = block.StartAligned + ((ulong)index << block.Order);
}
if (address != 0)
{
// If we are using a larger order than best fit, then we should
// split it into smaller blocks.
ulong firstFreeBlockSize = 1UL << block.Order;
if (firstFreeBlockSize > bestFitBlockSize)
{
FreePages(address + bestFitBlockSize, (firstFreeBlockSize - bestFitBlockSize) / KPageTableBase.PageSize);
}
}
return address;
} }
private void FreePages(ulong address, ulong pagesCount) public void FreePages(KPageList pageList)
{ {
lock (_blocks) lock (_pageHeap)
{ {
ulong endAddr = address + pagesCount * KPageTableBase.PageSize; foreach (KPageNode pageNode in pageList)
int blockIndex = _blockOrdersCount - 1;
ulong addressRounded = 0;
ulong endAddrTruncated = 0;
for (; blockIndex >= 0; blockIndex--)
{ {
KMemoryRegionBlock allocInfo = _blocks[blockIndex]; _pageHeap.Free(pageNode.Address, pageNode.PagesCount);
int blockSize = 1 << allocInfo.Order;
addressRounded = BitUtils.AlignUp (address, blockSize);
endAddrTruncated = BitUtils.AlignDown(endAddr, blockSize);
if (addressRounded < endAddrTruncated)
{
break;
}
} }
}
}
void FreeRegion(ulong currAddress) public void FreePages(ulong address, ulong pagesCount)
{ {
for (int currBlockIndex = blockIndex; lock (_pageHeap)
currBlockIndex < _blockOrdersCount && currAddress != 0; {
currBlockIndex++) _pageHeap.Free(address, pagesCount);
{
KMemoryRegionBlock block = _blocks[currBlockIndex];
block.FreeCount++;
ulong freedBlocks = (currAddress - block.StartAligned) >> block.Order;
int index = (int)freedBlocks;
for (int level = block.MaxLevel - 1; level >= 0; level--, index /= 64)
{
long mask = block.Masks[level][index / 64];
block.Masks[level][index / 64] = mask | (1L << (index & 63));
if (mask != 0)
{
break;
}
}
int blockSizeDelta = 1 << (block.NextOrder - block.Order);
int freedBlocksTruncated = BitUtils.AlignDown((int)freedBlocks, blockSizeDelta);
if (!block.TryCoalesce(freedBlocksTruncated, blockSizeDelta))
{
break;
}
currAddress = block.StartAligned + ((ulong)freedBlocksTruncated << block.Order);
}
}
// Free inside aligned region.
ulong baseAddress = addressRounded;
while (baseAddress < endAddrTruncated)
{
ulong blockSize = 1UL << _blocks[blockIndex].Order;
FreeRegion(baseAddress);
baseAddress += blockSize;
}
int nextBlockIndex = blockIndex - 1;
// Free region between Address and aligned region start.
baseAddress = addressRounded;
for (blockIndex = nextBlockIndex; blockIndex >= 0; blockIndex--)
{
ulong blockSize = 1UL << _blocks[blockIndex].Order;
while (baseAddress - blockSize >= address)
{
baseAddress -= blockSize;
FreeRegion(baseAddress);
}
}
// Free region between aligned region end and End Address.
baseAddress = endAddrTruncated;
for (blockIndex = nextBlockIndex; blockIndex >= 0; blockIndex--)
{
ulong blockSize = 1UL << _blocks[blockIndex].Order;
while (baseAddress + blockSize <= endAddr)
{
FreeRegion(baseAddress);
baseAddress += blockSize;
}
}
} }
} }
public ulong GetFreePages() public ulong GetFreePages()
{ {
lock (_blocks) lock (_pageHeap)
{ {
return GetFreePagesImpl(); return _pageHeap.GetFreePagesCount();
} }
} }
private ulong GetFreePagesImpl()
{
ulong availablePages = 0;
for (int blockIndex = 0; blockIndex < _blockOrdersCount; blockIndex++)
{
KMemoryRegionBlock block = _blocks[blockIndex];
ulong blockPagesCount = (1UL << block.Order) / KPageTableBase.PageSize;
availablePages += blockPagesCount * block.FreeCount;
}
return availablePages;
}
public void IncrementPagesReferenceCount(ulong address, ulong pagesCount) public void IncrementPagesReferenceCount(ulong address, ulong pagesCount)
{ {
ulong index = GetPageOffset(address); ulong index = GetPageOffset(address);

298
Ryujinx.HLE/HOS/Kernel/Memory/KPageBitmap.cs Normal file
View file

@ -0,0 +1,298 @@
using Ryujinx.Common;
using System;
using System.Numerics;
namespace Ryujinx.HLE.HOS.Kernel.Memory
{
class KPageBitmap
{
private struct RandomNumberGenerator
{
private uint _entropy;
private uint _bitsAvailable;
private void RefreshEntropy()
{
_entropy = 0;
_bitsAvailable = sizeof(uint) * 8;
}
private bool GenerateRandomBit()
{
if (_bitsAvailable == 0)
{
RefreshEntropy();
}
bool bit = (_entropy & 1) != 0;
_entropy >>= 1;
_bitsAvailable--;
return bit;
}
public int SelectRandomBit(ulong bitmap)
{
int selected = 0;
int bitsCount = UInt64BitSize / 2;
ulong mask = (1UL << bitsCount) - 1;
while (bitsCount != 0)
{
ulong low = bitmap & mask;
ulong high = (bitmap >> bitsCount) & mask;
bool chooseLow;
if (high == 0)
{
chooseLow = true;
}
else if (low == 0)
{
chooseLow = false;
}
else
{
chooseLow = GenerateRandomBit();
}
if (chooseLow)
{
bitmap = low;
}
else
{
bitmap = high;
selected += bitsCount;
}
bitsCount /= 2;
mask >>= bitsCount;
}
return selected;
}
}
private const int UInt64BitSize = sizeof(ulong) * 8;
private const int MaxDepth = 4;
private readonly RandomNumberGenerator _rng;
private readonly ArraySegment<ulong>[] _bitStorages;
private int _usedDepths;
public int BitsCount { get; private set; }
public int HighestDepthIndex => _usedDepths - 1;
public KPageBitmap()
{
_rng = new RandomNumberGenerator();
_bitStorages = new ArraySegment<ulong>[MaxDepth];
}
public ArraySegment<ulong> Initialize(ArraySegment<ulong> storage, ulong size)
{
_usedDepths = GetRequiredDepth(size);
for (int depth = HighestDepthIndex; depth >= 0; depth--)
{
_bitStorages[depth] = storage;
size = BitUtils.DivRoundUp(size, UInt64BitSize);
storage = storage.Slice((int)size);
}
return storage;
}
public ulong FindFreeBlock(bool random)
{
ulong offset = 0;
int depth = 0;
if (random)
{
do
{
ulong v = _bitStorages[depth][(int)offset];
if (v == 0)
{
return ulong.MaxValue;
}
offset = offset * UInt64BitSize + (ulong)_rng.SelectRandomBit(v);
}
while (++depth < _usedDepths);
}
else
{
do
{
ulong v = _bitStorages[depth][(int)offset];
if (v == 0)
{
return ulong.MaxValue;
}
offset = offset * UInt64BitSize + (ulong)BitOperations.TrailingZeroCount(v);
}
while (++depth < _usedDepths);
}
return offset;
}
public void SetBit(ulong offset)
{
SetBit(HighestDepthIndex, offset);
BitsCount++;
}
public void ClearBit(ulong offset)
{
ClearBit(HighestDepthIndex, offset);
BitsCount--;
}
public bool ClearRange(ulong offset, int count)
{
int depth = HighestDepthIndex;
var bits = _bitStorages[depth];
int bitInd = (int)(offset / UInt64BitSize);
if (count < UInt64BitSize)
{
int shift = (int)(offset % UInt64BitSize);
ulong mask = ((1UL << count) - 1) << shift;
ulong v = bits[bitInd];
if ((v & mask) != mask)
{
return false;
}
v &= ~mask;
bits[bitInd] = v;
if (v == 0)
{
ClearBit(depth - 1, (ulong)bitInd);
}
}
else
{
int remaining = count;
int i = 0;
do
{
if (bits[bitInd + i++] != ulong.MaxValue)
{
return false;
}
remaining -= UInt64BitSize;
}
while (remaining > 0);
remaining = count;
i = 0;
do
{
bits[bitInd + i] = 0;
ClearBit(depth - 1, (ulong)(bitInd + i));
i++;
remaining -= UInt64BitSize;
}
while (remaining > 0);
}
BitsCount -= count;
return true;
}
private void SetBit(int depth, ulong offset)
{
while (depth >= 0)
{
int ind = (int)(offset / UInt64BitSize);
int which = (int)(offset % UInt64BitSize);
ulong mask = 1UL << which;
ulong v = _bitStorages[depth][ind];
_bitStorages[depth][ind] = v | mask;
if (v != 0)
{
break;
}
offset = (ulong)ind;
depth--;
}
}
private void ClearBit(int depth, ulong offset)
{
while (depth >= 0)
{
int ind = (int)(offset / UInt64BitSize);
int which = (int)(offset % UInt64BitSize);
ulong mask = 1UL << which;
ulong v = _bitStorages[depth][ind];
v &= ~mask;
_bitStorages[depth][ind] = v;
if (v != 0)
{
break;
}
offset = (ulong)ind;
depth--;
}
}
private static int GetRequiredDepth(ulong regionSize)
{
int depth = 0;
do
{
regionSize /= UInt64BitSize;
depth++;
}
while (regionSize != 0);
return depth;
}
public static int CalculateManagementOverheadSize(ulong regionSize)
{
int overheadBits = 0;
for (int depth = GetRequiredDepth(regionSize) - 1; depth >= 0; depth--)
{
regionSize = BitUtils.DivRoundUp(regionSize, UInt64BitSize);
overheadBits += (int)regionSize;
}
return overheadBits * sizeof(ulong);
}
}
}

283
Ryujinx.HLE/HOS/Kernel/Memory/KPageHeap.cs Normal file
View file

@ -0,0 +1,283 @@
using Ryujinx.Common;
using System;
namespace Ryujinx.HLE.HOS.Kernel.Memory
{
class KPageHeap
{
private class Block
{
private KPageBitmap _bitmap = new KPageBitmap();
private ulong _heapAddress;
private ulong _endOffset;
public int Shift { get; private set; }
public int NextShift { get; private set; }
public ulong Size => 1UL << Shift;
public int PagesCount => (int)(Size / KPageTableBase.PageSize);
public int FreeBlocksCount => _bitmap.BitsCount;
public int FreePagesCount => FreeBlocksCount * PagesCount;
public ArraySegment<ulong> Initialize(ulong address, ulong size, int blockShift, int nextBlockShift, ArraySegment<ulong> bitStorage)
{
Shift = blockShift;
NextShift = nextBlockShift;
ulong endAddress = address + size;
ulong align = nextBlockShift != 0
? 1UL << nextBlockShift
: 1UL << blockShift;
address = BitUtils.AlignDown(address, align);
endAddress = BitUtils.AlignUp (endAddress, align);
_heapAddress = address;
_endOffset = (endAddress - address) / (1UL << blockShift);
return _bitmap.Initialize(bitStorage, _endOffset);
}
public ulong PushBlock(ulong address)
{
ulong offset = (address - _heapAddress) >> Shift;
_bitmap.SetBit(offset);
if (NextShift != 0)
{
int diff = 1 << (NextShift - Shift);
offset = BitUtils.AlignDown(offset, diff);
if (_bitmap.ClearRange(offset, diff))
{
return _heapAddress + (offset << Shift);
}
}
return 0;
}
public ulong PopBlock(bool random)
{
long sOffset = (long)_bitmap.FindFreeBlock(random);
if (sOffset < 0L)
{
return 0;
}
ulong offset = (ulong)sOffset;
_bitmap.ClearBit(offset);
return _heapAddress + (offset << Shift);
}
public static int CalculateManagementOverheadSize(ulong regionSize, int currBlockShift, int nextBlockShift)
{
ulong currBlockSize = 1UL << currBlockShift;
ulong nextBlockSize = 1UL << nextBlockShift;
ulong align = nextBlockShift != 0 ? nextBlockSize : currBlockSize;
return KPageBitmap.CalculateManagementOverheadSize((align * 2 + BitUtils.AlignUp(regionSize, align)) / currBlockSize);
}
}
private static readonly int[] _memoryBlockPageShifts = new int[] { 12, 16, 21, 22, 25, 29, 30 };
private readonly ulong _heapAddress;
private readonly ulong _heapSize;
private ulong _usedSize;
private readonly int _blocksCount;
private readonly Block[] _blocks;
public KPageHeap(ulong address, ulong size) : this(address, size, _memoryBlockPageShifts)
{
}
public KPageHeap(ulong address, ulong size, int[] blockShifts)
{
_heapAddress = address;
_heapSize = size;
_blocksCount = blockShifts.Length;
_blocks = new Block[_memoryBlockPageShifts.Length];
var currBitmapStorage = new ArraySegment<ulong>(new ulong[CalculateManagementOverheadSize(size, blockShifts)]);
for (int i = 0; i < blockShifts.Length; i++)
{
int currBlockShift = blockShifts[i];
int nextBlockShift = i != blockShifts.Length - 1 ? blockShifts[i + 1] : 0;
_blocks[i] = new Block();
currBitmapStorage = _blocks[i].Initialize(address, size, currBlockShift, nextBlockShift, currBitmapStorage);
}
}
public void UpdateUsedSize()
{
_usedSize = _heapSize - (GetFreePagesCount() * KPageTableBase.PageSize);
}
public ulong GetFreePagesCount()
{
ulong freeCount = 0;
for (int i = 0; i < _blocksCount; i++)
{
freeCount += (ulong)_blocks[i].FreePagesCount;
}
return freeCount;
}
public ulong AllocateBlock(int index, bool random)
{
ulong neededSize = _blocks[index].Size;
for (int i = index; i < _blocksCount; i++)
{
ulong address = _blocks[i].PopBlock(random);
if (address != 0)
{
ulong allocatedSize = _blocks[i].Size;
if (allocatedSize > neededSize)
{
Free(address + neededSize, (allocatedSize - neededSize) / KPageTableBase.PageSize);
}
return address;
}
}
return 0;
}
private void FreeBlock(ulong block, int index)
{
do
{
block = _blocks[index++].PushBlock(block);
}
while (block != 0);
}
public void Free(ulong address, ulong pagesCount)
{
if (pagesCount == 0)
{
return;
}
int bigIndex = _blocksCount - 1;
ulong start = address;
ulong end = address + pagesCount * KPageTableBase.PageSize;
ulong beforeStart = start;
ulong beforeEnd = start;
ulong afterStart = end;
ulong afterEnd = end;
while (bigIndex >= 0)
{
ulong blockSize = _blocks[bigIndex].Size;
ulong bigStart = BitUtils.AlignUp (start, blockSize);
ulong bigEnd = BitUtils.AlignDown(end, blockSize);
if (bigStart < bigEnd)
{
for (ulong block = bigStart; block < bigEnd; block += blockSize)
{
FreeBlock(block, bigIndex);
}
beforeEnd = bigStart;
afterStart = bigEnd;
break;
}
bigIndex--;
}
for (int i = bigIndex - 1; i >= 0; i--)
{
ulong blockSize = _blocks[i].Size;
while (beforeStart + blockSize <= beforeEnd)
{
beforeEnd -= blockSize;
FreeBlock(beforeEnd, i);
}
}
for (int i = bigIndex - 1; i >= 0; i--)
{
ulong blockSize = _blocks[i].Size;
while (afterStart + blockSize <= afterEnd)
{
FreeBlock(afterStart, i);
afterStart += blockSize;
}
}
}
public static int GetAlignedBlockIndex(ulong pagesCount, ulong alignPages)
{
ulong targetPages = Math.Max(pagesCount, alignPages);
for (int i = 0; i < _memoryBlockPageShifts.Length; i++)
{
if (targetPages <= GetBlockPagesCount(i))
{
return i;
}
}
return -1;
}
public static int GetBlockIndex(ulong pagesCount)
{
for (int i = _memoryBlockPageShifts.Length - 1; i >= 0; i--)
{
if (pagesCount >= GetBlockPagesCount(i))
{
return i;
}
}
return -1;
}
public static ulong GetBlockSize(int index)
{
return 1UL << _memoryBlockPageShifts[index];
}
public static ulong GetBlockPagesCount(int index)
{
return GetBlockSize(index) / KPageTableBase.PageSize;
}
private static int CalculateManagementOverheadSize(ulong regionSize, int[] blockShifts)
{
int overheadSize = 0;
for (int i = 0; i < blockShifts.Length; i++)
{
int currBlockShift = blockShifts[i];
int nextBlockShift = i != blockShifts.Length - 1 ? blockShifts[i + 1] : 0;
overheadSize += Block.CalculateManagementOverheadSize(regionSize, currBlockShift, nextBlockShift);
}
return BitUtils.AlignUp(overheadSize, KPageTableBase.PageSize);
}
}
}

6
Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs
View file

@ -555,7 +555,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
{ {
KMemoryRegionManager region = GetMemoryRegionManager(); KMemoryRegionManager region = GetMemoryRegionManager();
KernelResult result = region.AllocatePages(pagesCount, _aslrDisabled, out KPageList pageList); KernelResult result = region.AllocatePages(out KPageList pageList, pagesCount);
if (result != KernelResult.Success) if (result != KernelResult.Success)
{ {
@ -712,7 +712,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
KMemoryRegionManager region = GetMemoryRegionManager(); KMemoryRegionManager region = GetMemoryRegionManager();
KernelResult result = region.AllocatePages(pagesCount, _aslrDisabled, out KPageList pageList); KernelResult result = region.AllocatePages(out KPageList pageList, pagesCount);
using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager)); using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager));
@ -1276,7 +1276,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
KMemoryRegionManager region = GetMemoryRegionManager(); KMemoryRegionManager region = GetMemoryRegionManager();
KernelResult result = region.AllocatePages(remainingPages, _aslrDisabled, out KPageList pageList); KernelResult result = region.AllocatePages(out KPageList pageList, remainingPages);
using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager)); using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager));

2
Ryujinx.HLE/HOS/ProgramLoader.cs
View file

@ -90,7 +90,7 @@ namespace Ryujinx.HLE.HOS
KMemoryRegionManager region = context.MemoryManager.MemoryRegions[(int)memoryRegion]; KMemoryRegionManager region = context.MemoryManager.MemoryRegions[(int)memoryRegion];
KernelResult result = region.AllocatePages((ulong)codePagesCount, false, out KPageList pageList); KernelResult result = region.AllocatePages(out KPageList pageList, (ulong)codePagesCount);
if (result != KernelResult.Success) if (result != KernelResult.Success)
{ {