Implement lazy flush-on-read for Buffers (SSBO/Copy) (#1790)
* Initial implementation of buffer flush (VERY WIP) * Host shaders need to be rebuilt for the SSBO write flag. * New approach with reserved regions and gl sync * Fix a ton of buffer issues. * Remove unused buffer unmapped behaviour * Revert "Remove unused buffer unmapped behaviour" This reverts commit f1700e52fb8760180ac5e0987a07d409d1e70ece. * Delete modified ranges on unmap Fixes potential crashes in Super Smash Bros, where a previously modified range could lie on either side of an unmap. * Cache some more delegates. * Dispose Sync on Close * Also create host sync for GPFifo syncpoint increment. * Copy buffer optimization, add docs * Fix race condition with OpenGL Sync * Enable read tracking on CommandBuffer, insert syncpoint on WaitForIdle * Performance: Only flush individual pages of SSBO at a time This avoids flushing large amounts of data when only a small amount is actually used. * Signal Modified rather than flushing after clear * Fix some docs and code style. * Introduce a new test for tracking memory protection. Sucessfully demonstrates that the bug causing write protection to be cleared by a read action has been fixed. (these tests fail on master) * Address Comments * Add host sync for SetReference This ensures that any indirect draws will correctly flush any related buffer data written before them. Fixes some flashing and misplaced world geometry in MH rise. * Make PageAlign static * Re-enable read tracking, for reads.
This commit is contained in:
riperiperi
GitHub
parent
c4f56c5704
commit
a1f77a5b6a
28 changed files with 1073 additions and 57 deletions
|
|
@ -1,6 +1,7 @@
|
|||
using Ryujinx.Cpu.Tracking;
|
||||
using Ryujinx.Graphics.GAL;
|
||||
using Ryujinx.Memory.Range;
|
||||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Graphics.Gpu.Memory
|
||||
|
|
@ -34,12 +35,28 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
public ulong EndAddress => Address + Size;
|
||||
|
||||
/// <summary>
|
||||
/// Ranges of the buffer that have been modified on the GPU.
|
||||
/// Ranges defined here cannot be updated from CPU until a CPU waiting sync point is reached.
|
||||
/// Then, write tracking will signal, wait for GPU sync (generated at the syncpoint) and flush these regions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is null until at least one modification occurs.
|
||||
/// </remarks>
|
||||
private BufferModifiedRangeList _modifiedRanges = null;
|
||||
|
||||
private CpuMultiRegionHandle _memoryTrackingGranular;
|
||||
|
||||
private CpuRegionHandle _memoryTracking;
|
||||
|
||||
private readonly RegionSignal _externalFlushDelegate;
|
||||
private readonly Action<ulong, ulong> _loadDelegate;
|
||||
private readonly Action<ulong, ulong> _modifiedDelegate;
|
||||
|
||||
private int _sequenceNumber;
|
||||
|
||||
private bool _useGranular;
|
||||
private bool _syncActionRegistered;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the buffer.
|
||||
|
|
@ -66,6 +83,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
_memoryTracking = context.PhysicalMemory.BeginTracking(address, size);
|
||||
}
|
||||
|
||||
_externalFlushDelegate = new RegionSignal(ExternalFlush);
|
||||
_loadDelegate = new Action<ulong, ulong>(LoadRegion);
|
||||
_modifiedDelegate = new Action<ulong, ulong>(RegionModified);
|
||||
}
|
||||
|
||||
|
|
@ -116,12 +135,131 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
if (_memoryTracking.Dirty && _context.SequenceNumber != _sequenceNumber)
|
||||
{
|
||||
_memoryTracking.Reprotect();
|
||||
_context.Renderer.SetBufferData(Handle, 0, _context.PhysicalMemory.GetSpan(Address, (int)Size));
|
||||
|
||||
if (_modifiedRanges != null)
|
||||
{
|
||||
_modifiedRanges.ExcludeModifiedRegions(Address, Size, _loadDelegate);
|
||||
}
|
||||
else
|
||||
{
|
||||
_context.Renderer.SetBufferData(Handle, 0, _context.PhysicalMemory.GetSpan(Address, (int)Size));
|
||||
}
|
||||
|
||||
_sequenceNumber = _context.SequenceNumber;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensure that the modified range list exists.
|
||||
/// </summary>
|
||||
private void EnsureRangeList()
|
||||
{
|
||||
if (_modifiedRanges == null)
|
||||
{
|
||||
_modifiedRanges = new BufferModifiedRangeList(_context);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Signal that the given region of the buffer has been modified.
|
||||
/// </summary>
|
||||
/// <param name="address">The start address of the modified region</param>
|
||||
/// <param name="size">The size of the modified region</param>
|
||||
public void SignalModified(ulong address, ulong size)
|
||||
{
|
||||
EnsureRangeList();
|
||||
|
||||
_modifiedRanges.SignalModified(address, size);
|
||||
|
||||
if (!_syncActionRegistered)
|
||||
{
|
||||
_context.RegisterSyncAction(SyncAction);
|
||||
_syncActionRegistered = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Indicate that mofifications in a given region of this buffer have been overwritten.
|
||||
/// </summary>
|
||||
/// <param name="address">The start address of the region</param>
|
||||
/// <param name="size">The size of the region</param>
|
||||
public void ClearModified(ulong address, ulong size)
|
||||
{
|
||||
if (_modifiedRanges != null)
|
||||
{
|
||||
_modifiedRanges.Clear(address, size);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Action to be performed when a syncpoint is reached after modification.
|
||||
/// This will register read/write tracking to flush the buffer from GPU when its memory is used.
|
||||
/// </summary>
|
||||
private void SyncAction()
|
||||
{
|
||||
_syncActionRegistered = false;
|
||||
|
||||
if (_useGranular)
|
||||
{
|
||||
_modifiedRanges.GetRanges(Address, Size, (address, size) =>
|
||||
{
|
||||
_memoryTrackingGranular.RegisterAction(address, size, _externalFlushDelegate);
|
||||
SynchronizeMemory(address, size);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
_memoryTracking.RegisterAction(_externalFlushDelegate);
|
||||
SynchronizeMemory(Address, Size);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inherit modified ranges from another buffer.
|
||||
/// </summary>
|
||||
/// <param name="from">The buffer to inherit from</param>
|
||||
public void InheritModifiedRanges(Buffer from)
|
||||
{
|
||||
if (from._modifiedRanges != null)
|
||||
{
|
||||
if (from._syncActionRegistered && !_syncActionRegistered)
|
||||
{
|
||||
_context.RegisterSyncAction(SyncAction);
|
||||
_syncActionRegistered = true;
|
||||
}
|
||||
|
||||
EnsureRangeList();
|
||||
_modifiedRanges.InheritRanges(from._modifiedRanges, (ulong address, ulong size) =>
|
||||
{
|
||||
if (_useGranular)
|
||||
{
|
||||
_memoryTrackingGranular.RegisterAction(address, size, _externalFlushDelegate);
|
||||
}
|
||||
else
|
||||
{
|
||||
_memoryTracking.RegisterAction(_externalFlushDelegate);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine if a given region of the buffer has been modified, and must be flushed.
|
||||
/// </summary>
|
||||
/// <param name="address">The start address of the region</param>
|
||||
/// <param name="size">The size of the region</param>
|
||||
/// <returns></returns>
|
||||
public bool IsModified(ulong address, ulong size)
|
||||
{
|
||||
if (_modifiedRanges != null)
|
||||
{
|
||||
return _modifiedRanges.HasRange(address, size);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Indicate that a region of the buffer was modified, and must be loaded from memory.
|
||||
/// </summary>
|
||||
|
|
@ -141,6 +279,23 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
mSize = maxSize;
|
||||
}
|
||||
|
||||
if (_modifiedRanges != null)
|
||||
{
|
||||
_modifiedRanges.ExcludeModifiedRegions(mAddress, mSize, _loadDelegate);
|
||||
}
|
||||
else
|
||||
{
|
||||
LoadRegion(mAddress, mSize);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Load a region of the buffer from memory.
|
||||
/// </summary>
|
||||
/// <param name="mAddress">Start address of the modified region</param>
|
||||
/// <param name="mSize">Size of the modified region</param>
|
||||
private void LoadRegion(ulong mAddress, ulong mSize)
|
||||
{
|
||||
int offset = (int)(mAddress - Address);
|
||||
|
||||
_context.Renderer.SetBufferData(Handle, offset, _context.PhysicalMemory.GetSpan(mAddress, (int)mSize));
|
||||
|
|
@ -172,15 +327,62 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
_context.PhysicalMemory.WriteUntracked(address, data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Align a given address and size region to page boundaries.
|
||||
/// </summary>
|
||||
/// <param name="address">The start address of the region</param>
|
||||
/// <param name="size">The size of the region</param>
|
||||
/// <returns>The page aligned address and size</returns>
|
||||
private static (ulong address, ulong size) PageAlign(ulong address, ulong size)
|
||||
{
|
||||
ulong pageMask = MemoryManager.PageMask;
|
||||
ulong rA = address & ~pageMask;
|
||||
ulong rS = ((address + size + pageMask) & ~pageMask) - rA;
|
||||
return (rA, rS);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flush modified ranges of the buffer from another thread.
|
||||
/// This will flush all modifications made before the active SyncNumber was set, and may block to wait for GPU sync.
|
||||
/// </summary>
|
||||
/// <param name="address">Address of the memory action</param>
|
||||
/// <param name="size">Size in bytes</param>
|
||||
public void ExternalFlush(ulong address, ulong size)
|
||||
{
|
||||
_context.Renderer.BackgroundContextAction(() =>
|
||||
{
|
||||
var ranges = _modifiedRanges;
|
||||
|
||||
if (ranges != null)
|
||||
{
|
||||
(address, size) = PageAlign(address, size);
|
||||
ranges.WaitForAndGetRanges(address, size, Flush);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Called when part of the memory for this buffer has been unmapped.
|
||||
/// Calls are from non-GPU threads.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address of the unmapped region</param>
|
||||
/// <param name="size">Size of the unmapped region</param>
|
||||
public void Unmapped(ulong address, ulong size)
|
||||
{
|
||||
_modifiedRanges?.Clear(address, size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the host buffer.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_context.Renderer.DeleteBuffer(Handle);
|
||||
_modifiedRanges?.Clear();
|
||||
|
||||
_memoryTrackingGranular?.Dispose();
|
||||
_memoryTracking?.Dispose();
|
||||
|
||||
_context.Renderer.DeleteBuffer(Handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
using Ryujinx.Graphics.Shader;
|
||||
|
||||
namespace Ryujinx.Graphics.Gpu.Memory
|
||||
{
|
||||
/// <summary>
|
||||
|
|
@ -15,15 +17,22 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
public ulong Size { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Buffer usage flags.
|
||||
/// </summary>
|
||||
public BufferUsageFlags Flags { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new buffer region.
|
||||
/// </summary>
|
||||
/// <param name="address">Region address</param>
|
||||
/// <param name="size">Region size</param>
|
||||
public BufferBounds(ulong address, ulong size)
|
||||
/// <param name="flags">Buffer usage flags</param>
|
||||
public BufferBounds(ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
|
||||
{
|
||||
Address = address;
|
||||
Size = size;
|
||||
Flags = flags;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -68,9 +68,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// <param name="index">Buffer slot</param>
|
||||
/// <param name="address">Region virtual address</param>
|
||||
/// <param name="size">Region size in bytes</param>
|
||||
public void SetBounds(int index, ulong address, ulong size)
|
||||
/// <param name="flags">Buffer usage flags</param>
|
||||
public void SetBounds(int index, ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
|
||||
{
|
||||
Buffers[index] = new BufferBounds(address, size);
|
||||
Buffers[index] = new BufferBounds(address, size, flags);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -219,7 +220,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// <param name="index">Index of the storage buffer</param>
|
||||
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
|
||||
/// <param name="size">Size in bytes of the storage buffer</param>
|
||||
public void SetComputeStorageBuffer(int index, ulong gpuVa, ulong size)
|
||||
/// <param name="flags">Buffer usage flags</param>
|
||||
public void SetComputeStorageBuffer(int index, ulong gpuVa, ulong size, BufferUsageFlags flags)
|
||||
{
|
||||
size += gpuVa & ((ulong)_context.Capabilities.StorageBufferOffsetAlignment - 1);
|
||||
|
||||
|
|
@ -227,7 +229,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
|
||||
ulong address = TranslateAndCreateBuffer(gpuVa, size);
|
||||
|
||||
_cpStorageBuffers.SetBounds(index, address, size);
|
||||
_cpStorageBuffers.SetBounds(index, address, size, flags);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -238,7 +240,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// <param name="index">Index of the storage buffer</param>
|
||||
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
|
||||
/// <param name="size">Size in bytes of the storage buffer</param>
|
||||
public void SetGraphicsStorageBuffer(int stage, int index, ulong gpuVa, ulong size)
|
||||
/// <param name="flags">Buffer usage flags</param>
|
||||
public void SetGraphicsStorageBuffer(int stage, int index, ulong gpuVa, ulong size, BufferUsageFlags flags)
|
||||
{
|
||||
size += gpuVa & ((ulong)_context.Capabilities.StorageBufferOffsetAlignment - 1);
|
||||
|
||||
|
|
@ -252,7 +255,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
_gpStorageBuffersDirty = true;
|
||||
}
|
||||
|
||||
_gpStorageBuffers[stage].SetBounds(index, address, size);
|
||||
_gpStorageBuffers[stage].SetBounds(index, address, size, flags);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -385,6 +388,30 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
return mask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles removal of buffers written to a memory region being unmapped.
|
||||
/// </summary>
|
||||
/// <param name="sender">Sender object</param>
|
||||
/// <param name="e">Event arguments</param>
|
||||
public void MemoryUnmappedHandler(object sender, UnmapEventArgs e)
|
||||
{
|
||||
Buffer[] overlaps = new Buffer[10];
|
||||
int overlapCount;
|
||||
|
||||
ulong address = _context.MemoryManager.Translate(e.Address);
|
||||
ulong size = e.Size;
|
||||
|
||||
lock (_buffers)
|
||||
{
|
||||
overlapCount = _buffers.FindOverlaps(address, size, ref overlaps);
|
||||
}
|
||||
|
||||
for (int i = 0; i < overlapCount; i++)
|
||||
{
|
||||
overlaps[i].Unmapped(address, size);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs address translation of the GPU virtual address, and creates a
|
||||
/// new buffer, if needed, for the specified range.
|
||||
|
|
@ -443,7 +470,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// <param name="size">Size in bytes of the buffer</param>
|
||||
private void CreateBufferAligned(ulong address, ulong size)
|
||||
{
|
||||
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
|
||||
int overlapsCount;
|
||||
|
||||
lock (_buffers)
|
||||
{
|
||||
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
|
||||
}
|
||||
|
||||
if (overlapsCount != 0)
|
||||
{
|
||||
|
|
@ -463,15 +495,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
address = Math.Min(address, buffer.Address);
|
||||
endAddress = Math.Max(endAddress, buffer.EndAddress);
|
||||
|
||||
buffer.SynchronizeMemory(buffer.Address, buffer.Size);
|
||||
|
||||
_buffers.Remove(buffer);
|
||||
lock (_buffers)
|
||||
{
|
||||
_buffers.Remove(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
Buffer newBuffer = new Buffer(_context, address, endAddress - address);
|
||||
newBuffer.SynchronizeMemory(address, endAddress - address);
|
||||
|
||||
_buffers.Add(newBuffer);
|
||||
lock (_buffers)
|
||||
{
|
||||
_buffers.Add(newBuffer);
|
||||
}
|
||||
|
||||
for (int index = 0; index < overlapsCount; index++)
|
||||
{
|
||||
|
|
@ -479,7 +515,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
|
||||
int dstOffset = (int)(buffer.Address - newBuffer.Address);
|
||||
|
||||
buffer.SynchronizeMemory(buffer.Address, buffer.Size);
|
||||
|
||||
buffer.CopyTo(newBuffer, dstOffset);
|
||||
newBuffer.InheritModifiedRanges(buffer);
|
||||
|
||||
buffer.Dispose();
|
||||
}
|
||||
|
|
@ -493,7 +532,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
// No overlap, just create a new buffer.
|
||||
Buffer buffer = new Buffer(_context, address, size);
|
||||
|
||||
_buffers.Add(buffer);
|
||||
lock (_buffers)
|
||||
{
|
||||
_buffers.Add(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
ShrinkOverlapsBufferIfNeeded();
|
||||
|
|
@ -549,7 +591,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
|
||||
if (bounds.Address != 0)
|
||||
{
|
||||
sRanges[bindingInfo.Binding] = GetBufferRange(bounds.Address, bounds.Size);
|
||||
sRanges[bindingInfo.Binding] = GetBufferRange(bounds.Address, bounds.Size, bounds.Flags.HasFlag(BufferUsageFlags.Write));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -722,7 +764,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
|
||||
if (bounds.Address != 0)
|
||||
{
|
||||
ranges[bindingInfo.Binding] = GetBufferRange(bounds.Address, bounds.Size);
|
||||
ranges[bindingInfo.Binding] = GetBufferRange(bounds.Address, bounds.Size, bounds.Flags.HasFlag(BufferUsageFlags.Write));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -818,7 +860,17 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
dstOffset,
|
||||
(int)size);
|
||||
|
||||
dstBuffer.Flush(dstAddress, size);
|
||||
if (srcBuffer.IsModified(srcAddress, size))
|
||||
{
|
||||
dstBuffer.SignalModified(dstAddress, size);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Optimization: If the data being copied is already in memory, then copy it directly instead of flushing from GPU.
|
||||
|
||||
dstBuffer.ClearModified(dstAddress, size);
|
||||
_context.PhysicalMemory.WriteUntracked(dstAddress, _context.PhysicalMemory.GetSpan(srcAddress, (int)size));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -840,7 +892,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
|
||||
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)size, value);
|
||||
|
||||
buffer.Flush(address, size);
|
||||
buffer.SignalModified(address, size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -848,10 +900,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
/// <param name="address">Start address of the memory range</param>
|
||||
/// <param name="size">Size in bytes of the memory range</param>
|
||||
/// <param name="write">Whether the buffer will be written to by this use</param>
|
||||
/// <returns>The buffer sub-range for the given range</returns>
|
||||
private BufferRange GetBufferRange(ulong address, ulong size)
|
||||
private BufferRange GetBufferRange(ulong address, ulong size, bool write = false)
|
||||
{
|
||||
return GetBuffer(address, size).GetRange(address, size);
|
||||
return GetBuffer(address, size, write).GetRange(address, size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -860,20 +913,32 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
/// <param name="address">Start address of the memory range</param>
|
||||
/// <param name="size">Size in bytes of the memory range</param>
|
||||
/// <param name="write">Whether the buffer will be written to by this use</param>
|
||||
/// <returns>The buffer where the range is fully contained</returns>
|
||||
private Buffer GetBuffer(ulong address, ulong size)
|
||||
private Buffer GetBuffer(ulong address, ulong size, bool write = false)
|
||||
{
|
||||
Buffer buffer;
|
||||
|
||||
if (size != 0)
|
||||
{
|
||||
buffer = _buffers.FindFirstOverlap(address, size);
|
||||
lock (_buffers)
|
||||
{
|
||||
buffer = _buffers.FindFirstOverlap(address, size);
|
||||
}
|
||||
|
||||
buffer.SynchronizeMemory(address, size);
|
||||
|
||||
if (write)
|
||||
{
|
||||
buffer.SignalModified(address, size);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer = _buffers.FindFirstOverlap(address, 1);
|
||||
lock (_buffers)
|
||||
{
|
||||
buffer = _buffers.FindFirstOverlap(address, 1);
|
||||
}
|
||||
}
|
||||
|
||||
return buffer;
|
||||
|
|
@ -888,7 +953,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
{
|
||||
if (size != 0)
|
||||
{
|
||||
Buffer buffer = _buffers.FindFirstOverlap(address, size);
|
||||
Buffer buffer;
|
||||
|
||||
lock (_buffers)
|
||||
{
|
||||
buffer = _buffers.FindFirstOverlap(address, size);
|
||||
}
|
||||
|
||||
buffer.SynchronizeMemory(address, size);
|
||||
}
|
||||
|
|
@ -900,9 +970,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (Buffer buffer in _buffers)
|
||||
lock (_buffers)
|
||||
{
|
||||
buffer.Dispose();
|
||||
foreach (Buffer buffer in _buffers)
|
||||
{
|
||||
buffer.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
367
Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
Normal file
367
Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
using Ryujinx.Memory.Range;
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace Ryujinx.Graphics.Gpu.Memory
|
||||
{
|
||||
/// <summary>
|
||||
/// A range within a buffer that has been modified by the GPU.
|
||||
/// </summary>
|
||||
class BufferModifiedRange : IRange
|
||||
{
|
||||
/// <summary>
|
||||
/// Start address of the range in guest memory.
|
||||
/// </summary>
|
||||
public ulong Address { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Size of the range in bytes.
|
||||
/// </summary>
|
||||
public ulong Size { get; }
|
||||
|
||||
/// <summary>
|
||||
/// End address of the range in guest memory.
|
||||
/// </summary>
|
||||
public ulong EndAddress => Address + Size;
|
||||
|
||||
/// <summary>
|
||||
/// The GPU sync number at the time of the last modification.
|
||||
/// </summary>
|
||||
public ulong SyncNumber { get; internal set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of a modified range.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
/// <param name="syncNumber">The GPU sync number at the time of creation</param>
|
||||
public BufferModifiedRange(ulong address, ulong size, ulong syncNumber)
|
||||
{
|
||||
Address = address;
|
||||
Size = size;
|
||||
SyncNumber = syncNumber;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a given range overlaps with the modified range.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address of the range</param>
|
||||
/// <param name="size">Size in bytes of the range</param>
|
||||
/// <returns>True if the range overlaps, false otherwise</returns>
|
||||
public bool OverlapsWith(ulong address, ulong size)
|
||||
{
|
||||
return Address < address + size && address < EndAddress;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A structure used to track GPU modified ranges within a buffer.
|
||||
/// </summary>
|
||||
class BufferModifiedRangeList : RangeList<BufferModifiedRange>
|
||||
{
|
||||
private GpuContext _context;
|
||||
|
||||
private object _lock = new object();
|
||||
|
||||
// The list can be accessed from both the GPU thread, and a background thread.
|
||||
private BufferModifiedRange[] _foregroundOverlaps = new BufferModifiedRange[1];
|
||||
private BufferModifiedRange[] _backgroundOverlaps = new BufferModifiedRange[1];
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of a modified range list.
|
||||
/// </summary>
|
||||
/// <param name="context">GPU context that the buffer range list belongs to</param>
|
||||
public BufferModifiedRangeList(GpuContext context)
|
||||
{
|
||||
_context = context;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Given an input range, calls the given action with sub-ranges which exclude any of the modified regions.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address of the query range</param>
|
||||
/// <param name="size">Size of the query range in bytes</param>
|
||||
/// <param name="action">Action to perform for each remaining sub-range of the input range</param>
|
||||
public void ExcludeModifiedRegions(ulong address, ulong size, Action<ulong, ulong> action)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
// Slices a given region using the modified regions in the list. Calls the action for the new slices.
|
||||
int count = FindOverlapsNonOverlapping(address, size, ref _foregroundOverlaps);
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
BufferModifiedRange overlap = _foregroundOverlaps[i];
|
||||
|
||||
if (overlap.Address > address)
|
||||
{
|
||||
// The start of the remaining region is uncovered by this overlap. Call the action for it.
|
||||
action(address, overlap.Address - address);
|
||||
}
|
||||
|
||||
// Remaining region is after this overlap.
|
||||
size -= overlap.EndAddress - address;
|
||||
address = overlap.EndAddress;
|
||||
}
|
||||
|
||||
if ((long)size > 0)
|
||||
{
|
||||
// If there is any region left after removing the overlaps, signal it.
|
||||
action(address, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Signal that a region of the buffer has been modified, and add the new region to the range list.
|
||||
/// Any overlapping ranges will be (partially) removed.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address of the modified region</param>
|
||||
/// <param name="size">Size of the modified region in bytes</param>
|
||||
public void SignalModified(ulong address, ulong size)
|
||||
{
|
||||
// Must lock, as this can affect flushes from the background thread.
|
||||
lock (_lock)
|
||||
{
|
||||
// We may overlap with some existing modified regions. They must be cut into by the new entry.
|
||||
int count = FindOverlapsNonOverlapping(address, size, ref _foregroundOverlaps);
|
||||
|
||||
ulong endAddress = address + size;
|
||||
ulong syncNumber = _context.SyncNumber;
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
// The overlaps must be removed or split.
|
||||
|
||||
BufferModifiedRange overlap = _foregroundOverlaps[i];
|
||||
|
||||
if (overlap.Address == address && overlap.Size == size)
|
||||
{
|
||||
// Region already exists. Just update the existing sync number.
|
||||
overlap.SyncNumber = syncNumber;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Remove(overlap);
|
||||
|
||||
if (overlap.Address < address && overlap.EndAddress > address)
|
||||
{
|
||||
// A split item must be created behind this overlap.
|
||||
|
||||
Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber));
|
||||
}
|
||||
|
||||
if (overlap.Address < endAddress && overlap.EndAddress > endAddress)
|
||||
{
|
||||
// A split item must be created after this overlap.
|
||||
|
||||
Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber));
|
||||
}
|
||||
}
|
||||
|
||||
Add(new BufferModifiedRange(address, size, syncNumber));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets modified ranges within the specified region, and then fires the given action for each range individually.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address to query</param>
|
||||
/// <param name="size">Size to query</param>
|
||||
/// <param name="rangeAction">The action to call for each modified range</param>
|
||||
public void GetRanges(ulong address, ulong size, Action<ulong, ulong> rangeAction)
|
||||
{
|
||||
int count = 0;
|
||||
|
||||
// Range list must be consistent for this operation.
|
||||
lock (_lock)
|
||||
{
|
||||
count = FindOverlapsNonOverlapping(address, size, ref _foregroundOverlaps);
|
||||
}
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
BufferModifiedRange overlap = _foregroundOverlaps[i];
|
||||
rangeAction(overlap.Address, overlap.Size);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queries if a range exists within the specified region.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address to query</param>
|
||||
/// <param name="size">Size to query</param>
|
||||
/// <returns>True if a range exists in the specified region, false otherwise</returns>
|
||||
public bool HasRange(ulong address, ulong size)
|
||||
{
|
||||
// Range list must be consistent for this operation.
|
||||
lock (_lock)
|
||||
{
|
||||
return FindOverlapsNonOverlapping(address, size, ref _foregroundOverlaps) > 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets modified ranges within the specified region, waits on ones from a previous sync number,
|
||||
/// and then fires the given action for each range individually.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This function assumes it is called from the background thread.
|
||||
/// Modifications from the current sync number are ignored because the guest should not expect them to be available yet.
|
||||
/// They will remain reserved, so that any data sync prioritizes the data in the GPU.
|
||||
/// </remarks>
|
||||
/// <param name="address">Start address to query</param>
|
||||
/// <param name="size">Size to query</param>
|
||||
/// <param name="rangeAction">The action to call for each modified range</param>
|
||||
public void WaitForAndGetRanges(ulong address, ulong size, Action<ulong, ulong> rangeAction)
|
||||
{
|
||||
ulong endAddress = address + size;
|
||||
ulong currentSync = _context.SyncNumber;
|
||||
|
||||
int rangeCount = 0;
|
||||
|
||||
// Range list must be consistent for this operation
|
||||
lock (_lock)
|
||||
{
|
||||
rangeCount = FindOverlapsNonOverlapping(address, size, ref _backgroundOverlaps);
|
||||
}
|
||||
|
||||
if (rangeCount == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// First, determine which syncpoint to wait on.
|
||||
// This is the latest syncpoint that is not equal to the current sync.
|
||||
|
||||
long highestDiff = long.MinValue;
|
||||
|
||||
for (int i = 0; i < rangeCount; i++)
|
||||
{
|
||||
BufferModifiedRange overlap = _backgroundOverlaps[i];
|
||||
|
||||
long diff = (long)(overlap.SyncNumber - currentSync);
|
||||
|
||||
if (diff < 0 && diff > highestDiff)
|
||||
{
|
||||
highestDiff = diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (highestDiff == long.MinValue)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait for the syncpoint.
|
||||
_context.Renderer.WaitSync(currentSync + (ulong)highestDiff);
|
||||
|
||||
// Flush and remove all regions with the older syncpoint.
|
||||
lock (_lock)
|
||||
{
|
||||
for (int i = 0; i < rangeCount; i++)
|
||||
{
|
||||
BufferModifiedRange overlap = _backgroundOverlaps[i];
|
||||
|
||||
long diff = (long)(overlap.SyncNumber - currentSync);
|
||||
|
||||
if (diff <= highestDiff)
|
||||
{
|
||||
ulong clampAddress = Math.Max(address, overlap.Address);
|
||||
ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
|
||||
|
||||
ClearPart(overlap, clampAddress, clampEnd);
|
||||
|
||||
rangeAction(clampAddress, clampEnd - clampAddress);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inherit ranges from another modified range list.
|
||||
/// </summary>
|
||||
/// <param name="ranges">The range list to inherit from</param>
|
||||
/// <param name="rangeAction">The action to call for each modified range</param>
|
||||
public void InheritRanges(BufferModifiedRangeList ranges, Action<ulong, ulong> rangeAction)
|
||||
{
|
||||
BufferModifiedRange[] inheritRanges;
|
||||
|
||||
lock (ranges._lock)
|
||||
{
|
||||
inheritRanges = ranges.ToArray();
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (BufferModifiedRange range in inheritRanges)
|
||||
{
|
||||
Add(range);
|
||||
}
|
||||
}
|
||||
|
||||
ulong currentSync = _context.SyncNumber;
|
||||
foreach (BufferModifiedRange range in inheritRanges)
|
||||
{
|
||||
if (range.SyncNumber != currentSync)
|
||||
{
|
||||
rangeAction(range.Address, range.Size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress)
|
||||
{
|
||||
Remove(overlap);
|
||||
|
||||
// If the overlap extends outside of the clear range, make sure those parts still exist.
|
||||
|
||||
if (overlap.Address < address)
|
||||
{
|
||||
Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber));
|
||||
}
|
||||
|
||||
if (overlap.EndAddress > endAddress)
|
||||
{
|
||||
Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear modified ranges within the specified area.
|
||||
/// </summary>
|
||||
/// <param name="address">Start address to clear</param>
|
||||
/// <param name="size">Size to clear</param>
|
||||
public void Clear(ulong address, ulong size)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
// This function can be called from any thread, so it cannot use the arrays for background or foreground.
|
||||
BufferModifiedRange[] toClear = new BufferModifiedRange[1];
|
||||
|
||||
int rangeCount = FindOverlapsNonOverlapping(address, size, ref toClear);
|
||||
|
||||
ulong endAddress = address + size;
|
||||
|
||||
for (int i = 0; i < rangeCount; i++)
|
||||
{
|
||||
BufferModifiedRange overlap = toClear[i];
|
||||
|
||||
ClearPart(overlap, address, endAddress);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clear all modified ranges.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
Items.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -61,6 +61,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
/// <param name="va">GPU virtual address where the data is located</param>
|
||||
/// <param name="size">Size of the data</param>
|
||||
/// <param name="tracked">True if read tracking is triggered on the span</param>
|
||||
/// <returns>The span of the data at the specified memory location</returns>
|
||||
public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue