Implement GPU syncpoints (#980)

* Implement GPU syncpoints

This adds support for GPU syncpoints on the GPU backend & nvservices.

Everything that was implemented here is based on my researches,
hardware testing of the GM20B and reversing of nvservices (8.1.0).

Thanks to @fincs for the informations about some behaviours of the pusher
and for the initial informations about syncpoints.

* syncpoint: address gdkchan's comments

* Add some missing logic to handle SubmitGpfifo correctly

* Handle the NV event API correctly

* evnt => hostEvent

* Finish addressing gdkchan's comments

* nvservices: write the output buffer even when an error is returned

* dma pusher: Implemnet prefetch barrier

lso fix when the commands should be prefetch.

* Partially fix prefetch barrier

* Add a missing syncpoint check in QueryEvent of NvHostSyncPt

* Address Ac_K's comments and fix GetSyncpoint for ChannelResourcePolicy == Channel

* fix SyncptWait & SyncptWaitEx cmds logic

* Address ripinperi's comments

* Address gdkchan's comments

* Move user event management to the control channel

* Fix mm implementation, nvdec works again

* Address ripinperi's comments

* Address gdkchan's comments

* Implement nvhost-ctrl close accurately + make nvservices dispose channels when stopping the emulator

* Fix typo in MultiMediaOperationType

Ryujinx.Graphics.Gpu/ClassId.cs

@@ -9,6 +9,7 @@ namespace Ryujinx.Graphics.Gpu
         Engine3D            = 0xb197,
         EngineCompute       = 0xb1c0,
         EngineInline2Memory = 0xa140,
-        EngineDma           = 0xb0b5
+        EngineDma           = 0xb0b5,
+        EngineGpfifo        = 0xb06f
     }
 }

Ryujinx.Graphics.Gpu/DmaPusher.cs

@@ -1,4 +1,6 @@
+using System;
 using System.Collections.Concurrent;
+using System.Runtime.InteropServices;
 using System.Threading;
 
 namespace Ryujinx.Graphics.Gpu
@@ -8,10 +10,61 @@ namespace Ryujinx.Graphics.Gpu
     /// </summary>
     public class DmaPusher
     {
-        private ConcurrentQueue<ulong> _ibBuffer;
+        private ConcurrentQueue<CommandBuffer> _commandBufferQueue;
 
-        private ulong _dmaPut;
-        private ulong _dmaGet;
+        private enum CommandBufferType
+        {
+            Prefetch,
+            NoPrefetch,
+        }
+
+        private struct CommandBuffer
+        {
+            /// <summary>
+            /// The type of the command buffer.
+            /// </summary>
+            public CommandBufferType Type;
+
+            /// <summary>
+            /// Fetched data.
+            /// </summary>
+            public int[] Words;
+
+            /// <summary>
+            /// The GPFIFO entry address. (used in NoPrefetch mode)
+            /// </summary>
+            public ulong EntryAddress;
+
+            /// <summary>
+            /// The count of entries inside this GPFIFO entry.
+            /// </summary>
+            public uint EntryCount;
+
+            /// <summary>
+            /// Fetch the command buffer.
+            /// </summary>
+            public void Fetch(GpuContext context)
+            {
+                if (Words == null)
+                {
+                    Words = MemoryMarshal.Cast<byte, int>(context.MemoryAccessor.GetSpan(EntryAddress, EntryCount * 4)).ToArray();
+                }
+            }
+
+            /// <summary>
+            /// Read inside the command buffer.
+            /// </summary>
+            /// <param name="context">The GPU context</param>
+            /// <param name="index">The index inside the command buffer</param>
+            /// <returns>The value read</returns>
+            public int ReadAt(GpuContext context, int index)
+            {
+                return Words[index];
+            }
+        }
+
+        private CommandBuffer _currentCommandBuffer;
+        private int           _wordsPosition;
 
         /// <summary>
         /// Internal GPFIFO state.
@@ -32,9 +85,6 @@ namespace Ryujinx.Graphics.Gpu
         private bool _sliActive;
 
         private bool _ibEnable;
-        private bool _nonMain;
-
-        private ulong _dmaMGet;
 
         private GpuContext _context;
 
@@ -48,24 +98,91 @@ namespace Ryujinx.Graphics.Gpu
         {
             _context = context;
 
-            _ibBuffer = new ConcurrentQueue<ulong>();
-
             _ibEnable = true;
 
+            _commandBufferQueue = new ConcurrentQueue<CommandBuffer>();
+
             _event = new AutoResetEvent(false);
         }
 
         /// <summary>
-        /// Pushes a GPFIFO entry.
+        /// Signal the pusher that there are new entries to process.
         /// </summary>
-        /// <param name="entry">GPFIFO entry</param>
-        public void Push(ulong entry)
+        public void SignalNewEntries()
         {
-            _ibBuffer.Enqueue(entry);
-
             _event.Set();
         }
 
+        /// <summary>
+        /// Push a GPFIFO entry in the form of a prefetched command buffer.
+        /// It is intended to be used by nvservices to handle special cases.
+        /// </summary>
+        /// <param name="commandBuffer">The command buffer containing the prefetched commands</param>
+        public void PushHostCommandBuffer(int[] commandBuffer)
+        {
+            _commandBufferQueue.Enqueue(new CommandBuffer
+            {
+                Type         = CommandBufferType.Prefetch,
+                Words        = commandBuffer,
+                EntryAddress = ulong.MaxValue,
+                EntryCount   = (uint)commandBuffer.Length
+            });
+        }
+
+        /// <summary>
+        /// Create a CommandBuffer from a GPFIFO entry.
+        /// </summary>
+        /// <param name="entry">The GPFIFO entry</param>
+        /// <returns>A new CommandBuffer based on the GPFIFO entry</returns>
+        private CommandBuffer CreateCommandBuffer(ulong entry)
+        {
+            ulong length       = (entry >> 42) & 0x1fffff;
+            ulong startAddress = entry & 0xfffffffffc;
+
+            bool noPrefetch = (entry & (1UL << 63)) != 0;
+
+            CommandBufferType type = CommandBufferType.Prefetch;
+
+            if (noPrefetch)
+            {
+                type = CommandBufferType.NoPrefetch;
+            }
+
+            return new CommandBuffer
+            {
+                Type         = type,
+                Words        = null,
+                EntryAddress = startAddress,
+                EntryCount   = (uint)length
+            };
+        }
+
+        /// <summary>
+        /// Pushes GPFIFO entries.
+        /// </summary>
+        /// <param name="entries">GPFIFO entries</param>
+        public void PushEntries(ReadOnlySpan<ulong> entries)
+        {
+            bool beforeBarrier = true;
+
+            foreach (ulong entry in entries)
+            {
+                CommandBuffer commandBuffer = CreateCommandBuffer(entry);
+
+                if (beforeBarrier && commandBuffer.Type == CommandBufferType.Prefetch)
+                {
+                    commandBuffer.Fetch(_context);
+                }
+
+                if (commandBuffer.Type == CommandBufferType.NoPrefetch)
+                {
+                    beforeBarrier = false;
+                }
+
+                _commandBufferQueue.Enqueue(commandBuffer);
+            }
+        }
+
         /// <summary>
         /// Waits until commands are pushed to the FIFO.
         /// </summary>
@@ -89,16 +206,9 @@ namespace Ryujinx.Graphics.Gpu
         /// <returns>True if the FIFO still has commands to be processed, false otherwise</returns>
         private bool Step()
         {
-            if (_dmaGet != _dmaPut)
+            if (_wordsPosition != _currentCommandBuffer.EntryCount)
             {
-                int word = _context.MemoryAccessor.ReadInt32(_dmaGet);
-
-                _dmaGet += 4;
-
-                if (!_nonMain)
-                {
-                    _dmaMGet = _dmaGet;
-                }
+                int word = _currentCommandBuffer.ReadAt(_context, _wordsPosition++);
 
                 if (_state.LengthPending != 0)
                 {
@@ -170,14 +280,12 @@ namespace Ryujinx.Graphics.Gpu
                     }
                 }
             }
-            else if (_ibEnable && _ibBuffer.TryDequeue(out ulong entry))
+            else if (_ibEnable && _commandBufferQueue.TryDequeue(out CommandBuffer entry))
             {
-                ulong length = (entry >> 42) & 0x1fffff;
+                _currentCommandBuffer = entry;
+                _wordsPosition        = 0;
 
-                _dmaGet = entry & 0xfffffffffc;
-                _dmaPut = _dmaGet + length * 4;
-
-                _nonMain = (entry & (1UL << 41)) != 0;
+                _currentCommandBuffer.Fetch(_context);
             }
             else
             {

Ryujinx.Graphics.Gpu/Engine/MethodFifo.cs

@@ -0,0 +1,77 @@
+using Ryujinx.Graphics.Gpu.State;
+using System;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Gpu.Engine
+{
+    partial class Methods
+    {
+        /// <summary>
+        /// Waits for the GPU to be idle.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void WaitForIdle(GpuState state, int argument)
+        {
+            PerformDeferredDraws();
+
+            _context.Renderer.Pipeline.Barrier();
+        }
+
+        /// <summary>
+        /// Send macro code/data to the MME.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void SendMacroCodeData(GpuState state, int argument)
+        {
+            int macroUploadAddress = state.Get<int>(MethodOffset.MacroUploadAddress);
+
+            _context.Fifo.SendMacroCodeData(macroUploadAddress++, argument);
+
+            state.Write((int)MethodOffset.MacroUploadAddress, macroUploadAddress);
+        }
+
+        /// <summary>
+        /// Bind a macro index to a position for the MME.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void BindMacro(GpuState state, int argument)
+        {
+            int macroBindingIndex = state.Get<int>(MethodOffset.MacroBindingIndex);
+
+            _context.Fifo.BindMacro(macroBindingIndex++, argument);
+
+            state.Write((int)MethodOffset.MacroBindingIndex, macroBindingIndex);
+        }
+
+        public void SetMmeShadowRamControl(GpuState state, int argument)
+        {
+            _context.Fifo.SetMmeShadowRamControl((ShadowRamControl)argument);
+        }
+
+        /// <summary>
+        /// Apply a fence operation on a syncpoint.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void FenceAction(GpuState state, int argument)
+        {
+            uint threshold = state.Get<uint>(MethodOffset.FenceValue);
+
+            FenceActionOperation operation = (FenceActionOperation)(argument & 1);
+
+            uint syncpointId = (uint)(argument >> 8) & 0xFF;
+
+            if (operation == FenceActionOperation.Acquire)
+            {
+                _context.Synchronization.WaitOnSyncpoint(syncpointId, threshold, Timeout.InfiniteTimeSpan);
+            }
+            else if (operation == FenceActionOperation.Increment)
+            {
+                _context.Synchronization.IncrementSyncpoint(syncpointId);
+            }
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Engine/MethodIncrementSyncpoint.cs

@@ -0,0 +1,19 @@
+using Ryujinx.Graphics.Gpu.State;
+
+namespace Ryujinx.Graphics.Gpu.Engine
+{
+    partial class Methods
+    {
+        /// <summary>
+        /// Performs an incrementation on a syncpoint.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void IncrementSyncpoint(GpuState state, int argument)
+        {
+            uint syncpointId = (uint)(argument) & 0xFFFF;
+
+            _context.Synchronization.IncrementSyncpoint(syncpointId);
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Engine/MethodReport.cs

@@ -26,16 +26,16 @@ namespace Ryujinx.Graphics.Gpu.Engine
 
             switch (mode)
             {
-                case ReportMode.Semaphore: ReportSemaphore(state);     break;
-                case ReportMode.Counter:   ReportCounter(state, type); break;
+                case ReportMode.Release: ReleaseSemaphore(state);    break;
+                case ReportMode.Counter: ReportCounter(state, type); break;
             }
         }
 
         /// <summary>
-        /// Writes a GPU semaphore value to guest memory.
+        /// Writes (or Releases) a GPU semaphore value to guest memory.
         /// </summary>
         /// <param name="state">Current GPU state</param>
-        private void ReportSemaphore(GpuState state)
+        private void ReleaseSemaphore(GpuState state)
         {
             var rs = state.Get<ReportState>(MethodOffset.ReportState);
 

Ryujinx.Graphics.Gpu/Engine/Methods.cs

@@ -65,6 +65,8 @@ namespace Ryujinx.Graphics.Gpu.Engine
 
             state.RegisterCallback(MethodOffset.Dispatch, Dispatch);
 
+            state.RegisterCallback(MethodOffset.SyncpointAction, IncrementSyncpoint);
+
             state.RegisterCallback(MethodOffset.CopyBuffer,  CopyBuffer);
             state.RegisterCallback(MethodOffset.CopyTexture, CopyTexture);
 
@@ -94,6 +96,19 @@ namespace Ryujinx.Graphics.Gpu.Engine
             state.RegisterCallback(MethodOffset.UniformBufferBindFragment,       UniformBufferBindFragment);
         }
 
+        /// <summary>
+        /// Register callback for Fifo method calls that triggers an action on the GPFIFO.
+        /// </summary>
+        /// <param name="state">GPU state where the triggers will be registered</param>
+        public void RegisterCallbacksForFifo(GpuState state)
+        {
+            state.RegisterCallback(MethodOffset.FenceAction,            FenceAction);
+            state.RegisterCallback(MethodOffset.WaitForIdle,            WaitForIdle);
+            state.RegisterCallback(MethodOffset.SendMacroCodeData,      SendMacroCodeData);
+            state.RegisterCallback(MethodOffset.BindMacro,              BindMacro);
+            state.RegisterCallback(MethodOffset.SetMmeShadowRamControl, SetMmeShadowRamControl);
+        }
+
         /// <summary>
         /// Updates host state based on the current guest GPU state.
         /// </summary>

Ryujinx.Graphics.Gpu/GpuContext.cs

@@ -1,6 +1,7 @@
 using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Gpu.Engine;
 using Ryujinx.Graphics.Gpu.Memory;
+using Ryujinx.Graphics.Gpu.Synchronization;
 using System;
 
 namespace Ryujinx.Graphics.Gpu
@@ -45,6 +46,11 @@ namespace Ryujinx.Graphics.Gpu
         /// </summary>
         public DmaPusher DmaPusher { get; }
 
+        /// <summary>
+        /// GPU synchronization manager.
+        /// </summary>
+        public SynchronizationManager Synchronization { get; }
+
         /// <summary>
         /// Presentation window.
         /// </summary>
@@ -81,6 +87,8 @@ namespace Ryujinx.Graphics.Gpu
 
             DmaPusher = new DmaPusher(this);
 
+            Synchronization = new SynchronizationManager();
+
             Window = new Window(this);
 
             _caps = new Lazy<Capabilities>(Renderer.GetCapabilities);

Ryujinx.Graphics.Gpu/NvGpuFifo.cs

@@ -123,6 +123,8 @@ namespace Ryujinx.Graphics.Gpu
 
         private SubChannel[] _subChannels;
 
+        private SubChannel _fifoChannel;
+
         /// <summary>
         /// Creates a new instance of the GPU commands FIFO.
         /// </summary>
@@ -135,76 +137,68 @@ namespace Ryujinx.Graphics.Gpu
 
             _mme = new int[MmeWords];
 
+            _fifoChannel = new SubChannel();
+
+            _context.Methods.RegisterCallbacksForFifo(_fifoChannel.State);
+
             _subChannels = new SubChannel[8];
 
             for (int index = 0; index < _subChannels.Length; index++)
             {
                 _subChannels[index] = new SubChannel();
 
-                context.Methods.RegisterCallbacks(_subChannels[index].State);
+                _context.Methods.RegisterCallbacks(_subChannels[index].State);
             }
         }
 
+        /// <summary>
+        /// Send macro code/data to the MME
+        /// </summary>
+        /// <param name="index">The index in the MME</param>
+        /// <param name="data">The data to use</param>
+        public void SendMacroCodeData(int index, int data)
+        {
+            _mme[index] = data;
+        }
+
+        /// <summary>
+        /// Bind a macro index to a position for the MME
+        /// </summary>
+        /// <param name="index">The macro index</param>
+        /// <param name="position">The position of the macro</param>
+        public void BindMacro(int index, int position)
+        {
+            _macros[index] = new CachedMacro(position);
+        }
+
+        /// <summary>
+        /// Change the shadow RAM setting
+        /// </summary>
+        /// <param name="shadowCtrl">The new Shadow RAM setting</param>
+        public void SetMmeShadowRamControl(ShadowRamControl shadowCtrl)
+        {
+            _shadowCtrl = shadowCtrl;
+        }
+
         /// <summary>
         /// Calls a GPU method.
         /// </summary>
         /// <param name="meth">GPU method call parameters</param>
         public void CallMethod(MethodParams meth)
         {
-            if ((NvGpuFifoMeth)meth.Method == NvGpuFifoMeth.BindChannel)
+            if ((MethodOffset)meth.Method == MethodOffset.BindChannel)
             {
-                _subChannels[meth.SubChannel].Class = (ClassId)meth.Argument;
+                _subChannels[meth.SubChannel] = new SubChannel
+                {
+                    Class = (ClassId)meth.Argument
+                };
+
+                _context.Methods.RegisterCallbacks(_subChannels[meth.SubChannel].State);
             }
             else if (meth.Method < 0x60)
             {
-                switch ((NvGpuFifoMeth)meth.Method)
-                {
-                    case NvGpuFifoMeth.WaitForIdle:
-                    {
-                        _context.Methods.PerformDeferredDraws();
-
-                        _context.Renderer.Pipeline.Barrier();
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SetMacroUploadAddress:
-                    {
-                        _currMacroPosition = meth.Argument;
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SendMacroCodeData:
-                    {
-                        _mme[_currMacroPosition++] = meth.Argument;
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SetMacroBindingIndex:
-                    {
-                        _currMacroBindIndex = meth.Argument;
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.BindMacro:
-                    {
-                        int position = meth.Argument;
-
-                        _macros[_currMacroBindIndex++] = new CachedMacro(position);
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SetMmeShadowRamControl:
-                    {
-                        _shadowCtrl = (ShadowRamControl)meth.Argument;
-
-                        break;
-                    }
-                }
+                // TODO: check if macros are shared between subchannels or not. For now let's assume they are.
+                _fifoChannel.State.CallMethod(meth);
             }
             else if (meth.Method < 0xe00)
             {

Ryujinx.Graphics.Gpu/NvGpuFifoMeth.cs

@@ -1,16 +0,0 @@
-namespace Ryujinx.Graphics.Gpu
-{
-    /// <summary>
-    /// GPU commands FIFO processor commands.
-    /// </summary>
-    enum NvGpuFifoMeth
-    {
-        BindChannel            = 0,
-        WaitForIdle            = 0x44,
-        SetMacroUploadAddress  = 0x45,
-        SendMacroCodeData      = 0x46,
-        SetMacroBindingIndex   = 0x47,
-        BindMacro              = 0x48,
-        SetMmeShadowRamControl = 0x49
-    }
-}

Ryujinx.Graphics.Gpu/State/FenceActionOperation.cs

@@ -0,0 +1,11 @@
+namespace Ryujinx.Graphics.Gpu.State
+{
+    /// <summary>
+    /// Fence action operations.
+    /// </summary>
+    enum FenceActionOperation
+    {
+        Acquire   = 0,
+        Increment = 1
+    }
+}

Ryujinx.Graphics.Gpu/State/MethodOffset.cs

@@ -8,6 +8,15 @@ namespace Ryujinx.Graphics.Gpu.State
     /// </remarks>
     enum MethodOffset
     {
+        BindChannel                     = 0x00,
+        FenceValue                      = 0x1c,
+        FenceAction                     = 0x1d,
+        WaitForIdle                     = 0x44,
+        MacroUploadAddress              = 0x45,
+        SendMacroCodeData               = 0x46,
+        MacroBindingIndex               = 0x47,
+        BindMacro                       = 0x48,
+        SetMmeShadowRamControl          = 0x49,
         I2mParams                       = 0x60,
         LaunchDma                       = 0x6c,
         LoadInlineData                  = 0x6d,
@@ -15,6 +24,7 @@ namespace Ryujinx.Graphics.Gpu.State
         CopySrcTexture                  = 0x8c,
         DispatchParamsAddress           = 0xad,
         Dispatch                        = 0xaf,
+        SyncpointAction                 = 0xb2,
         CopyBuffer                      = 0xc0,
         RasterizeEnable                 = 0xdf,
         CopyBufferParams                = 0x100,

Ryujinx.Graphics.Gpu/State/ReportMode.cs

@@ -5,7 +5,8 @@ namespace Ryujinx.Graphics.Gpu.State
     /// </summary>
     enum ReportMode
     {
-        Semaphore = 0,
-        Counter   = 2
+        Release = 0,
+        Acquire = 1,
+        Counter = 2
     }
 }

Ryujinx.Graphics.Gpu/Synchronization/SynchronizationManager.cs

@@ -0,0 +1,134 @@
+using System;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Gpu.Synchronization
+{
+    /// <summary>
+    /// GPU synchronization manager.
+    /// </summary>
+    public class SynchronizationManager
+    {
+        /// <summary>
+        /// The maximum number of syncpoints supported by the GM20B.
+        /// </summary>
+        public const int MaxHardwareSyncpoints = 192;
+
+        /// <summary>
+        /// Array containing all hardware syncpoints.
+        /// </summary>
+        private Syncpoint[] _syncpoints;
+
+        public SynchronizationManager()
+        {
+            _syncpoints = new Syncpoint[MaxHardwareSyncpoints];
+
+            for (uint i = 0; i < _syncpoints.Length; i++)
+            {
+                _syncpoints[i] = new Syncpoint(i);
+            }
+        }
+
+        /// <summary>
+        /// Increment the value of a syncpoint with a given id.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>The incremented value of the syncpoint</returns>
+        public uint IncrementSyncpoint(uint id)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            return _syncpoints[id].Increment();
+        }
+
+        /// <summary>
+        /// Get the value of a syncpoint with a given id.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>The value of the syncpoint</returns>
+        public uint GetSyncpointValue(uint id)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            return _syncpoints[id].Value;
+        }
+
+        /// <summary>
+        /// Register a new callback on a syncpoint with a given id at a target threshold.
+        /// The callback will be called once the threshold is reached and will automatically be unregistered.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <param name="threshold">The target threshold</param>
+        /// <param name="callback">The callback to call when the threshold is reached</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>The created SyncpointWaiterHandle object or null if already past threshold</returns>
+        public SyncpointWaiterHandle RegisterCallbackOnSyncpoint(uint id, uint threshold, Action callback)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            return _syncpoints[id].RegisterCallback(threshold, callback);
+        }
+
+        /// <summary>
+        /// Unregister a callback on a given syncpoint.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <param name="waiterInformation">The waiter information to unregister</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        public void UnregisterCallback(uint id, SyncpointWaiterHandle waiterInformation)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            _syncpoints[id].UnregisterCallback(waiterInformation);
+        }
+
+        /// <summary>
+        /// Wait on a syncpoint with a given id at a target threshold.
+        /// The callback will be called once the threshold is reached and will automatically be unregistered.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <param name="threshold">The target threshold</param>
+        /// <param name="timeout">The timeout</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>True if timed out</returns>
+        public bool WaitOnSyncpoint(uint id, uint threshold, TimeSpan timeout)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            using (ManualResetEvent waitEvent = new ManualResetEvent(false))
+            {
+                var info = _syncpoints[id].RegisterCallback(threshold, () => waitEvent.Set());
+
+                if (info == null)
+                {
+                    return false;
+                }
+
+                bool signaled = waitEvent.WaitOne(timeout);
+
+                if (!signaled && info != null)
+                {
+                    _syncpoints[id].UnregisterCallback(info);
+                }
+
+                return !signaled;
+            }
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Synchronization/Syncpoint.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Gpu.Synchronization
+{
+    /// <summary>
+    /// Represents GPU hardware syncpoint.
+    /// </summary>
+    class Syncpoint
+    {
+        private int _storedValue;
+
+        public readonly uint Id;
+
+        // TODO: get rid of this lock
+        private object _listLock = new object();
+
+        /// <summary>
+        /// The value of the syncpoint.
+        /// </summary>
+        public uint Value => (uint)_storedValue;
+
+        // TODO: switch to something handling concurrency?
+        private List<SyncpointWaiterHandle> _waiters;
+
+        public Syncpoint(uint id)
+        {
+            Id       = id;
+            _waiters = new List<SyncpointWaiterHandle>();
+        }
+
+        /// <summary>
+        /// Register a new callback for a target threshold.
+        /// The callback will be called once the threshold is reached and will automatically be unregistered.
+        /// </summary>
+        /// <param name="threshold">The target threshold</param>
+        /// <param name="callback">The callback to call when the threshold is reached</param>
+        /// <returns>The created SyncpointWaiterHandle object or null if already past threshold</returns>
+        public SyncpointWaiterHandle RegisterCallback(uint threshold, Action callback)
+        {
+            lock (_listLock)
+            {
+                if (Value >= threshold)
+                {
+                    callback();
+
+                    return null;
+                }
+                else
+                {
+                    SyncpointWaiterHandle waiterInformation = new SyncpointWaiterHandle
+                    {
+                        Threshold = threshold,
+                        Callback  = callback
+                    };
+
+                    _waiters.Add(waiterInformation);
+
+                    return waiterInformation;
+                }
+            }
+        }
+
+        public void UnregisterCallback(SyncpointWaiterHandle waiterInformation)
+        {
+            lock (_listLock)
+            {
+                _waiters.Remove(waiterInformation);
+            }
+        }
+
+        /// <summary>
+        /// Increment the syncpoint
+        /// </summary>
+        /// <returns>The incremented value of the syncpoint</returns>
+        public uint Increment()
+        {
+            uint currentValue = (uint)Interlocked.Increment(ref _storedValue);
+
+            lock (_listLock)
+            {
+                _waiters.RemoveAll(item =>
+                {
+                    bool isPastThreshold = currentValue >= item.Threshold;
+
+                    if (isPastThreshold)
+                    {
+                        item.Callback();
+                    }
+
+                    return isPastThreshold;
+                });
+            }
+
+            return currentValue;
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Synchronization/SyncpointWaiterHandle.cs

@@ -0,0 +1,10 @@
+using System;
+
+namespace Ryujinx.Graphics.Gpu.Synchronization
+{
+    public class SyncpointWaiterHandle
+    {
+        internal uint   Threshold;
+        internal Action Callback;
+    }
+}

Ryujinx.Graphics.Gpu/Window.cs

@@ -30,12 +30,17 @@ namespace Ryujinx.Graphics.Gpu
             public ImageCrop Crop { get; }
 
             /// <summary>
-            /// Texture release callback.
+            /// Texture acquire callback.
             /// </summary>
-            public Action<object> Callback { get; }
+            public Action<GpuContext, object> AcquireCallback { get; }
 
             /// <summary>
-            /// User defined object, passed to the release callback.
+            /// Texture release callback.
+            /// </summary>
+            public Action<object> ReleaseCallback { get; }
+
+            /// <summary>
+            /// User defined object, passed to the various callbacks.
             /// </summary>
             public object UserObj { get; }
 
@@ -44,18 +49,21 @@ namespace Ryujinx.Graphics.Gpu
             /// </summary>
             /// <param name="info">Information of the texture to be presented</param>
             /// <param name="crop">Texture crop region</param>
-            /// <param name="callback">Texture release callback</param>
+            /// <param name="acquireCallback">Texture acquire callback</param>
+            /// <param name="releaseCallback">Texture release callback</param>
             /// <param name="userObj">User defined object passed to the release callback, can be used to identify the texture</param>
             public PresentationTexture(
-                TextureInfo    info,
-                ImageCrop      crop,
-                Action<object> callback,
-                object         userObj)
+                TextureInfo                info,
+                ImageCrop                  crop,
+                Action<GpuContext, object> acquireCallback,
+                Action<object>             releaseCallback,
+                object                     userObj)
             {
-                Info     = info;
-                Crop     = crop;
-                Callback = callback;
-                UserObj  = userObj;
+                Info            = info;
+                Crop            = crop;
+                AcquireCallback = acquireCallback;
+                ReleaseCallback = releaseCallback;
+                UserObj         = userObj;
             }
         }
 
@@ -87,20 +95,22 @@ namespace Ryujinx.Graphics.Gpu
         /// <param name="format">Texture format</param>
         /// <param name="bytesPerPixel">Texture format bytes per pixel (must match the format)</param>
         /// <param name="crop">Texture crop region</param>
-        /// <param name="callback">Texture release callback</param>
+        /// <param name="acquireCallback">Texture acquire callback</param>
+        /// <param name="releaseCallback">Texture release callback</param>
         /// <param name="userObj">User defined object passed to the release callback</param>
         public void EnqueueFrameThreadSafe(
-            ulong          address,
-            int            width,
-            int            height,
-            int            stride,
-            bool           isLinear,
-            int            gobBlocksInY,
-            Format         format,
-            int            bytesPerPixel,
-            ImageCrop      crop,
-            Action<object> callback,
-            object         userObj)
+            ulong                      address,
+            int                        width,
+            int                        height,
+            int                        stride,
+            bool                       isLinear,
+            int                        gobBlocksInY,
+            Format                     format,
+            int                        bytesPerPixel,
+            ImageCrop                  crop,
+            Action<GpuContext, object> acquireCallback,
+            Action<object>             releaseCallback,
+            object                     userObj)
         {
             FormatInfo formatInfo = new FormatInfo(format, 1, 1, bytesPerPixel);
 
@@ -120,7 +130,7 @@ namespace Ryujinx.Graphics.Gpu
                 Target.Texture2D,
                 formatInfo);
 
-            _frameQueue.Enqueue(new PresentationTexture(info, crop, callback, userObj));
+            _frameQueue.Enqueue(new PresentationTexture(info, crop, acquireCallback, releaseCallback, userObj));
         }
 
         /// <summary>
@@ -134,6 +144,8 @@ namespace Ryujinx.Graphics.Gpu
 
             if (_frameQueue.TryDequeue(out PresentationTexture pt))
             {
+                pt.AcquireCallback(_context, pt.UserObj);
+
                 Texture texture = _context.Methods.TextureManager.FindOrCreateTexture(pt.Info);
 
                 texture.SynchronizeMemory();
@@ -142,7 +154,7 @@ namespace Ryujinx.Graphics.Gpu
 
                 swapBuffersCallback();
 
-                pt.Callback(pt.UserObj);
+                pt.ReleaseCallback(pt.UserObj);
             }
         }
     }