mirrors/ryujinx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

nvdec: Adding Vp8 codec support (#2707)

Browse source 
* first try

* second try

* working update

* Final impl

* Fixes nits

* Fix everything

* remove leftover

* Update FFmpegContext.cs

* Update Surface.cs

* Addresses gdkchan feedback

* bool not byte

* Addresses gdkchan feedback
This commit is contained in:
Ac_K 2021-10-12 22:55:57 +02:00 • committed by GitHub
parent a7109c767b
commit d1604aa762
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 220 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

162
Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs Normal file
View file

@ -0,0 +1,162 @@
using FFmpeg.AutoGen;
using Ryujinx.Common.Logging;
using System;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.Nvdec.FFmpeg
{
unsafe class FFmpegContext : IDisposable
{
private readonly AVCodec_decode _decodeFrame;
private static readonly av_log_set_callback_callback _logFunc;
private readonly AVCodec* _codec;
private AVPacket* _packet;
private AVCodecContext* _context;
public FFmpegContext(AVCodecID codecId)
{
_codec = ffmpeg.avcodec_find_decoder(codecId);
_context = ffmpeg.avcodec_alloc_context3(_codec);
ffmpeg.avcodec_open2(_context, _codec, null);
_packet = ffmpeg.av_packet_alloc();
_decodeFrame = Marshal.GetDelegateForFunctionPointer<AVCodec_decode>(_codec->decode.Pointer);
}
static FFmpegContext()
{
SetRootPath();
_logFunc = Log;
// Redirect log output.
ffmpeg.av_log_set_level(ffmpeg.AV_LOG_MAX_OFFSET);
ffmpeg.av_log_set_callback(_logFunc);
}
private static void SetRootPath()
{
if (OperatingSystem.IsLinux())
{
// Configure FFmpeg search path
Process lddProcess = Process.Start(new ProcessStartInfo
{
FileName = "/bin/sh",
Arguments = "-c \"ldd $(which ffmpeg 2>/dev/null) | grep libavfilter\" 2>/dev/null",
UseShellExecute = false,
RedirectStandardOutput = true
});
string lddOutput = lddProcess.StandardOutput.ReadToEnd();
lddProcess.WaitForExit();
lddProcess.Close();
if (lddOutput.Contains(" => "))
{
ffmpeg.RootPath = Path.GetDirectoryName(lddOutput.Split(" => ")[1]);
}
else
{
Logger.Error?.PrintMsg(LogClass.FFmpeg, "FFmpeg wasn't found. Make sure that you have it installed and up to date.");
}
}
}
private static void Log(void* p0, int level, string format, byte* vl)
{
if (level > ffmpeg.av_log_get_level())
{
return;
}
int lineSize = 1024;
byte* lineBuffer = stackalloc byte[lineSize];
int printPrefix = 1;
ffmpeg.av_log_format_line(p0, level, format, vl, lineBuffer, lineSize, &printPrefix);
string line = Marshal.PtrToStringAnsi((IntPtr)lineBuffer).Trim();
switch (level)
{
case ffmpeg.AV_LOG_PANIC:
case ffmpeg.AV_LOG_FATAL:
case ffmpeg.AV_LOG_ERROR:
Logger.Error?.Print(LogClass.FFmpeg, line);
break;
case ffmpeg.AV_LOG_WARNING:
Logger.Warning?.Print(LogClass.FFmpeg, line);
break;
case ffmpeg.AV_LOG_INFO:
Logger.Info?.Print(LogClass.FFmpeg, line);
break;
case ffmpeg.AV_LOG_VERBOSE:
case ffmpeg.AV_LOG_DEBUG:
case ffmpeg.AV_LOG_TRACE:
Logger.Debug?.Print(LogClass.FFmpeg, line);
break;
}
}
public int DecodeFrame(Surface output, ReadOnlySpan<byte> bitstream)
{
ffmpeg.av_frame_unref(output.Frame);
int result;
int gotFrame;
fixed (byte* ptr = bitstream)
{
_packet->data = ptr;
_packet->size = bitstream.Length;
result = _decodeFrame(_context, output.Frame, &gotFrame, _packet);
}
if (gotFrame == 0)
{
ffmpeg.av_frame_unref(output.Frame);
// If the frame was not delivered, it was probably delayed.
// Get the next delayed frame by passing a 0 length packet.
_packet->data = null;
_packet->size = 0;
result = _decodeFrame(_context, output.Frame, &gotFrame, _packet);
// We need to set B frames to 0 as we already consumed all delayed frames.
// This prevents the decoder from trying to return a delayed frame next time.
_context->has_b_frames = 0;
}
ffmpeg.av_packet_unref(_packet);
if (gotFrame == 0)
{
ffmpeg.av_frame_unref(output.Frame);
return -1;
}
return result < 0 ? result : 0;
}
public void Dispose()
{
fixed (AVPacket** ppPacket = &_packet)
{
ffmpeg.av_packet_free(ppPacket);
}
ffmpeg.avcodec_close(_context);
fixed (AVCodecContext** ppContext = &_context)
{
ffmpeg.avcodec_free_context(ppContext);
}
}
}
}

56
Ryujinx.Graphics.Nvdec.FFmpeg/H264/Decoder.cs Normal file
View file

@ -0,0 +1,56 @@
using FFmpeg.AutoGen;
using Ryujinx.Graphics.Video;
using System;
namespace Ryujinx.Graphics.Nvdec.FFmpeg.H264
{
public sealed class Decoder : IH264Decoder
{
public bool IsHardwareAccelerated => false;
private const int WorkBufferSize = 0x200;
private readonly byte[] _workBuffer = new byte[WorkBufferSize];
private FFmpegContext _context = new FFmpegContext(AVCodecID.AV_CODEC_ID_H264);
private int _oldOutputWidth;
private int _oldOutputHeight;
public ISurface CreateSurface(int width, int height)
{
return new Surface(width, height);
}
public bool Decode(ref H264PictureInfo pictureInfo, ISurface output, ReadOnlySpan<byte> bitstream)
{
Surface outSurf = (Surface)output;
if (outSurf.RequestedWidth != _oldOutputWidth ||
outSurf.RequestedHeight != _oldOutputHeight)
{
_context.Dispose();
_context = new FFmpegContext(AVCodecID.AV_CODEC_ID_H264);
_oldOutputWidth = outSurf.RequestedWidth;
_oldOutputHeight = outSurf.RequestedHeight;
}
Span<byte> bs = Prepend(bitstream, SpsAndPpsReconstruction.Reconstruct(ref pictureInfo, _workBuffer));
return _context.DecodeFrame(outSurf, bs) == 0;
}
private static byte[] Prepend(ReadOnlySpan<byte> data, ReadOnlySpan<byte> prep)
{
byte[] output = new byte[data.Length + prep.Length];
prep.CopyTo(output);
data.CopyTo(new Span<byte>(output).Slice(prep.Length));
return output;
}
public void Dispose() => _context.Dispose();
}
}

121
Ryujinx.Graphics.Nvdec.FFmpeg/H264/H264BitStreamWriter.cs Normal file
View file

@ -0,0 +1,121 @@
using System;
using System.Numerics;
namespace Ryujinx.Graphics.Nvdec.FFmpeg.H264
{
struct H264BitStreamWriter
{
private const int BufferSize = 8;
private readonly byte[] _workBuffer;
private int _offset;
private int _buffer;
private int _bufferPos;
public H264BitStreamWriter(byte[] workBuffer)
{
_workBuffer = workBuffer;
_offset = 0;
_buffer = 0;
_bufferPos = 0;
}
public void WriteBit(bool value)
{
WriteBits(value ? 1 : 0, 1);
}
public void WriteBits(int value, int valueSize)
{
int valuePos = 0;
int remaining = valueSize;
while (remaining > 0)
{
int copySize = remaining;
int free = GetFreeBufferBits();
if (copySize > free)
{
copySize = free;
}
int mask = (1 << copySize) - 1;
int srcShift = (valueSize - valuePos) - copySize;
int dstShift = (BufferSize - _bufferPos) - copySize;
_buffer |= ((value >> srcShift) & mask) << dstShift;
valuePos += copySize;
_bufferPos += copySize;
remaining -= copySize;
}
}
private int GetFreeBufferBits()
{
if (_bufferPos == BufferSize)
{
Flush();
}
return BufferSize - _bufferPos;
}
public void Flush()
{
if (_bufferPos != 0)
{
_workBuffer[_offset++] = (byte)_buffer;
_buffer = 0;
_bufferPos = 0;
}
}
public void End()
{
WriteBit(true);
Flush();
}
public Span<byte> AsSpan()
{
return new Span<byte>(_workBuffer).Slice(0, _offset);
}
public void WriteU(uint value, int valueSize) => WriteBits((int)value, valueSize);
public void WriteSe(int value) => WriteExpGolombCodedInt(value);
public void WriteUe(uint value) => WriteExpGolombCodedUInt(value);
private void WriteExpGolombCodedInt(int value)
{
int sign = value <= 0 ? 0 : 1;
if (value < 0)
{
value = -value;
}
value = (value << 1) - sign;
WriteExpGolombCodedUInt((uint)value);
}
private void WriteExpGolombCodedUInt(uint value)
{
int size = 32 - BitOperations.LeadingZeroCount(value + 1);
WriteBits(1, size);
value -= (1u << (size - 1)) - 1;
WriteBits((int)value, size - 1);
}
}
}

159
Ryujinx.Graphics.Nvdec.FFmpeg/H264/SpsAndPpsReconstruction.cs Normal file
View file

@ -0,0 +1,159 @@
using Ryujinx.Common.Memory;
using Ryujinx.Graphics.Video;
using System;
namespace Ryujinx.Graphics.Nvdec.FFmpeg.H264
{
static class SpsAndPpsReconstruction
{
public static Span<byte> Reconstruct(ref H264PictureInfo pictureInfo, byte[] workBuffer)
{
H264BitStreamWriter writer = new H264BitStreamWriter(workBuffer);
// Sequence Parameter Set.
writer.WriteU(1, 24);
writer.WriteU(0, 1);
writer.WriteU(3, 2);
writer.WriteU(7, 5);
writer.WriteU(100, 8); // Profile idc
writer.WriteU(0, 8); // Reserved
writer.WriteU(31, 8); // Level idc
writer.WriteUe(0); // Seq parameter set id
writer.WriteUe(pictureInfo.ChromaFormatIdc);
if (pictureInfo.ChromaFormatIdc == 3)
{
writer.WriteBit(false); // Separate colour plane flag
}
writer.WriteUe(0); // Bit depth luma minus 8
writer.WriteUe(0); // Bit depth chroma minus 8
writer.WriteBit(pictureInfo.QpprimeYZeroTransformBypassFlag);
writer.WriteBit(false); // Scaling matrix present flag
writer.WriteUe(pictureInfo.Log2MaxFrameNumMinus4);
writer.WriteUe(pictureInfo.PicOrderCntType);
if (pictureInfo.PicOrderCntType == 0)
{
writer.WriteUe(pictureInfo.Log2MaxPicOrderCntLsbMinus4);
}
else if (pictureInfo.PicOrderCntType == 1)
{
writer.WriteBit(pictureInfo.DeltaPicOrderAlwaysZeroFlag);
writer.WriteSe(0); // Offset for non-ref pic
writer.WriteSe(0); // Offset for top to bottom field
writer.WriteUe(0); // Num ref frames in pic order cnt cycle
}
writer.WriteUe(16); // Max num ref frames
writer.WriteBit(false); // Gaps in frame num value allowed flag
writer.WriteUe(pictureInfo.PicWidthInMbsMinus1);
writer.WriteUe(pictureInfo.PicHeightInMapUnitsMinus1);
writer.WriteBit(pictureInfo.FrameMbsOnlyFlag);
if (!pictureInfo.FrameMbsOnlyFlag)
{
writer.WriteBit(pictureInfo.MbAdaptiveFrameFieldFlag);
}
writer.WriteBit(pictureInfo.Direct8x8InferenceFlag);
writer.WriteBit(false); // Frame cropping flag
writer.WriteBit(false); // VUI parameter present flag
writer.End();
// Picture Parameter Set.
writer.WriteU(1, 24);
writer.WriteU(0, 1);
writer.WriteU(3, 2);
writer.WriteU(8, 5);
writer.WriteUe(0); // Pic parameter set id
writer.WriteUe(0); // Seq parameter set id
writer.WriteBit(pictureInfo.EntropyCodingModeFlag);
writer.WriteBit(pictureInfo.PicOrderPresentFlag);
writer.WriteUe(0); // Num slice groups minus 1
writer.WriteUe(pictureInfo.NumRefIdxL0ActiveMinus1);
writer.WriteUe(pictureInfo.NumRefIdxL1ActiveMinus1);
writer.WriteBit(pictureInfo.WeightedPredFlag);
writer.WriteU(pictureInfo.WeightedBipredIdc, 2);
writer.WriteSe(pictureInfo.PicInitQpMinus26);
writer.WriteSe(0); // Pic init qs minus 26
writer.WriteSe(pictureInfo.ChromaQpIndexOffset);
writer.WriteBit(pictureInfo.DeblockingFilterControlPresentFlag);
writer.WriteBit(pictureInfo.ConstrainedIntraPredFlag);
writer.WriteBit(pictureInfo.RedundantPicCntPresentFlag);
writer.WriteBit(pictureInfo.Transform8x8ModeFlag);
writer.WriteBit(pictureInfo.ScalingMatrixPresent);
if (pictureInfo.ScalingMatrixPresent)
{
for (int index = 0; index < 6; index++)
{
writer.WriteBit(true);
WriteScalingList(ref writer, pictureInfo.ScalingLists4x4[index]);
}
if (pictureInfo.Transform8x8ModeFlag)
{
for (int index = 0; index < 2; index++)
{
writer.WriteBit(true);
WriteScalingList(ref writer, pictureInfo.ScalingLists8x8[index]);
}
}
}
writer.WriteSe(pictureInfo.SecondChromaQpIndexOffset);
writer.End();
return writer.AsSpan();
}
// ZigZag LUTs from libavcodec.
private static readonly byte[] ZigZagDirect = new byte[]
{
0, 1, 8, 16, 9, 2, 3, 10,
17, 24, 32, 25, 18, 11, 4, 5,
12, 19, 26, 33, 40, 48, 41, 34,
27, 20, 13, 6, 7, 14, 21, 28,
35, 42, 49, 56, 57, 50, 43, 36,
29, 22, 15, 23, 30, 37, 44, 51,
58, 59, 52, 45, 38, 31, 39, 46,
53, 60, 61, 54, 47, 55, 62, 63
};
private static readonly byte[] ZigZagScan = new byte[]
{
0 + 0 * 4, 1 + 0 * 4, 0 + 1 * 4, 0 + 2 * 4,
1 + 1 * 4, 2 + 0 * 4, 3 + 0 * 4, 2 + 1 * 4,
1 + 2 * 4, 0 + 3 * 4, 1 + 3 * 4, 2 + 2 * 4,
3 + 1 * 4, 3 + 2 * 4, 2 + 3 * 4, 3 + 3 * 4
};
private static void WriteScalingList(ref H264BitStreamWriter writer, IArray<byte> list)
{
byte[] scan = list.Length == 16 ? ZigZagScan : ZigZagDirect;
int lastScale = 8;
for (int index = 0; index < list.Length; index++)
{
byte value = list[scan[index]];
int deltaScale = value - lastScale;
writer.WriteSe(deltaScale);
lastScale = value;
}
}
}
}

17
Ryujinx.Graphics.Nvdec.FFmpeg/Ryujinx.Graphics.Nvdec.FFmpeg.csproj Normal file
View file

@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FFmpeg.AutoGen" Version="4.4.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Ryujinx.Common\Ryujinx.Common.csproj" />
<ProjectReference Include="..\Ryujinx.Graphics.Video\Ryujinx.Graphics.Video.csproj" />
</ItemGroup>
</Project>

39
Ryujinx.Graphics.Nvdec.FFmpeg/Surface.cs Normal file
View file

@ -0,0 +1,39 @@
using FFmpeg.AutoGen;
using Ryujinx.Graphics.Video;
using System;
namespace Ryujinx.Graphics.Nvdec.FFmpeg
{
unsafe class Surface : ISurface
{
public AVFrame* Frame { get; }
public int RequestedWidth { get; }
public int RequestedHeight { get; }
public Plane YPlane => new Plane((IntPtr)Frame->data[0], Stride * Height);
public Plane UPlane => new Plane((IntPtr)Frame->data[1], UvStride * UvHeight);
public Plane VPlane => new Plane((IntPtr)Frame->data[2], UvStride * UvHeight);
public int Width => Frame->width;
public int Height => Frame->height;
public int Stride => Frame->linesize[0];
public int UvWidth => (Frame->width + 1) >> 1;
public int UvHeight => (Frame->height + 1) >> 1;
public int UvStride => Frame->linesize[1];
public Surface(int width, int height)
{
RequestedWidth = width;
RequestedHeight = height;
Frame = ffmpeg.av_frame_alloc();
}
public void Dispose()
{
ffmpeg.av_frame_unref(Frame);
ffmpeg.av_free(Frame);
}
}
}

53
Ryujinx.Graphics.Nvdec.FFmpeg/Vp8/Decoder.cs Normal file
View file

@ -0,0 +1,53 @@
using FFmpeg.AutoGen;
using Ryujinx.Graphics.Video;
using System;
namespace Ryujinx.Graphics.Nvdec.FFmpeg.Vp8
{
public sealed class Decoder : IDecoder
{
public bool IsHardwareAccelerated => false;
private readonly FFmpegContext _context = new FFmpegContext(AVCodecID.AV_CODEC_ID_VP8);
public ISurface CreateSurface(int width, int height)
{
return new Surface(width, height);
}
public bool Decode(ref Vp8PictureInfo pictureInfo, ISurface output, ReadOnlySpan<byte> bitstream)
{
Surface outSurf = (Surface)output;
int uncompHeaderSize = pictureInfo.KeyFrame ? 10 : 3;
byte[] frame = new byte[bitstream.Length + uncompHeaderSize];
uint firstPartSizeShifted = pictureInfo.FirstPartSize << 5;
frame[0] = (byte)(pictureInfo.KeyFrame ? 0 : 1);
frame[0] |= (byte)((pictureInfo.Version & 7) << 1);
frame[0] |= 1 << 4;
frame[0] |= (byte)firstPartSizeShifted;
frame[1] |= (byte)(firstPartSizeShifted >> 8);
frame[2] |= (byte)(firstPartSizeShifted >> 16);
if (pictureInfo.KeyFrame)
{
frame[3] = 0x9d;
frame[4] = 0x01;
frame[5] = 0x2a;
frame[6] = (byte)pictureInfo.FrameWidth;
frame[7] = (byte)((pictureInfo.FrameWidth >> 8) & 0x3F);
frame[8] = (byte)pictureInfo.FrameHeight;
frame[9] = (byte)((pictureInfo.FrameHeight >> 8) & 0x3F);
}
bitstream.CopyTo(new Span<byte>(frame).Slice(uncompHeaderSize));
return _context.DecodeFrame(outSurf, frame) == 0;
}
public void Dispose() => _context.Dispose();
}
}