From dbf78ba55eb879439e0539d539d34d22c777598f Mon Sep 17 00:00:00 2001 From: gdkchan Date: Sat, 22 Oct 2022 21:28:28 -0300 Subject: [PATCH] Global memory emulation using NV_shader_buffer_store and VK_EXT_buffer_device_address * Make global memory emulation work on OpenGL again * Shader cache version bump --- src/Ryujinx.Graphics.GAL/IPipeline.cs | 1 + src/Ryujinx.Graphics.GAL/IRenderer.cs | 1 + .../Multithreading/CommandHelper.cs | 2 + .../Multithreading/CommandType.cs | 2 + .../Buffer/BufferGetGpuAddressCommand.cs | 24 +++ .../UpdatePageTableGpuAddressCommand.cs | 18 ++ .../Multithreading/ThreadedPipeline.cs | 6 + .../Multithreading/ThreadedRenderer.cs | 16 ++ .../SupportBufferUpdater.cs | 14 ++ .../Engine/Compute/ComputeClass.cs | 12 +- .../Engine/Threed/StateUpdater.cs | 43 +++- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 11 + .../Memory/BufferCache.cs | 14 +- .../Memory/BufferManager.cs | 90 +++++++- .../Memory/BufferPageTable.cs | 201 ++++++++++++++++++ src/Ryujinx.Graphics.Gpu/Memory/Mapping.cs | 22 ++ .../Memory/MemoryManager.cs | 85 ++++++++ .../Shader/DiskCache/DiskCacheHostStorage.cs | 16 +- .../Shader/ResourceCounts.cs | 1 + src/Ryujinx.Graphics.OpenGL/Buffer.cs | 8 + src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs | 5 + src/Ryujinx.Graphics.OpenGL/Pipeline.cs | 5 + .../CodeGen/Glsl/Declarations.cs | 23 ++ .../Glsl/HelperFunctions/GlobalMemory.glsl | 19 ++ .../Glsl/HelperFunctions/GlobalMemoryVk.glsl | 34 +++ .../HelperFunctions/HelperFunctionNames.cs | 2 + .../CodeGen/Glsl/Instructions/InstGen.cs | 13 ++ .../Glsl/Instructions/InstGenHelper.cs | 2 + .../Glsl/Instructions/InstGenMemory.cs | 56 +++++ .../CodeGen/Spirv/CodeGenContext.cs | 1 + .../CodeGen/Spirv/Declarations.cs | 14 ++ .../CodeGen/Spirv/Instructions.cs | 102 ++++++++- .../CodeGen/Spirv/SpirvGenerator.cs | 7 + .../Decoders/Decoder.cs | 5 - .../Ryujinx.Graphics.Shader.csproj | 2 + .../ShaderProgramInfo.cs | 6 + .../StructuredIr/HelperFunctionsMask.cs | 21 +- .../StructuredIr/StructuredProgram.cs | 31 ++- src/Ryujinx.Graphics.Shader/SupportBuffer.cs | 9 +- .../Translation/FeatureFlags.cs | 10 +- .../Translation/Rewriter.cs | 4 +- .../Translation/ShaderConfig.cs | 12 +- src/Ryujinx.Graphics.Vulkan/BufferHolder.cs | 17 +- src/Ryujinx.Graphics.Vulkan/BufferManager.cs | 22 +- src/Ryujinx.Graphics.Vulkan/Constants.cs | 2 +- .../DescriptorSetManager.cs | 2 +- .../HardwareCapabilities.cs | 3 + src/Ryujinx.Graphics.Vulkan/PipelineBase.cs | 5 + .../VulkanInitialization.cs | 15 ++ src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs | 6 + src/Ryujinx.Memory/Range/RangeList.cs | 12 ++ .../Autogenerated/CoreGrammar.cs | 12 +- 52 files changed, 1013 insertions(+), 53 deletions(-) create mode 100644 src/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetGpuAddressCommand.cs create mode 100644 src/Ryujinx.Graphics.GAL/Multithreading/Commands/UpdatePageTableGpuAddressCommand.cs create mode 100644 src/Ryujinx.Graphics.Gpu/Memory/BufferPageTable.cs create mode 100644 src/Ryujinx.Graphics.Gpu/Memory/Mapping.cs create mode 100644 src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl create mode 100644 src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemoryVk.glsl diff --git a/src/Ryujinx.Graphics.GAL/IPipeline.cs b/src/Ryujinx.Graphics.GAL/IPipeline.cs index 0a362081c..46f60d6f4 100644 --- a/src/Ryujinx.Graphics.GAL/IPipeline.cs +++ b/src/Ryujinx.Graphics.GAL/IPipeline.cs @@ -108,6 +108,7 @@ namespace Ryujinx.Graphics.GAL bool TryHostConditionalRendering(ICounterEvent value, ICounterEvent compare, bool isEqual); void EndHostConditionalRendering(); + void UpdatePageTableGpuAddress(ulong address); void UpdateRenderScale(ReadOnlySpan scales, int totalCount, int fragmentCount); } } diff --git a/src/Ryujinx.Graphics.GAL/IRenderer.cs b/src/Ryujinx.Graphics.GAL/IRenderer.cs index b668d56ec..da787cb9d 100644 --- a/src/Ryujinx.Graphics.GAL/IRenderer.cs +++ b/src/Ryujinx.Graphics.GAL/IRenderer.cs @@ -36,6 +36,7 @@ namespace Ryujinx.Graphics.GAL void DeleteBuffer(BufferHandle buffer); PinnedSpan GetBufferData(BufferHandle buffer, int offset, int size); + ulong GetBufferGpuAddress(BufferHandle buffer); Capabilities GetCapabilities(); ulong GetCurrentSync(); diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs b/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs index 9f6e483cd..eda664062 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs @@ -57,6 +57,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading Register(CommandType.BufferDispose); Register(CommandType.BufferGetData); + Register(CommandType.BufferGetGpuAddress); Register(CommandType.BufferSetData); Register(CommandType.CounterEventDispose); @@ -138,6 +139,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading Register(CommandType.TextureBarrierTiled); Register(CommandType.TryHostConditionalRendering); Register(CommandType.TryHostConditionalRenderingFlush); + Register(CommandType.UpdatePageTableGpuAddress); Register(CommandType.UpdateRenderScale); return maxCommandSize; diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs b/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs index 8d9c1ec8d..847d9370e 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs @@ -19,6 +19,7 @@ BufferDispose, BufferGetData, + BufferGetGpuAddress, BufferSetData, CounterEventDispose, @@ -100,6 +101,7 @@ TextureBarrierTiled, TryHostConditionalRendering, TryHostConditionalRenderingFlush, + UpdatePageTableGpuAddress, UpdateRenderScale } } diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetGpuAddressCommand.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetGpuAddressCommand.cs new file mode 100644 index 000000000..c04d9532c --- /dev/null +++ b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetGpuAddressCommand.cs @@ -0,0 +1,24 @@ +using Ryujinx.Graphics.GAL.Multithreading.Model; + +namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Buffer +{ + struct BufferGetGpuAddressCommand : IGALCommand, IGALCommand + { + public CommandType CommandType => CommandType.BufferGetGpuAddress; + private BufferHandle _buffer; + private TableRef> _result; + + public void Set(BufferHandle buffer, TableRef> result) + { + _buffer = buffer; + _result = result; + } + + public static void Run(ref BufferGetGpuAddressCommand command, ThreadedRenderer threaded, IRenderer renderer) + { + ulong result = renderer.GetBufferGpuAddress(threaded.Buffers.MapBuffer(command._buffer)); + + command._result.Get(threaded).Result = result; + } + } +} diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/UpdatePageTableGpuAddressCommand.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/UpdatePageTableGpuAddressCommand.cs new file mode 100644 index 000000000..042df604f --- /dev/null +++ b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/UpdatePageTableGpuAddressCommand.cs @@ -0,0 +1,18 @@ +namespace Ryujinx.Graphics.GAL.Multithreading.Commands +{ + struct UpdatePageTableGpuAddressCommand : IGALCommand, IGALCommand + { + public CommandType CommandType => CommandType.UpdatePageTableGpuAddress; + private ulong _address; + + public void Set(ulong address) + { + _address = address; + } + + public static void Run(ref UpdatePageTableGpuAddressCommand command, ThreadedRenderer threaded, IRenderer renderer) + { + renderer.Pipeline.UpdatePageTableGpuAddress(command._address); + } + } +} diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs index 1bdc9cf48..6bf4aafca 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs @@ -371,6 +371,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading return false; } + public void UpdatePageTableGpuAddress(ulong address) + { + _renderer.New().Set(address); + _renderer.QueueCommand(); + } + public void UpdateRenderScale(ReadOnlySpan scales, int totalCount, int fragmentCount) { _renderer.New().Set(_renderer.CopySpan(scales.Slice(0, totalCount)), totalCount, fragmentCount); diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs index e6169d895..1e558dd11 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs @@ -361,6 +361,22 @@ namespace Ryujinx.Graphics.GAL.Multithreading } } + public ulong GetBufferGpuAddress(BufferHandle buffer) + { + if (IsGpuThread()) + { + ResultBox box = new ResultBox(); + New().Set(buffer, Ref(box)); + InvokeCommand(); + + return box.Result; + } + else + { + return _baseRenderer.GetBufferGpuAddress(Buffers.MapBufferBlocking(buffer)); + } + } + public Capabilities GetCapabilities() { ResultBox box = new ResultBox(); diff --git a/src/Ryujinx.Graphics.GAL/SupportBufferUpdater.cs b/src/Ryujinx.Graphics.GAL/SupportBufferUpdater.cs index 6eeddb6c0..34756aa74 100644 --- a/src/Ryujinx.Graphics.GAL/SupportBufferUpdater.cs +++ b/src/Ryujinx.Graphics.GAL/SupportBufferUpdater.cs @@ -44,6 +44,20 @@ namespace Ryujinx.Graphics.GAL } } + public void UpdatePageTableBasePointer(ulong address) + { + uint addrLow = (uint)address; + uint addrHigh = (uint)(address >> 32); + + if (Data.PageTableBasePointer.X != addrLow || Data.PageTableBasePointer.Y != addrHigh) + { + Data.PageTableBasePointer.X = addrLow; + Data.PageTableBasePointer.Y = addrHigh; + + MarkDirty(SupportBuffer.PageTableBasePointerOffset, sizeof(ulong)); + } + } + public void UpdateFragmentRenderScaleCount(int count) { if (Data.FragmentRenderScaleCount.X != count) diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs b/src/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs index 998ece224..0ef81d260 100644 --- a/src/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs +++ b/src/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs @@ -212,14 +212,22 @@ namespace Ryujinx.Graphics.Gpu.Engine.Compute } _channel.BufferManager.SetComputeBufferBindings(cs.Bindings); - _channel.TextureManager.SetComputeBindings(cs.Bindings); + if (info.UsesGlobalMemory) + { + _channel.BufferManager.SynchronizeComputeStorageBuffers(info.UsesGlobalMemoryWrite); + } + // Should never return false for mismatching spec state, since the shader was fetched above. _channel.TextureManager.CommitComputeBindings(cs.SpecializationState); - _channel.BufferManager.CommitComputeBindings(); + if (info.UsesGlobalMemory) + { + _channel.BufferManager.UpdatePageTable(); + } + _context.Renderer.Pipeline.DispatchCompute(qmd.CtaRasterWidth, qmd.CtaRasterHeight, qmd.CtaRasterDepth); _3dEngine.ForceShaderUpdate(); diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs b/src/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs index 5fa4702b8..ed6aa35b7 100644 --- a/src/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs +++ b/src/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs @@ -37,6 +37,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed private ProgramPipelineState _pipeline; + private uint _globalMemoryUseMask; + private uint _globalMemoryWriteMask; private bool _vsUsesDrawParameters; private bool _vtgWritesRtLayer; private byte _vsClipDistancesWritten; @@ -320,9 +322,15 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed /// private void CommitBindings() { - UpdateStorageBuffers(); - bool unalignedChanged = _currentSpecState.SetHasUnalignedStorageBuffer(_channel.BufferManager.HasUnalignedStorageBuffers); + bool usesGlobalMemory = _globalMemoryUseMask != 0; + + UpdateStorageBuffers(); + + if (usesGlobalMemory) + { + _channel.BufferManager.SynchronizeGraphicsStorageBuffers(_globalMemoryUseMask, _globalMemoryWriteMask); + } if (!_channel.TextureManager.CommitGraphicsBindings(_shaderSpecState) || unalignedChanged) { @@ -331,6 +339,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed } _channel.BufferManager.CommitGraphicsBindings(); + + if (usesGlobalMemory) + { + _channel.BufferManager.UpdatePageTable(); + } } /// @@ -1374,16 +1387,32 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed UpdateShaderBindings(gs.Bindings); + _globalMemoryUseMask = 0; + _globalMemoryWriteMask = 0; + for (int stageIndex = 0; stageIndex < Constants.ShaderStages; stageIndex++) { ShaderProgramInfo info = gs.Shaders[stageIndex + 1]?.Info; - if (info?.UsesRtLayer == true) - { - _vtgWritesRtLayer = true; - } - _currentProgramInfo[stageIndex] = info; + + if (info != null) + { + if (info.UsesGlobalMemory) + { + _globalMemoryUseMask |= 1u << stageIndex; + } + + if (info.UsesGlobalMemoryWrite) + { + _globalMemoryWriteMask |= 1u << stageIndex; + } + + if (info?.UsesRtLayer == true) + { + _vtgWritesRtLayer = true; + } + } } _context.Renderer.Pipeline.SetProgram(gs.HostProgram); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index dc5037c56..76f2898ec 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -66,6 +66,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private bool _syncActionRegistered; private int _referenceCount = 1; + private ulong _hostGpuAddress; private ulong _dirtyStart = ulong.MaxValue; private ulong _dirtyEnd = ulong.MaxValue; @@ -170,6 +171,16 @@ namespace Ryujinx.Graphics.Gpu.Memory return new BufferRange(Handle, offset, (int)size); } + public ulong GetHostGpuAddress(ulong address) + { + if (_hostGpuAddress == 0) + { + _hostGpuAddress = _context.Renderer.GetBufferGpuAddress(Handle); + } + + return _hostGpuAddress + (address - Address); + } + /// /// Checks if a given range overlaps with the buffer. /// diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index a5a9b75e9..ea117856e 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -27,7 +27,6 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Must lock for any access from other threads. /// private readonly RangeList _buffers; - private Buffer[] _bufferOverlaps; private readonly Dictionary _dirtyCache; @@ -395,6 +394,11 @@ namespace Ryujinx.Graphics.Gpu.Memory return GetBuffer(address, size, write).GetRange(address, size); } + public ulong GetBufferHostGpuAddress(ulong address, ulong size, bool write = false) + { + return GetBuffer(address, size, write).GetHostGpuAddress(address); + } + /// /// Gets a buffer for a given memory range. /// A buffer overlapping with the specified range is assumed to already exist on the cache. @@ -431,13 +435,19 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address of the memory range /// Size in bytes of the memory range - public void SynchronizeBufferRange(ulong address, ulong size) + /// Whether the buffer will be written to by this use + public void SynchronizeBufferRange(ulong address, ulong size, bool write = false) { if (size != 0) { Buffer buffer = _buffers.FindFirstOverlap(address, size); buffer.SynchronizeMemory(address, size); + + if (write) + { + buffer.SignalModified(address, size); + } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index e20e1bb68..30ff31015 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -1,5 +1,6 @@ using Ryujinx.Common; using Ryujinx.Graphics.GAL; +using Ryujinx.Graphics.Gpu.Engine.Types; using Ryujinx.Graphics.Gpu.Image; using Ryujinx.Graphics.Gpu.Shader; using Ryujinx.Graphics.Shader; @@ -107,6 +108,9 @@ namespace Ryujinx.Graphics.Gpu.Memory private bool _transformFeedbackBuffersDirty; private bool _rebind; + private bool _rebindPageTable; + + private BufferPageTable _bufferPageTable; /// /// Creates a new instance of the buffer manager. @@ -137,6 +141,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _bufferTextures = new List(); _ranges = new BufferAssignment[Constants.TotalGpUniformBuffers * Constants.ShaderStages]; + + _bufferPageTable = new BufferPageTable(context); } @@ -438,7 +444,7 @@ namespace Ryujinx.Graphics.Gpu.Memory CommitBufferTextureBindings(); // Force rebind after doing compute work. - Rebind(); + Rebind(rebindPageTable: false); } /// @@ -747,8 +753,90 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Force all bound textures and images to be rebound the next time CommitBindings is called. /// public void Rebind() + { + Rebind(rebindPageTable: true); + } + + /// + /// Force all bound textures and images to be rebound the next time CommitBindings is called. + /// + /// Indicates that the page table needs to also be rebound + public void Rebind(bool rebindPageTable) { _rebind = true; + + if (rebindPageTable) + { + _rebindPageTable = true; + } + } + + public void SynchronizeComputeStorageBuffers(bool write) + { + MemoryManager memoryManager = _channel.MemoryManager; + + var bufferCache = memoryManager.Physical.BufferCache; + + for (int index = 0; index < 16; index++) + { + ulong sbDescAddress = GetComputeUniformBufferAddress(0); + + int sbDescOffset = 0x310 + index * 0x10; + + sbDescAddress += (ulong)sbDescOffset; + + SbDescriptor sbDescriptor = _channel.MemoryManager.Physical.Read(sbDescAddress); + + ulong address = bufferCache.TranslateAndCreateBuffer(memoryManager, sbDescriptor.PackAddress(), (ulong)sbDescriptor.Size); + + if (address != 0) + { + bufferCache.SynchronizeBufferRange(address, (ulong)sbDescriptor.Size, write); + } + } + } + + public void SynchronizeGraphicsStorageBuffers(uint globalMemoryUseMask, uint globalMemoryWriteMask) + { + MemoryManager memoryManager = _channel.MemoryManager; + + var bufferCache = memoryManager.Physical.BufferCache; + + for (int stage = 0; stage < Constants.ShaderStages; stage++) + { + if ((globalMemoryUseMask & (1u << stage)) == 0) + { + continue; + } + + bool write = (globalMemoryWriteMask & (1u << stage)) != 0; + + for (int index = 0; index < 16; index++) + { + ulong sbDescAddress = GetGraphicsUniformBufferAddress(stage, 0); + + int sbDescOffset = 0x110 + stage * 0x100 + index * 0x10; + + sbDescAddress += (ulong)sbDescOffset; + + SbDescriptor sbDescriptor = memoryManager.Physical.Read(sbDescAddress); + + ulong address = bufferCache.TranslateAndCreateBuffer(memoryManager, sbDescriptor.PackAddress(), (ulong)sbDescriptor.Size); + + if (address != 0) + { + bufferCache.SynchronizeBufferRange(address, (ulong)sbDescriptor.Size, write); + } + } + } + } + + public void UpdatePageTable() + { + MemoryManager memoryManager = _channel.MemoryManager; + + _bufferPageTable.Update(memoryManager, _rebindPageTable); + _rebindPageTable = false; } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferPageTable.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferPageTable.cs new file mode 100644 index 000000000..2752e2853 --- /dev/null +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferPageTable.cs @@ -0,0 +1,201 @@ +using Ryujinx.Graphics.GAL; +using System; +using System.Numerics; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Ryujinx.Graphics.Gpu.Memory +{ + class BufferPageTable + { + private const int PageBits = MemoryManager.PtPageBits; + private const ulong PageSize = MemoryManager.PageSize; + private const ulong PageMask = MemoryManager.PageMask; + + private const int AsBits = 40; + private const ulong AsSize = 1UL << AsBits; + private const int AsPtBits = AsBits - PageBits; + private const int AsPtLevels = 2; + private const int AsPtLevelBits = AsPtBits / AsPtLevels; + + private const int PtLevel0Shift = PageBits; + private const int PtLevel1Shift = PtLevel0Shift + AsPtLevelBits; + private const ulong PtLevelMask = (1UL << AsPtLevelBits) - 1; + + private readonly GpuContext _context; + + private struct BufferMapping + { + public readonly ulong CpuAddress; + public readonly ulong GpuAddress; + public readonly ulong Size; + + public BufferMapping(ulong cpuAddress, ulong gpuAddress, ulong size) + { + CpuAddress = cpuAddress; + GpuAddress = gpuAddress; + Size = size; + } + } + + private BufferMapping[] _mappings; + private BufferHandle _bufferMap; + private ulong _bufferMapHostGpuAddress; + private int _bufferMapSize; + + private readonly Dictionary _blockIdMap; + private readonly ulong[] _blockBitmap; + + private readonly int[] _idMap; + private bool _idMapDataDirty; + + public BufferPageTable(GpuContext context) + { + _context = context; + + _blockIdMap = new Dictionary(); + _blockBitmap = new ulong[((1 << AsPtLevelBits) + 63) / 64]; + + _idMap = new int[1 << AsPtLevelBits]; + } + + public void Update(MemoryManager memoryManager, bool forceUpdate) + { + BufferCache bufferCache = memoryManager.Physical.BufferCache; + + if (memoryManager.MappingsModified || forceUpdate) + { + Mapping[] mappings = memoryManager.GetMappings(); + + BufferMapping[] bufferMappings = new BufferMapping[mappings.Length]; + + for (int i = 0; i < mappings.Length; i++) + { + Mapping mapping = mappings[i]; + ulong cpuAddress = bufferCache.TranslateAndCreateBuffer(memoryManager, mapping.Address, mapping.Size); + + bufferMappings[i] = new BufferMapping(cpuAddress, mapping.Address, mapping.Size); + } + + _mappings = bufferMappings; + + for (int i = 0; i < bufferMappings.Length; i++) + { + BufferMapping mapping = bufferMappings[i]; + + ulong hostAddress = 0; + + if (mapping.CpuAddress != 0) + { + hostAddress = bufferCache.GetBufferHostGpuAddress(mapping.CpuAddress, mapping.Size); + } + + Map(hostAddress, mapping.GpuAddress, mapping.Size); + } + + if (_idMapDataDirty) + { + BufferHandle bufferMap = EnsureBufferMap(_idMap.Length * sizeof(int)); + _context.Renderer.SetBufferData(bufferMap, 0, MemoryMarshal.Cast(_idMap)); + + _idMapDataDirty = false; + } + + _context.Renderer.Pipeline.UpdatePageTableGpuAddress(_bufferMapHostGpuAddress); + } + } + + private void Map(ulong hostAddress, ulong guestAddress, ulong size) + { + ulong endGuestAddress = guestAddress + size; + ulong blockSize = PageSize << AsPtLevelBits; + + while (guestAddress < endGuestAddress) + { + ulong nextGuestAddress = (guestAddress + blockSize) & ~(blockSize - 1); + + ulong chunckSize = Math.Min(nextGuestAddress - guestAddress, endGuestAddress - guestAddress); + + int pages = (int)(chunckSize / PageSize); + + int blockRegionOffset = sizeof(uint) << AsPtLevelBits; + int blockOffset = GetBlockId(guestAddress) * (sizeof(ulong) << AsPtLevelBits); + int blockInnerOffset = (int)((guestAddress >> PtLevel0Shift) & PtLevelMask) * sizeof(ulong); + int baseOffset = blockRegionOffset + blockOffset + blockInnerOffset; + + ulong[] data = new ulong[pages]; + + for (int page = 0; page < pages; page++) + { + data[page] = hostAddress; + + if (hostAddress != 0) + { + hostAddress += PageSize; + } + } + + BufferHandle bufferMap = EnsureBufferMap(blockRegionOffset + blockOffset + (sizeof(ulong) << AsPtLevelBits)); + _context.Renderer.SetBufferData(bufferMap, baseOffset, MemoryMarshal.Cast(data)); + + guestAddress += chunckSize; + } + } + + private BufferHandle EnsureBufferMap(int requiredSize) + { + if (requiredSize > _bufferMapSize) + { + BufferHandle newBuffer = _context.Renderer.CreateBuffer(requiredSize); + + if (_bufferMap != BufferHandle.Null) + { + _context.Renderer.Pipeline.CopyBuffer(_bufferMap, newBuffer, 0, 0, _bufferMapSize); + _context.Renderer.DeleteBuffer(_bufferMap); + } + + _bufferMap = newBuffer; + _bufferMapHostGpuAddress = _context.Renderer.GetBufferGpuAddress(_bufferMap); + _bufferMapSize = requiredSize; + } + + return _bufferMap; + } + + private int GetBlockId(ulong address) + { + int blockIndex = (int)((address >> PtLevel1Shift) & PtLevelMask); + + if (!_blockIdMap.TryGetValue(blockIndex, out int mappedIndex)) + { + mappedIndex = AllocateNewBlock(_blockBitmap); + + _idMap[blockIndex] = mappedIndex << AsPtLevelBits; + _idMapDataDirty = true; + + _blockIdMap.Add(blockIndex, mappedIndex); + } + + return mappedIndex; + } + + private static int AllocateNewBlock(ulong[] bitmap) + { + for (int index = 0; index < bitmap.Length; index++) + { + ref ulong v = ref bitmap[index]; + + if (v == ulong.MaxValue) + { + continue; + } + + int firstFreeBit = BitOperations.TrailingZeroCount(~v); + v |= 1UL << firstFreeBit; + return index * 64 + firstFreeBit; + } + + throw new InvalidOperationException("No free space left on the texture or sampler table."); + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Mapping.cs b/src/Ryujinx.Graphics.Gpu/Memory/Mapping.cs new file mode 100644 index 000000000..e30d07bea --- /dev/null +++ b/src/Ryujinx.Graphics.Gpu/Memory/Mapping.cs @@ -0,0 +1,22 @@ +using Ryujinx.Memory.Range; + +namespace Ryujinx.Graphics.Gpu.Memory +{ + struct Mapping : IRange + { + public ulong Address { get; } + public ulong Size { get; } + public ulong EndAddress => Address + Size; + + public bool OverlapsWith(ulong address, ulong size) + { + return Address < address + size && address < EndAddress; + } + + public Mapping(ulong address, ulong size) + { + Address = address; + Size = size; + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index c7a138c98..a67065a17 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -32,6 +32,9 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly ulong[][] _pageTable; + private readonly RangeList _mappings; + internal bool MappingsModified { get; private set; } + public event EventHandler MemoryUnmapped; /// @@ -53,6 +56,7 @@ namespace Ryujinx.Graphics.Gpu.Memory Physical = physicalMemory; CounterCache = new CounterCache(); _pageTable = new ulong[PtLvl0Size][]; + _mappings = new RangeList(); MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler; MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler; MemoryUnmapped += CounterCache.MemoryUnmappedHandler; @@ -395,6 +399,11 @@ namespace Ryujinx.Graphics.Gpu.Memory { lock (_pageTable) { + if (kind == PteKind.Pitch) + { + AddMapping(va, size); + } + UnmapEventArgs e = new(va, size); MemoryUnmapped?.Invoke(this, e); @@ -416,6 +425,8 @@ namespace Ryujinx.Graphics.Gpu.Memory { lock (_pageTable) { + RemoveMapping(va, size); + // Event handlers are not expected to be thread safe. UnmapEventArgs e = new(va, size); MemoryUnmapped?.Invoke(this, e); @@ -758,5 +769,79 @@ namespace Ryujinx.Graphics.Gpu.Memory { return pte & 0xffffffffffffffUL; } + + private void AddMapping(ulong va, ulong size) + { + lock (_mappings) + { + ulong startAddress = va; + ulong endAddress = va + size; + + Mapping[] overlaps = Array.Empty(); + + int overlapsCount = _mappings.FindOverlapsNonOverlapping(va, size, ref overlaps); + for (int i = 0; i < overlapsCount; i++) + { + Mapping overlap = overlaps[i]; + + if (overlap.Address < startAddress) + { + startAddress = overlap.Address; + } + + if (overlap.EndAddress > endAddress) + { + endAddress = overlap.EndAddress; + } + + _mappings.Remove(overlap); + } + + _mappings.Add(new Mapping(startAddress, endAddress - startAddress)); + MappingsModified = true; + } + } + + private void RemoveMapping(ulong va, ulong size) + { + lock (_mappings) + { + ulong endAddress = va + size; + + Mapping[] overlaps = Array.Empty(); + + int overlapsCount = _mappings.FindOverlapsNonOverlapping(va, size, ref overlaps); + for (int i = 0; i < overlapsCount; i++) + { + Mapping overlap = overlaps[i]; + + _mappings.Remove(overlap); + + if (overlap.Address < va) + { + _mappings.Add(new Mapping(overlap.Address, va - overlap.Address)); + } + + if (overlap.EndAddress > endAddress) + { + _mappings.Add(new Mapping(endAddress, overlap.EndAddress - endAddress)); + } + } + + if (overlapsCount != 0) + { + MappingsModified = true; + } + } + } + + internal Mapping[] GetMappings() + { + lock (_mappings) + { + MappingsModified = false; + return _mappings.ToArray(); + } + } } } \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs b/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs index 7f83f5880..d834f55d4 100644 --- a/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs +++ b/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs @@ -22,7 +22,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache private const ushort FileFormatVersionMajor = 1; private const ushort FileFormatVersionMinor = 2; private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor; - private const uint CodeGenVersion = 5027; + private const uint CodeGenVersion = 4735; private const string SharedTocFileName = "shared.toc"; private const string SharedDataFileName = "shared.data"; @@ -140,6 +140,16 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache /// public ShaderStage Stage; + /// + /// Indicates if the shader uses instructions that access global memory, such as LDG, STG and ATOM. + /// + public bool UsesGlobalMemory; + + /// + /// Indicates if the shader uses instructions that modify global memory, such as STG and ATOM. + /// + public bool UsesGlobalMemoryWrite; + /// /// Indicates if the shader accesses the Instance ID built-in variable. /// @@ -774,6 +784,8 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache ShaderIdentification.None, 0, dataInfo.Stage, + dataInfo.UsesGlobalMemory, + dataInfo.UsesGlobalMemoryWrite, dataInfo.UsesInstanceId, dataInfo.UsesDrawParameters, dataInfo.UsesRtLayer, @@ -800,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache dataInfo.TexturesCount = (ushort)info.Textures.Count; dataInfo.ImagesCount = (ushort)info.Images.Count; dataInfo.Stage = info.Stage; + dataInfo.UsesGlobalMemory = info.UsesGlobalMemory; + dataInfo.UsesGlobalMemoryWrite = info.UsesGlobalMemoryWrite; dataInfo.UsesInstanceId = info.UsesInstanceId; dataInfo.UsesDrawParameters = info.UsesDrawParameters; dataInfo.UsesRtLayer = info.UsesRtLayer; diff --git a/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs b/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs index b85423cb3..481616517 100644 --- a/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs +++ b/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs @@ -31,6 +31,7 @@ namespace Ryujinx.Graphics.Gpu.Shader public ResourceCounts() { UniformBuffersCount = 1; // The first binding is reserved for the support buffer. + StorageBuffersCount = 1; // The first binding is reserved for the buffer mappings table for GPU address translation. } } } \ No newline at end of file diff --git a/src/Ryujinx.Graphics.OpenGL/Buffer.cs b/src/Ryujinx.Graphics.OpenGL/Buffer.cs index 2a5143101..a678d35ef 100644 --- a/src/Ryujinx.Graphics.OpenGL/Buffer.cs +++ b/src/Ryujinx.Graphics.OpenGL/Buffer.cs @@ -94,6 +94,14 @@ namespace Ryujinx.Graphics.OpenGL } } + public static ulong GetGpuAddress(BufferHandle handle) + { + GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle.ToInt32()); + GL.NV.MakeBufferResident((NvShaderBufferLoad)BufferTarget.CopyWriteBuffer, (NvShaderBufferLoad)All.ReadWrite); + GL.NV.GetBufferParameter(BufferTargetArb.CopyWriteBuffer, NvShaderBufferLoad.BufferGpuAddressNv, out ulong gpuAddress); + return gpuAddress; + } + public static void Resize(BufferHandle handle, int size) { GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle.ToInt32()); diff --git a/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs b/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs index 161191b85..1fe62cd6e 100644 --- a/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs +++ b/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs @@ -124,6 +124,11 @@ namespace Ryujinx.Graphics.OpenGL return Buffer.GetData(this, buffer, offset, size); } + public ulong GetBufferGpuAddress(BufferHandle buffer) + { + return Buffer.GetGpuAddress(buffer); + } + public Capabilities GetCapabilities() { bool intelWindows = HwCapabilities.Vendor == HwCapabilities.GpuVendor.IntelWindows; diff --git a/src/Ryujinx.Graphics.OpenGL/Pipeline.cs b/src/Ryujinx.Graphics.OpenGL/Pipeline.cs index 6b6d0289c..151dd9f77 100644 --- a/src/Ryujinx.Graphics.OpenGL/Pipeline.cs +++ b/src/Ryujinx.Graphics.OpenGL/Pipeline.cs @@ -1548,6 +1548,11 @@ namespace Ryujinx.Graphics.OpenGL return (_boundDrawFramebuffer, _boundReadFramebuffer); } + public void UpdatePageTableGpuAddress(ulong address) + { + _supportBuffer.UpdatePageTableBasePointer(address); + } + public void UpdateRenderScale(ReadOnlySpan scales, int totalCount, int fragmentCount) { bool changed = false; diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs index 1bd0182b5..91cb23667 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs @@ -60,6 +60,22 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl context.AppendLine("#extension GL_NV_geometry_shader_passthrough : enable"); } + if ((info.HelperFunctionsMask & HelperFunctionsMask.GlobalMemory) != 0) + { + context.AppendLine("#extension GL_EXT_shader_16bit_storage : enable"); + context.AppendLine("#extension GL_EXT_shader_8bit_storage : enable"); + + if (context.Config.Options.TargetApi == TargetApi.Vulkan) + { + context.AppendLine("#extension GL_EXT_buffer_reference : enable"); + context.AppendLine("#extension GL_EXT_buffer_reference_uvec2 : enable"); + } + else + { + context.AppendLine("#extension GL_NV_shader_buffer_load : enable"); + } + } + if (context.Config.GpuAccessor.QueryHostSupportsViewportMask()) { context.AppendLine("#extension GL_NV_viewport_array2 : enable"); @@ -255,6 +271,13 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl AppendHelperFunction(context, "Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/AtomicMinMaxS32Storage.glsl"); } + if ((info.HelperFunctionsMask & HelperFunctionsMask.GlobalMemory) != 0) + { + AppendHelperFunction(context, context.Config.Options.TargetApi == TargetApi.Vulkan + ? "Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemoryVk.glsl" + : "Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl"); + } + if ((info.HelperFunctionsMask & HelperFunctionsMask.MultiplyHighS32) != 0) { AppendHelperFunction(context, "Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/MultiplyHighS32.glsl"); diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl new file mode 100644 index 000000000..488d006aa --- /dev/null +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl @@ -0,0 +1,19 @@ +struct PTType +{ + uint blockIndices[1 << 14]; + uvec2 pointers[1 << 28]; +}; + +uvec2 Helper_TranslateAddress(uvec2 address) +{ + PTType* br = (PTType*)packPtr(s_page_table.xy); + + uint l0 = (address.x >> 12) & 0x3fff; + uint l1 = ((address.x >> 26) & 0x3f) | ((address.y << 6) & 0x3fc0); + + uvec2 hostAddress = br->pointers[br->blockIndices[l1] + l0]; + + hostAddress.x += (address.x & 0xfff); + + return hostAddress; +} \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemoryVk.glsl b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemoryVk.glsl new file mode 100644 index 000000000..6031e8723 --- /dev/null +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemoryVk.glsl @@ -0,0 +1,34 @@ +layout (buffer_reference, std430, buffer_reference_align = 8) buffer buffer_regions_block +{ + uint blockIndices[1 << 14]; + uvec2 pointers[]; +}; + +layout (buffer_reference, std430, buffer_reference_align = 1) buffer uint8_t_ptr +{ + uint8_t value; +}; + +layout (buffer_reference, std430, buffer_reference_align = 2) buffer uint16_t_ptr +{ + uint16_t value; +}; + +layout (buffer_reference, std430, buffer_reference_align = 4) buffer uint_ptr +{ + uint value; +}; + +uvec2 Helper_TranslateAddress(uvec2 address) +{ + buffer_regions_block br = buffer_regions_block(s_page_table.xy); + + uint l0 = (address.x >> 12) & 0x3fff; + uint l1 = ((address.x >> 26) & 0x3f) | ((address.y << 6) & 0x3fc0); + + uvec2 hostAddress = br.pointers[br.blockIndices[l1] + l0]; + + hostAddress.x += (address.x & 0xfff); + + return hostAddress; +} \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs index 54f35b15a..3cb420f07 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs @@ -18,5 +18,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl public static string StoreShared8 = "Helper_StoreShared8"; public static string StoreStorage16 = "Helper_StoreStorage16"; public static string StoreStorage8 = "Helper_StoreStorage8"; + + public static string TranslateAddress = "Helper_TranslateAddress"; } } \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs index 24ea66d02..8d297df3d 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs @@ -87,6 +87,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions { switch (operation.StorageKind) { + case StorageKind.GlobalMemory: args += LoadGlobal(context, operation); break; case StorageKind.SharedMemory: args += LoadShared(context, operation); break; case StorageKind.StorageBuffer: args += LoadStorage(context, operation); break; @@ -167,6 +168,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions case Instruction.Load: return Load(context, operation); + case Instruction.LoadGlobal: + return LoadGlobal(context, operation); + case Instruction.LoadLocal: return LoadLocal(context, operation); @@ -191,6 +195,15 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions case Instruction.Store: return Store(context, operation); + case Instruction.StoreGlobal: + return StoreGlobal(context, operation); + + case Instruction.StoreGlobal16: + return StoreGlobal16(context, operation); + + case Instruction.StoreGlobal8: + return StoreGlobal8(context, operation); + case Instruction.StoreLocal: return StoreLocal(context, operation); diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs index 6cf36a2a6..0eda6f87f 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs @@ -83,6 +83,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions Add(Instruction.ImageAtomic, InstType.Special); Add(Instruction.IsNan, InstType.CallUnary, "isnan"); Add(Instruction.Load, InstType.Special); + Add(Instruction.LoadGlobal, InstType.Special); Add(Instruction.LoadLocal, InstType.Special); Add(Instruction.LoadShared, InstType.Special); Add(Instruction.LoadStorage, InstType.Special); @@ -119,6 +120,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions Add(Instruction.Sine, InstType.CallUnary, "sin"); Add(Instruction.SquareRoot, InstType.CallUnary, "sqrt"); Add(Instruction.Store, InstType.Special); + Add(Instruction.StoreGlobal, InstType.Special); Add(Instruction.StoreLocal, InstType.Special); Add(Instruction.StoreShared, InstType.Special); Add(Instruction.StoreShared16, InstType.Special); diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs index dfc8197b6..1997ef275 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs @@ -191,6 +191,24 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions return GenerateLoadOrStore(context, operation, isStore: false); } + public static string LoadGlobal(CodeGenContext context, AstOperation operation) + { + IAstNode src1 = operation.GetSource(0); + IAstNode src2 = operation.GetSource(1); + + string addressLowExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0)); + string addressHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1)); + + if (context.Config.Options.TargetApi == TargetApi.Vulkan) + { + return $"uint_ptr({HelperFunctionNames.TranslateAddress}(uvec2({addressLowExpr}, {addressHighExpr}))).value"; + } + else + { + return $"*(uint*)packPtr({HelperFunctionNames.TranslateAddress}(uvec2({addressLowExpr}, {addressHighExpr})))"; + } + } + public static string LoadLocal(CodeGenContext context, AstOperation operation) { return LoadLocalOrShared(context, operation, DefaultNames.LocalMemoryName); @@ -274,6 +292,44 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions return GenerateLoadOrStore(context, operation, isStore: true); } + public static string StoreGlobal(CodeGenContext context, AstOperation operation) + { + return StoreGlobal(context, operation, "uint"); + } + + public static string StoreGlobal16(CodeGenContext context, AstOperation operation) + { + return StoreGlobal(context, operation, "uint16_t"); + } + + public static string StoreGlobal8(CodeGenContext context, AstOperation operation) + { + return StoreGlobal(context, operation, "uint8_t"); + } + + private static string StoreGlobal(CodeGenContext context, AstOperation operation, string type) + { + IAstNode src1 = operation.GetSource(0); + IAstNode src2 = operation.GetSource(1); + IAstNode src3 = operation.GetSource(2); + + string addressLowExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0)); + string addressHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1)); + + AggregateType srcType = OperandManager.GetNodeDestType(context, src3); + + string src = TypeConversion.ReinterpretCast(context, src3, srcType, AggregateType.U32); + + if (context.Config.Options.TargetApi == TargetApi.Vulkan) + { + return $"{type}_ptr({HelperFunctionNames.TranslateAddress}(uvec2({addressLowExpr}, {addressHighExpr}))).value = {src}"; + } + else + { + return $"*({type}*)packPtr({HelperFunctionNames.TranslateAddress}(uvec2({addressLowExpr}, {addressHighExpr}))) = {src}"; + } + } + public static string StoreLocal(CodeGenContext context, AstOperation operation) { return StoreLocalOrShared(context, operation, DefaultNames.LocalMemoryName); diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs index 7af6d316e..0fd5e9289 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs @@ -24,6 +24,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv public int InputVertices { get; } public Dictionary ConstantBuffers { get; } = new Dictionary(); + public Instruction PageTablePointerType { get; set; } public Instruction StorageBuffersArray { get; set; } public Instruction LocalMemory { get; set; } public Instruction SharedMemory { get; set; } diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs index 7c242589e..0e8d86b45 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs @@ -184,6 +184,20 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv context.AddGlobalVariable(ubVariable); context.ConstantBuffers.Add(buffer.Binding, ubVariable); } + + var blockArrayType = context.TypeArray(context.TypeU32(), context.Constant(context.TypeU32(), 1 << 14)); + var pointerArrayType = context.TypeRuntimeArray(context.TypeVector(context.TypeU32(), 2)); + + context.Decorate(blockArrayType, Decoration.ArrayStride, (LiteralInteger)4); + context.Decorate(pointerArrayType, Decoration.ArrayStride, (LiteralInteger)8); + + var ptStructType = context.TypeStruct(false, blockArrayType, pointerArrayType); + + context.MemberDecorate(ptStructType, 0, Decoration.Offset, (LiteralInteger)0); + context.MemberDecorate(ptStructType, 1, Decoration.Offset, (LiteralInteger)((1 << 14) * sizeof(uint))); + context.Decorate(ptStructType, Decoration.Block); + + context.PageTablePointerType = context.TypePointer(StorageClass.PhysicalStorageBuffer, ptStructType); } private static void DeclareStorageBuffers(CodeGenContext context, BufferDescriptor[] descriptors) diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs index f088a47f3..59704668c 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs @@ -97,6 +97,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv Add(Instruction.ImageStore, GenerateImageStore); Add(Instruction.IsNan, GenerateIsNan); Add(Instruction.Load, GenerateLoad); + Add(Instruction.LoadGlobal, GenerateLoadGlobal); Add(Instruction.LoadLocal, GenerateLoadLocal); Add(Instruction.LoadShared, GenerateLoadShared); Add(Instruction.LoadStorage, GenerateLoadStorage); @@ -133,6 +134,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv Add(Instruction.Sine, GenerateSine); Add(Instruction.SquareRoot, GenerateSquareRoot); Add(Instruction.Store, GenerateStore); + Add(Instruction.StoreGlobal, GenerateStoreGlobal); + // Add(Instruction.StoreGlobal16, GenerateStoreGlobal16); + // Add(Instruction.StoreGlobal8, GenerateStoreGlobal8); Add(Instruction.StoreLocal, GenerateStoreLocal); Add(Instruction.StoreShared, GenerateStoreShared); Add(Instruction.StoreShared16, GenerateStoreShared16); @@ -865,6 +869,14 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv return GenerateLoadOrStore(context, operation, isStore: false); } + private static OperationResult GenerateLoadGlobal(CodeGenContext context, AstOperation operation) + { + var elemPointer = GetGlobalElemPointer(context, operation, context.TypeU32()); + var value = context.Load(context.TypeU32(), elemPointer, MemoryAccessMask.Aligned, 4); + + return new OperationResult(AggregateType.U32, value); + } + private static OperationResult GenerateLoadLocal(CodeGenContext context, AstOperation operation) { return GenerateLoadLocalOrShared(context, operation, StorageClass.Private, context.LocalMemory); @@ -1268,6 +1280,30 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv return GenerateLoadOrStore(context, operation, isStore: true); } + private static OperationResult GenerateStoreGlobal(CodeGenContext context, AstOperation operation) + { + var elemPointer = GetGlobalElemPointer(context, operation, context.TypeU32()); + context.Store(elemPointer, context.Get(AggregateType.U32, operation.GetSource(2)), MemoryAccessMask.Aligned, 4); + + return OperationResult.Invalid; + } + + private static OperationResult GenerateStoreGlobal16(CodeGenContext context, AstOperation operation) + { + var elemPointer = GetGlobalElemPointer(context, operation, context.TypeInt(16, 0)); + context.Store(elemPointer, context.Get(AggregateType.U32, operation.GetSource(2)), MemoryAccessMask.Aligned, 2); + + return OperationResult.Invalid; + } + + private static OperationResult GenerateStoreGlobal8(CodeGenContext context, AstOperation operation) + { + var elemPointer = GetGlobalElemPointer(context, operation, context.TypeInt(8, 0)); + context.Store(elemPointer, context.Get(AggregateType.U32, operation.GetSource(2)), MemoryAccessMask.Aligned, 1); + + return OperationResult.Invalid; + } + private static OperationResult GenerateStoreLocal(CodeGenContext context, AstOperation operation) { return GenerateStoreLocalOrShared(context, operation, StorageClass.Private, context.LocalMemory); @@ -1853,7 +1889,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv SpvInstruction elemPointer; - if (operation.StorageKind == StorageKind.StorageBuffer) + if (operation.StorageKind == StorageKind.GlobalMemory) + { + elemPointer = GetGlobalElemPointer(context, operation, context.TypeU32()); + } + else if (operation.StorageKind == StorageKind.StorageBuffer) { elemPointer = GetStorageElemPointer(context, operation); } @@ -1880,7 +1920,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv SpvInstruction elemPointer; - if (operation.StorageKind == StorageKind.StorageBuffer) + if (operation.StorageKind == StorageKind.GlobalMemory) + { + elemPointer = GetGlobalElemPointer(context, operation, context.TypeU32()); + } + else if (operation.StorageKind == StorageKind.StorageBuffer) { elemPointer = GetStorageElemPointer(context, operation); } @@ -2115,6 +2159,60 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv context.AddLabel(loopEnd); } + private static SpvInstruction GetGlobalElemPointer(CodeGenContext context, AstOperation operation, SpvInstruction elemType) + { + var vec4UintType = context.TypeVector(context.TypeU32(), 4); + var ptBasePointer = context.AccessChain(context.TypePointer(StorageClass.Uniform, vec4UintType), context.ConstantBuffers[0], context.Constant(context.TypeU32(), 5)); + ptBasePointer = context.Load(vec4UintType, ptBasePointer); + ptBasePointer = context.VectorShuffle(context.TypeVector(context.TypeU32(), 2), ptBasePointer, ptBasePointer, 0, 1); + ptBasePointer = context.Bitcast(context.PageTablePointerType, ptBasePointer); + + var addrLow = context.Get(AggregateType.U32, operation.GetSource(0)); + var addrHigh = context.Get(AggregateType.U32, operation.GetSource(1)); + + // uint l0 = (addrLow >> 12) & 0x3fff; + // uint l1 = ((addrLow >> 26) & 0x3f) | ((addrHigh << 6) & 0x3fc0); + var l0 = ShiftRightAndMask(context, addrLow, 12, 0x3fff); + var l1 = context.BitwiseOr(context.TypeU32(), + ShiftRightAndMask(context, addrLow, 26, 0x3f), + ShiftLeftAndMask(context, addrHigh, 6, 0x3fc0)); + + var blockIndexPointerType = context.TypePointer(StorageClass.PhysicalStorageBuffer, context.TypeU32()); + var blockIndex = context.AccessChain(blockIndexPointerType, ptBasePointer, context.Constant(context.TypeS32(), 0), l1); + blockIndex = context.Load(context.TypeU32(), blockIndex, MemoryAccessMask.Aligned, 4); + + var offset = context.IAdd(context.TypeU32(), blockIndex, l0); + + var vec2UintType = context.TypeVector(context.TypeU32(), 2); + var vec2UintPointerType = context.TypePointer(StorageClass.PhysicalStorageBuffer, vec2UintType); + var hostPointer = context.AccessChain(vec2UintPointerType, ptBasePointer, context.Constant(context.TypeS32(), 1), offset); + hostPointer = context.Load(vec2UintType, hostPointer, MemoryAccessMask.Aligned, 8); + + var pageOffset = context.BitwiseAnd(context.TypeU32(), addrLow, context.Constant(context.TypeU32(), 0xfff)); + + var hostPointerLow = context.IAdd(context.TypeU32(), context.CompositeExtract(context.TypeU32(), hostPointer, 0), pageOffset); + var hostPointerHigh = context.CompositeExtract(context.TypeU32(), hostPointer, 1); + + hostPointer = context.CompositeConstruct(vec2UintType, hostPointerLow, hostPointerHigh); + + var elemStructType = context.TypeStruct(false, elemType); + var elemStructPointerType = context.TypePointer(StorageClass.PhysicalStorageBuffer, elemStructType); + var elemPointerType = context.TypePointer(StorageClass.PhysicalStorageBuffer, elemType); + return context.AccessChain(elemPointerType, context.Bitcast(elemStructPointerType, hostPointer), context.Constant(context.TypeS32(), 0)); + } + + private static SpvInstruction ShiftLeftAndMask(CodeGenContext context, SpvInstruction value, int shift, int mask) + { + value = context.ShiftLeftLogical(context.TypeU32(), value, context.Constant(context.TypeS32(), shift)); + return context.BitwiseAnd(context.TypeU32(), value, context.Constant(context.TypeU32(), mask)); + } + + private static SpvInstruction ShiftRightAndMask(CodeGenContext context, SpvInstruction value, int shift, int mask) + { + value = context.ShiftRightLogical(context.TypeU32(), value, context.Constant(context.TypeS32(), shift)); + return context.BitwiseAnd(context.TypeU32(), value, context.Constant(context.TypeU32(), mask)); + } + private static OperationResult GetZeroOperationResult( CodeGenContext context, AstTextureOperation texOp, diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs index a55e09fd3..cfc815e84 100644 --- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs +++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs @@ -93,6 +93,13 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv context.AddCapability(Capability.DrawParameters); } + if ((info.HelperFunctionsMask & HelperFunctionsMask.GlobalMemory) != 0) + { + context.AddCapability(Capability.PhysicalStorageBufferAddresses); + + context.AddExtension("SPV_KHR_physical_storage_buffer"); + } + if (context.Info.IoDefinitions.Contains(new IoDefinition(StorageKind.Output, IoVariable.ViewportMask))) { context.AddExtension("SPV_NV_viewport_array2"); diff --git a/src/Ryujinx.Graphics.Shader/Decoders/Decoder.cs b/src/Ryujinx.Graphics.Shader/Decoders/Decoder.cs index c619b9bbc..a7f32f6a7 100644 --- a/src/Ryujinx.Graphics.Shader/Decoders/Decoder.cs +++ b/src/Ryujinx.Graphics.Shader/Decoders/Decoder.cs @@ -234,11 +234,6 @@ namespace Ryujinx.Graphics.Shader.Decoders op = InstTable.GetOp(address, opCode); - if (op.Props.HasFlag(InstProps.TexB)) - { - config.SetUsedFeature(FeatureFlags.Bindless); - } - if (op.Name == InstName.Ald || op.Name == InstName.Ast || op.Name == InstName.Ipa) { SetUserAttributeUses(config, op.Name, opCode); diff --git a/src/Ryujinx.Graphics.Shader/Ryujinx.Graphics.Shader.csproj b/src/Ryujinx.Graphics.Shader/Ryujinx.Graphics.Shader.csproj index 2efcbca4f..0481ba37f 100644 --- a/src/Ryujinx.Graphics.Shader/Ryujinx.Graphics.Shader.csproj +++ b/src/Ryujinx.Graphics.Shader/Ryujinx.Graphics.Shader.csproj @@ -12,6 +12,8 @@ + + diff --git a/src/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs b/src/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs index 30f0ffaa2..60005b9a1 100644 --- a/src/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs +++ b/src/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs @@ -13,6 +13,8 @@ namespace Ryujinx.Graphics.Shader public ShaderIdentification Identification { get; } public int GpLayerInputAttribute { get; } public ShaderStage Stage { get; } + public bool UsesGlobalMemory { get; } + public bool UsesGlobalMemoryWrite { get; } public bool UsesInstanceId { get; } public bool UsesDrawParameters { get; } public bool UsesRtLayer { get; } @@ -27,6 +29,8 @@ namespace Ryujinx.Graphics.Shader ShaderIdentification identification, int gpLayerInputAttribute, ShaderStage stage, + bool usesGlobalMemory, + bool usesGlobalMemoryWrite, bool usesInstanceId, bool usesDrawParameters, bool usesRtLayer, @@ -41,6 +45,8 @@ namespace Ryujinx.Graphics.Shader Identification = identification; GpLayerInputAttribute = gpLayerInputAttribute; Stage = stage; + UsesGlobalMemory = usesGlobalMemory; + UsesGlobalMemoryWrite = usesGlobalMemoryWrite; UsesInstanceId = usesInstanceId; UsesDrawParameters = usesDrawParameters; UsesRtLayer = usesRtLayer; diff --git a/src/Ryujinx.Graphics.Shader/StructuredIr/HelperFunctionsMask.cs b/src/Ryujinx.Graphics.Shader/StructuredIr/HelperFunctionsMask.cs index d45f8d4ee..d1e738a78 100644 --- a/src/Ryujinx.Graphics.Shader/StructuredIr/HelperFunctionsMask.cs +++ b/src/Ryujinx.Graphics.Shader/StructuredIr/HelperFunctionsMask.cs @@ -7,15 +7,16 @@ namespace Ryujinx.Graphics.Shader.StructuredIr { AtomicMinMaxS32Shared = 1 << 0, AtomicMinMaxS32Storage = 1 << 1, - MultiplyHighS32 = 1 << 2, - MultiplyHighU32 = 1 << 3, - Shuffle = 1 << 4, - ShuffleDown = 1 << 5, - ShuffleUp = 1 << 6, - ShuffleXor = 1 << 7, - StoreSharedSmallInt = 1 << 8, - StoreStorageSmallInt = 1 << 9, - SwizzleAdd = 1 << 10, - FSI = 1 << 11 + GlobalMemory = 1 << 2, + MultiplyHighS32 = 1 << 3, + MultiplyHighU32 = 1 << 4, + Shuffle = 1 << 5, + ShuffleDown = 1 << 6, + ShuffleUp = 1 << 7, + ShuffleXor = 1 << 8, + StoreSharedSmallInt = 1 << 9, + StoreStorageSmallInt = 1 << 10, + SwizzleAdd = 1 << 11, + FSI = 1 << 12 } } \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs b/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs index 4405c07aa..1fa51462b 100644 --- a/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs +++ b/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs @@ -269,12 +269,16 @@ namespace Ryujinx.Graphics.Shader.StructuredIr operation.SourcesCount)); } - // Those instructions needs to be emulated by using helper functions, - // because they are NVIDIA specific. Those flags helps the backend to + // Those instructions need to be emulated by using helper functions, + // because they are NVIDIA specific. Those flags help the backend to // decide which helper functions are needed on the final generated code. switch (operation.Inst) { + case Instruction.AtomicAdd: + case Instruction.AtomicAnd : + case Instruction.AtomicCompareAndSwap: case Instruction.AtomicMaxS32: + case Instruction.AtomicMaxU32: case Instruction.AtomicMinS32: if (operation.StorageKind == StorageKind.SharedMemory) { @@ -285,6 +289,29 @@ namespace Ryujinx.Graphics.Shader.StructuredIr context.Info.HelperFunctionsMask |= HelperFunctionsMask.AtomicMinMaxS32Storage; } break; + case Instruction.AtomicMinU32: + case Instruction.AtomicOr: + case Instruction.AtomicSwap: + case Instruction.AtomicXor: + context.Config.SetUsedFeature(FeatureFlags.GlobalMemory); + context.Config.SetUsedFeature(FeatureFlags.GlobalMemoryWrite); + context.Info.HelperFunctionsMask |= HelperFunctionsMask.GlobalMemory; + break; + case Instruction.LoadGlobal: + case Instruction.StoreGlobal: + case Instruction.StoreGlobal16: + case Instruction.StoreGlobal8: + context.Config.SetUsedFeature(FeatureFlags.GlobalMemory); + + if (operation.Inst == Instruction.StoreGlobal || + operation.Inst == Instruction.StoreGlobal16 || + operation.Inst == Instruction.StoreGlobal8) + { + context.Config.SetUsedFeature(FeatureFlags.GlobalMemoryWrite); + } + + context.Info.HelperFunctionsMask |= HelperFunctionsMask.GlobalMemory; + break; case Instruction.MultiplyHighS32: context.Info.HelperFunctionsMask |= HelperFunctionsMask.MultiplyHighS32; break; diff --git a/src/Ryujinx.Graphics.Shader/SupportBuffer.cs b/src/Ryujinx.Graphics.Shader/SupportBuffer.cs index 5eb7fe467..25a4c164a 100644 --- a/src/Ryujinx.Graphics.Shader/SupportBuffer.cs +++ b/src/Ryujinx.Graphics.Shader/SupportBuffer.cs @@ -20,7 +20,8 @@ namespace Ryujinx.Graphics.Shader FragmentIsBgra, ViewportInverse, FragmentRenderScaleCount, - RenderScale + RenderScale, + PageTableBasePointer } public struct SupportBuffer @@ -36,6 +37,7 @@ namespace Ryujinx.Graphics.Shader public static int FragmentRenderScaleCountOffset; public static int GraphicsRenderScaleOffset; public static int ComputeRenderScaleOffset; + public static int PageTableBasePointerOffset; public const int FragmentIsBgraCount = 8; // One for the render target, 64 for the textures, and 8 for the images. @@ -59,6 +61,7 @@ namespace Ryujinx.Graphics.Shader FragmentRenderScaleCountOffset = OffsetOf(ref instance, ref instance.FragmentRenderScaleCount); GraphicsRenderScaleOffset = OffsetOf(ref instance, ref instance.RenderScale); ComputeRenderScaleOffset = GraphicsRenderScaleOffset + FieldSize; + PageTableBasePointerOffset = OffsetOf(ref instance, ref instance.PageTableBasePointer); } internal static StructureType GetStructureType() @@ -69,7 +72,8 @@ namespace Ryujinx.Graphics.Shader new StructureField(AggregateType.Array | AggregateType.U32, "s_is_bgra", FragmentIsBgraCount), new StructureField(AggregateType.Vector4 | AggregateType.FP32, "s_viewport_inverse"), new StructureField(AggregateType.S32, "s_frag_scale_count"), - new StructureField(AggregateType.Array | AggregateType.FP32, "s_render_scale", RenderScaleMaxCount) + new StructureField(AggregateType.Array | AggregateType.FP32, "s_render_scale", RenderScaleMaxCount), + new StructureField(AggregateType.Vector4 | AggregateType.U32, "s_page_table") }); } @@ -80,5 +84,6 @@ namespace Ryujinx.Graphics.Shader // Render scale max count: 1 + 64 + 8. First scale is fragment output scale, others are textures/image inputs. public Array73> RenderScale; + public Vector4 PageTableBasePointer; } } \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs b/src/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs index e55ed13da..2f6f6d23f 100644 --- a/src/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs +++ b/src/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs @@ -18,9 +18,11 @@ namespace Ryujinx.Graphics.Shader.Translation Bindless = 1 << 2, InstanceId = 1 << 3, DrawParameters = 1 << 4, - RtLayer = 1 << 5, - IaIndexing = 1 << 7, - OaIndexing = 1 << 8, - FixedFuncAttr = 1 << 9 + GlobalMemory = 1 << 5, + GlobalMemoryWrite = 1 << 6, + RtLayer = 1 << 7, + IaIndexing = 1 << 9, + OaIndexing = 1 << 10, + FixedFuncAttr = 1 << 11 } } diff --git a/src/Ryujinx.Graphics.Shader/Translation/Rewriter.cs b/src/Ryujinx.Graphics.Shader/Translation/Rewriter.cs index 866ae5223..79682bd12 100644 --- a/src/Ryujinx.Graphics.Shader/Translation/Rewriter.cs +++ b/src/Ryujinx.Graphics.Shader/Translation/Rewriter.cs @@ -77,10 +77,10 @@ namespace Ryujinx.Graphics.Shader.Translation nextNode = node.Next; } - else if (UsesGlobalMemory(operation.Inst, operation.StorageKind)) + /* else if (UsesGlobalMemory(operation.Inst, operation.StorageKind)) { nextNode = RewriteGlobalAccess(node, config)?.Next ?? nextNode; - } + } */ node = nextNode; } diff --git a/src/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs b/src/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs index 40a32e2dc..526ee8872 100644 --- a/src/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs +++ b/src/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs @@ -509,7 +509,15 @@ namespace Ryujinx.Graphics.Shader.Translation NextInputAttributesComponents = config.ThisInputAttributesComponents; NextUsedInputAttributesPerPatch = config.UsedInputAttributesPerPatch; NextUsesFixedFuncAttributes = config.UsedFeatures.HasFlag(FeatureFlags.FixedFuncAttr); - MergeOutputUserAttributes(config.UsedInputAttributes, config.UsedInputAttributesPerPatch); + MergeOutputUserAttributes(config.UsedInputAttributes | config.PassthroughAttributes, config.UsedInputAttributesPerPatch); + + int passthroughAttributes = config.PassthroughAttributes; + while (passthroughAttributes != 0) + { + int bit = BitOperations.TrailingZeroCount(passthroughAttributes); + NextInputAttributesComponents |= new UInt128(0, 0xf) << (bit * 4); + passthroughAttributes &= ~(1 << bit); + } if (UsedOutputAttributesPerPatch.Count != 0) { @@ -966,6 +974,8 @@ namespace Ryujinx.Graphics.Shader.Translation identification, GpLayerInputAttribute, Stage, + UsedFeatures.HasFlag(FeatureFlags.GlobalMemory), + UsedFeatures.HasFlag(FeatureFlags.GlobalMemoryWrite), UsedFeatures.HasFlag(FeatureFlags.InstanceId), UsedFeatures.HasFlag(FeatureFlags.DrawParameters), UsedFeatures.HasFlag(FeatureFlags.RtLayer), diff --git a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs index 6e10fad00..0b0688814 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs @@ -360,12 +360,6 @@ namespace Ryujinx.Graphics.Vulkan } } - public BufferHandle GetHandle() - { - var handle = _bufferHandle; - return Unsafe.As(ref handle); - } - public unsafe IntPtr Map(int offset, int mappingSize) { return _map; @@ -481,6 +475,17 @@ namespace Ryujinx.Graphics.Vulkan throw new InvalidOperationException("The buffer is not host mapped."); } + public ulong GetGpuAddress() + { + BufferDeviceAddressInfo info = new BufferDeviceAddressInfo() + { + SType = StructureType.BufferDeviceAddressInfo, + Buffer = GetBuffer().GetUnsafe().Value + }; + + return _gd.Api.GetBufferDeviceAddress(_device, info); + } + public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferScoped? cbs = null, Action endRenderPass = null) { int dataSize = Math.Min(data.Length, Size - offset); diff --git a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs index e046bf899..1eac7eae1 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs @@ -166,9 +166,17 @@ namespace Ryujinx.Graphics.Vulkan { usage |= BufferUsageFlags.ConditionalRenderingBitExt; } - else if (gd.Capabilities.SupportsIndirectParameters) + else { - usage |= BufferUsageFlags.IndirectBufferBit; + if (gd.Capabilities.SupportsIndirectParameters) + { + usage |= BufferUsageFlags.IndirectBufferBit; + } + + if (gd.Capabilities.SupportsBufferDeviceAddress) + { + usage |= BufferUsageFlags.ShaderDeviceAddressBitExt; + } } var bufferCreateInfo = new BufferCreateInfo() @@ -484,6 +492,16 @@ namespace Ryujinx.Graphics.Vulkan return new PinnedSpan(); } + public ulong GetBufferGpuAddress(BufferHandle handle) + { + if (TryGetBuffer(handle, out var holder)) + { + return holder.GetGpuAddress(); + } + + return 0; + } + public void SetData(BufferHandle handle, int offset, ReadOnlySpan data) where T : unmanaged { SetData(handle, offset, MemoryMarshal.Cast(data), null, null); diff --git a/src/Ryujinx.Graphics.Vulkan/Constants.cs b/src/Ryujinx.Graphics.Vulkan/Constants.cs index f43d815ab..083f33e47 100644 --- a/src/Ryujinx.Graphics.Vulkan/Constants.cs +++ b/src/Ryujinx.Graphics.Vulkan/Constants.cs @@ -12,7 +12,7 @@ public const int MaxStorageBuffersPerStage = 16; public const int MaxTexturesPerStage = 64; public const int MaxImagesPerStage = 16; - public const int MaxUniformBufferBindings = MaxUniformBuffersPerStage * MaxShaderStages; + public const int MaxUniformBufferBindings = MaxUniformBuffersPerStage * MaxShaderStages + 1; public const int MaxStorageBufferBindings = MaxStorageBuffersPerStage * MaxShaderStages; public const int MaxTextureBindings = MaxTexturesPerStage * MaxShaderStages; public const int MaxImageBindings = MaxImagesPerStage * MaxShaderStages; diff --git a/src/Ryujinx.Graphics.Vulkan/DescriptorSetManager.cs b/src/Ryujinx.Graphics.Vulkan/DescriptorSetManager.cs index a88bb7b12..6f3f2aacf 100644 --- a/src/Ryujinx.Graphics.Vulkan/DescriptorSetManager.cs +++ b/src/Ryujinx.Graphics.Vulkan/DescriptorSetManager.cs @@ -26,7 +26,7 @@ namespace Ryujinx.Graphics.Vulkan var poolSizes = new DescriptorPoolSize[] { - new DescriptorPoolSize(DescriptorType.UniformBuffer, (1 + Constants.MaxUniformBufferBindings) * DescriptorPoolMultiplier), + new DescriptorPoolSize(DescriptorType.UniformBuffer, Constants.MaxUniformBufferBindings * DescriptorPoolMultiplier), new DescriptorPoolSize(DescriptorType.StorageBuffer, Constants.MaxStorageBufferBindings * DescriptorPoolMultiplier), new DescriptorPoolSize(DescriptorType.CombinedImageSampler, Constants.MaxTextureBindings * DescriptorPoolMultiplier), new DescriptorPoolSize(DescriptorType.StorageImage, Constants.MaxImageBindings * DescriptorPoolMultiplier), diff --git a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs index f600d93f0..c56abedb3 100644 --- a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs +++ b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs @@ -27,6 +27,7 @@ namespace Ryujinx.Graphics.Vulkan public readonly bool SupportsGeometryShaderPassthrough; public readonly bool SupportsSubgroupSizeControl; public readonly bool SupportsShaderInt8; + public readonly bool SupportsBufferDeviceAddress; public readonly bool SupportsShaderStencilExport; public readonly bool SupportsShaderStorageImageMultisample; public readonly bool SupportsConditionalRendering; @@ -64,6 +65,7 @@ namespace Ryujinx.Graphics.Vulkan bool supportsGeometryShaderPassthrough, bool supportsSubgroupSizeControl, bool supportsShaderInt8, + bool supportsBufferDeviceAddress, bool supportsShaderStencilExport, bool supportsShaderStorageImageMultisample, bool supportsConditionalRendering, @@ -100,6 +102,7 @@ namespace Ryujinx.Graphics.Vulkan SupportsGeometryShaderPassthrough = supportsGeometryShaderPassthrough; SupportsSubgroupSizeControl = supportsSubgroupSizeControl; SupportsShaderInt8 = supportsShaderInt8; + SupportsBufferDeviceAddress = supportsBufferDeviceAddress; SupportsShaderStencilExport = supportsShaderStencilExport; SupportsShaderStorageImageMultisample = supportsShaderStorageImageMultisample; SupportsConditionalRendering = supportsConditionalRendering; diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs index 1ee03536d..3dd4493b9 100644 --- a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs +++ b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs @@ -1386,6 +1386,11 @@ namespace Ryujinx.Graphics.Vulkan TextureBarrier(); } + public void UpdatePageTableGpuAddress(ulong address) + { + SupportBufferUpdater.UpdatePageTableBasePointer(address); + } + public void UpdateRenderScale(ReadOnlySpan scales, int totalCount, int fragmentCount) { bool changed = false; diff --git a/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs b/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs index 51a3b129a..faa3fedf2 100644 --- a/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs +++ b/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs @@ -22,6 +22,7 @@ namespace Ryujinx.Graphics.Vulkan private static readonly string[] _desirableExtensions = new string[] { + ExtBufferDeviceAddress.ExtensionName, ExtConditionalRendering.ExtensionName, ExtExtendedDynamicState.ExtensionName, ExtTransformFeedback.ExtensionName, @@ -502,6 +503,20 @@ namespace Ryujinx.Graphics.Vulkan pExtendedFeatures = &featuresSubgroupSizeControl; } + PhysicalDeviceBufferDeviceAddressFeaturesEXT featuresBufferDeviceAddress; + + if (physicalDevice.IsDeviceExtensionPresent(ExtBufferDeviceAddress.ExtensionName)) + { + featuresBufferDeviceAddress = new PhysicalDeviceBufferDeviceAddressFeaturesEXT() + { + SType = StructureType.PhysicalDeviceBufferAddressFeaturesExt, + PNext = pExtendedFeatures, + BufferDeviceAddress = true + }; + + pExtendedFeatures = &featuresBufferDeviceAddress; + } + PhysicalDeviceCustomBorderColorFeaturesEXT featuresCustomBorderColor; if (physicalDevice.IsDeviceExtensionPresent("VK_EXT_custom_border_color") && diff --git a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs index 3987be9b4..2b0f93710 100644 --- a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs +++ b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs @@ -307,6 +307,7 @@ namespace Ryujinx.Graphics.Vulkan _physicalDevice.IsDeviceExtensionPresent("VK_NV_geometry_shader_passthrough"), supportsSubgroupSizeControl, featuresShaderInt8.ShaderInt8, + _physicalDevice.IsDeviceExtensionPresent(ExtBufferDeviceAddress.ExtensionName), _physicalDevice.IsDeviceExtensionPresent("VK_EXT_shader_stencil_export"), features2.Features.ShaderStorageImageMultisample, _physicalDevice.IsDeviceExtensionPresent(ExtConditionalRendering.ExtensionName), @@ -472,6 +473,11 @@ namespace Ryujinx.Graphics.Vulkan return BufferManager.GetData(buffer, offset, size); } + public ulong GetBufferGpuAddress(BufferHandle buffer) + { + return BufferManager.GetBufferGpuAddress(buffer); + } + public unsafe Capabilities GetCapabilities() { FormatFeatureFlags compressedFormatFeatureFlags = diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index 469195973..0f3bbdf84 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -464,6 +464,18 @@ namespace Ryujinx.Memory.Range return ~left; } + public T[] ToArray() + { + T[] output = new T[Count]; + + for (int i = 0; i < output.Length; i++) + { + output[i] = _items[i].Value; + } + + return output; + } + public IEnumerator GetEnumerator() { for (int i = 0; i < Count; i++) diff --git a/src/Spv.Generator/Autogenerated/CoreGrammar.cs b/src/Spv.Generator/Autogenerated/CoreGrammar.cs index 3b2f6fa65..849aafd0d 100644 --- a/src/Spv.Generator/Autogenerated/CoreGrammar.cs +++ b/src/Spv.Generator/Autogenerated/CoreGrammar.cs @@ -710,7 +710,7 @@ namespace Spv.Generator return result; } - public Instruction Load(Instruction resultType, Instruction pointer, MemoryAccessMask memoryAccess = (MemoryAccessMask)int.MaxValue) + public Instruction Load(Instruction resultType, Instruction pointer, MemoryAccessMask memoryAccess = (MemoryAccessMask)int.MaxValue, LiteralInteger operand2 = null) { Instruction result = NewInstruction(Op.OpLoad, GetNewId(), resultType); @@ -718,13 +718,17 @@ namespace Spv.Generator if (memoryAccess != (MemoryAccessMask)int.MaxValue) { result.AddOperand(memoryAccess); + if (operand2 != null) + { + result.AddOperand(operand2); + } } AddToFunctionDefinitions(result); return result; } - public Instruction Store(Instruction pointer, Instruction obj, MemoryAccessMask memoryAccess = (MemoryAccessMask)int.MaxValue) + public Instruction Store(Instruction pointer, Instruction obj, MemoryAccessMask memoryAccess = (MemoryAccessMask)int.MaxValue, LiteralInteger operand2 = null) { Instruction result = NewInstruction(Op.OpStore); @@ -733,6 +737,10 @@ namespace Spv.Generator if (memoryAccess != (MemoryAccessMask)int.MaxValue) { result.AddOperand(memoryAccess); + if (operand2 != null) + { + result.AddOperand(operand2); + } } AddToFunctionDefinitions(result);