Add an alternative queue, used on macOS to work around SPIRV-Cross stack overflows

- Increase stack size to 2MB

src/Ryujinx.Graphics.Vulkan/ShaderCollection.cs

@@ -60,7 +60,7 @@ namespace Ryujinx.Graphics.Vulkan
 
         private ProgramPipelineState _state;
         private DisposableRenderPass _dummyRenderPass;
-        private readonly Task _compileTask;
+        private ShaderCompilationRequest _compileRequest;
         private bool _firstBackgroundUse;
 
         public ShaderCollection(
@@ -139,7 +139,7 @@ namespace Ryujinx.Graphics.Vulkan
             // Updating buffer texture bindings using template updates crashes the Adreno driver on Windows.
             UpdateTexturesWithoutTemplate = gd.IsQualcommProprietary && usesBufferTextures;
 
-            _compileTask = Task.CompletedTask;
+            _compileRequest = new ShaderCompilationRequest(Task.CompletedTask);
             _firstBackgroundUse = false;
         }
 
@@ -153,7 +153,9 @@ namespace Ryujinx.Graphics.Vulkan
         {
             _state = state;
 
-            _compileTask = BackgroundCompilation();
+            _compileRequest = gd.ShaderCompilationQueue != null
+                ? gd.ShaderCompilationQueue.Add(BackgroundCompilation)
+                : new ShaderCompilationRequest(BackgroundCompilationAsync());
             _firstBackgroundUse = !fromCache;
         }
 
@@ -458,10 +460,25 @@ namespace Ryujinx.Graphics.Vulkan
             return (buffer, texture);
         }
 
-        private async Task BackgroundCompilation()
+        private async Task BackgroundCompilationAsync()
         {
             await Task.WhenAll(_shaders.Select(shader => shader.CompileTask));
 
+            BackgroundCompilationImpl();
+        }
+
+        private void BackgroundCompilation()
+        {
+            foreach (var shader in _shaders)
+            {
+                shader.CompileTask.Wait();
+            }
+
+            BackgroundCompilationImpl();
+        }
+
+        private void BackgroundCompilationImpl()
+        {
             if (Array.Exists(_shaders, shader => shader.CompileStatus == ProgramLinkStatus.Failure))
             {
                 LinkStatus = ProgramLinkStatus.Failure;
@@ -604,11 +621,11 @@ namespace Ryujinx.Graphics.Vulkan
                     }
                 }
 
-                if (!_compileTask.IsCompleted)
+                if (!_compileRequest.IsCompleted)
                 {
                     if (blocking)
                     {
-                        _compileTask.Wait();
+                        _compileRequest.Wait();
 
                         if (LinkStatus == ProgramLinkStatus.Failure)
                         {

src/Ryujinx.Graphics.Vulkan/ShaderCompilationQueue.cs

@@ -0,0 +1,131 @@
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Vulkan
+{
+    class ShaderCompilationQueue
+    {
+        private const int MaxParallelCompilations = 8;
+        private const int MaxThreadStackSize = 2 * 1024 * 1024; // MB
+
+        private struct Request
+        {
+            public readonly ulong Id;
+            public readonly Action Callback;
+
+            public Request(ulong id, Action callback)
+            {
+                Id = id;
+                Callback = callback;
+            }
+        }
+
+        private readonly Thread[] _workerThreads;
+        private readonly CancellationTokenSource _cts;
+        private readonly BlockingCollection<Request>[] _queues;
+        private readonly ulong[] _finishedIds;
+        private ulong _currentId;
+        private int _currentQueueIndex;
+
+        public ShaderCompilationQueue()
+        {
+            _workerThreads = new Thread[MaxParallelCompilations];
+            _queues = new BlockingCollection<Request>[MaxParallelCompilations];
+            _finishedIds = new ulong[MaxParallelCompilations];
+
+            _cts = new CancellationTokenSource();
+
+            for (int i = 0; i < MaxParallelCompilations; i++)
+            {
+                _queues[i] = new BlockingCollection<Request>();
+
+                Thread thread = new Thread(DoWork, MaxThreadStackSize) { Name = $"BackgroundShaderCompiler.{i}" };
+                thread.IsBackground = true;
+                thread.Start(i);
+
+                _workerThreads[i] = thread;
+            }
+        }
+
+        private void DoWork(object threadId)
+        {
+            int queueIndex = (int)threadId;
+
+            try
+            {
+                var queue = _queues[queueIndex];
+
+                foreach (var request in queue.GetConsumingEnumerable(_cts.Token))
+                {
+                    request.Callback();
+
+                    lock (queue)
+                    {
+                        _finishedIds[queueIndex] = request.Id;
+
+                        Monitor.PulseAll(queue);
+                    }
+                }
+            }
+            catch (OperationCanceledException)
+            {
+            }
+        }
+
+        public ShaderCompilationRequest Add(Action callback)
+        {
+            ulong newId = Interlocked.Increment(ref _currentId);
+
+            // Let's keep rotating between the queues to increase the chances
+            // that the selected queue thread is currently idle.
+            int queueIndex = Interlocked.Increment(ref _currentQueueIndex) % MaxParallelCompilations;
+
+            _queues[queueIndex].Add(new Request(newId, callback));
+
+            return new ShaderCompilationRequest(this, queueIndex, newId);
+        }
+
+        public void Wait(int queueIndex, ulong id)
+        {
+            var queue = _queues[queueIndex];
+
+            lock (queue)
+            {
+                while (_finishedIds[queueIndex] < id)
+                {
+                    Monitor.Wait(queue);
+                }
+            }
+        }
+
+        public bool IsCompleted(int queueIndex, ulong id)
+        {
+            var queue = _queues[queueIndex];
+
+            lock (queue)
+            {
+                return _finishedIds[queueIndex] >= id;
+            }
+        }
+
+        public void Dispose()
+        {
+            for (int i = 0; i < MaxParallelCompilations; i++)
+            {
+                _queues[i].CompleteAdding();
+            }
+
+            _cts.Cancel();
+
+            for (int i = 0; i < MaxParallelCompilations; i++)
+            {
+                _workerThreads[i].Join();
+
+                _queues[i].Dispose();
+            }
+
+            _cts.Dispose();
+        }
+    }
+}

src/Ryujinx.Graphics.Vulkan/ShaderCompilationRequest.cs

@@ -0,0 +1,55 @@
+using System.Threading.Tasks;
+
+namespace Ryujinx.Graphics.Vulkan
+{
+    struct ShaderCompilationRequest
+    {
+        private readonly Task _task;
+        private readonly ShaderCompilationQueue _queue;
+        private readonly int _queueIndex;
+        private readonly ulong _requestId;
+
+        public bool IsCompleted
+        {
+            get
+            {
+                if (_task != null)
+                {
+                    return _task.IsCompleted;
+                }
+                else
+                {
+                    return _queue.IsCompleted(_queueIndex, _requestId);
+                }
+            }
+        }
+
+        public ShaderCompilationRequest(Task task)
+        {
+            _task = task;
+            _queue = null;
+            _queueIndex = 0;
+            _requestId = 0;
+        }
+
+        public ShaderCompilationRequest(ShaderCompilationQueue queue, int queueIndex, ulong requestId)
+        {
+            _task = null;
+            _queue = queue;
+            _queueIndex = queueIndex;
+            _requestId = requestId;
+        }
+
+        public void Wait()
+        {
+            if (_task != null)
+            {
+                _task.Wait();
+            }
+            else
+            {
+                _queue.Wait(_queueIndex, _requestId);
+            }
+        }
+    }
+}

src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs

@@ -55,6 +55,7 @@ namespace Ryujinx.Graphics.Vulkan
         internal CommandBufferPool CommandBufferPool { get; private set; }
         internal PipelineLayoutCache PipelineLayoutCache { get; private set; }
         internal BackgroundResources BackgroundResources { get; private set; }
+        internal ShaderCompilationQueue ShaderCompilationQueue { get; private set; }
         internal Action<Action> InterruptAction { get; private set; }
         internal SyncManager SyncManager { get; private set; }
 
@@ -127,6 +128,12 @@ namespace Ryujinx.Graphics.Vulkan
 
                 // Any device running on MacOS is using MoltenVK, even Intel and AMD vendors.
                 IsMoltenVk = true;
+
+                // The default thread stack size on MacOS is low, and can cause stack overflow
+                // on SPIR-V Cross during shader compilation.
+                // As a workaround, we use this custom queue which allows us to specify the stack
+                // size of the threads used for compilation.
+                ShaderCompilationQueue = new ShaderCompilationQueue();
             }
 
             SupportsMTL31 = OperatingSystem.IsMacOSVersionAtLeast(14);
@@ -1112,6 +1119,8 @@ namespace Ryujinx.Graphics.Vulkan
 
             SurfaceApi.DestroySurface(_instance.Instance, _surface, null);
 
+            ShaderCompilationQueue?.Dispose();
+
             Api.DestroyDevice(_device, null);
 
             _debugMessenger.Dispose();