Memory changes 3

General memory improvements to decrease GC pressure and frequency.

Pool big arrays and objects that are created and deleted often.

Skip data copies when they aren't needed.

Inline flag checks to skip unneeded allocations.

From my testing the performance is about the same, but the GC frequency is much lower and collection is faster causing less and smaller spikes.

src/Ryujinx.HLE/HOS/Kernel/Common/KSynchronizationObject.cs

@@ -1,3 +1,4 @@
+using Ryujinx.Common;
 using Ryujinx.HLE.HOS.Kernel.Threading;
 using System.Collections.Generic;
 
@@ -5,6 +6,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Common
 {
     class KSynchronizationObject : KAutoObject
     {
+        private static readonly ObjectPool<LinkedListNode<KThread>> _nodePool = new(() => new LinkedListNode<KThread>(null));
+        
         public LinkedList<KThread> WaitingThreads { get; }
 
         public KSynchronizationObject(KernelContext context) : base(context)
@@ -14,12 +17,16 @@ namespace Ryujinx.HLE.HOS.Kernel.Common
 
         public LinkedListNode<KThread> AddWaitingThread(KThread thread)
         {
-            return WaitingThreads.AddLast(thread);
+            LinkedListNode<KThread> node = _nodePool.Allocate();
+            node.Value = thread;
+            WaitingThreads.AddLast(node);
+            return node;
         }
 
         public void RemoveWaitingThread(LinkedListNode<KThread> node)
         {
             WaitingThreads.Remove(node);
+            _nodePool.Release(node);
         }
 
         public virtual void Signal()

src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs

@@ -110,7 +110,22 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         {
             ulong size = PagesCount * KPageTableBase.PageSize;
 
-            return new KMemoryInfo(
+            return KMemoryInfo.Pool.Allocate().Set(
+                BaseAddress,
+                size,
+                State,
+                Permission,
+                Attribute,
+                SourcePermission,
+                IpcRefCount,
+                DeviceRefCount);
+        }
+        
+        public KMemoryInfo GetInfo(KMemoryInfo oldInfo)
+        {
+            ulong size = PagesCount * KPageTableBase.PageSize;
+
+            return oldInfo.Set(
                 BaseAddress,
                 size,
                 State,

src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs

@@ -1,19 +1,23 @@
+using Ryujinx.Common;
+
 namespace Ryujinx.HLE.HOS.Kernel.Memory
 {
     class KMemoryInfo
     {
-        public ulong Address { get; }
-        public ulong Size { get; }
+        public static readonly ObjectPool<KMemoryInfo> Pool = new(() => new KMemoryInfo());
+        
+        public ulong Address { get; private set; }
+        public ulong Size { get; private set; }
 
-        public MemoryState State { get; }
-        public KMemoryPermission Permission { get; }
-        public MemoryAttribute Attribute { get; }
-        public KMemoryPermission SourcePermission { get; }
+        public MemoryState State { get; private set; }
+        public KMemoryPermission Permission { get; private set; }
+        public MemoryAttribute Attribute { get;private set;  }
+        public KMemoryPermission SourcePermission { get; private set; }
 
-        public int IpcRefCount { get; }
-        public int DeviceRefCount { get; }
+        public int IpcRefCount { get; private set; }
+        public int DeviceRefCount { get; private set; }
 
-        public KMemoryInfo(
+        public KMemoryInfo Set(
             ulong address,
             ulong size,
             MemoryState state,
@@ -31,6 +35,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             SourcePermission = sourcePermission;
             IpcRefCount = ipcRefCount;
             DeviceRefCount = deviceRefCount;
+
+            return this;
         }
     }
 }

src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryPermission.cs

@@ -25,17 +25,17 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         {
             MemoryPermission output = MemoryPermission.None;
 
-            if (permission.HasFlag(KMemoryPermission.Read))
+            if ((permission & KMemoryPermission.Read) == KMemoryPermission.Read)
             {
                 output = MemoryPermission.Read;
             }
 
-            if (permission.HasFlag(KMemoryPermission.Write))
+            if ((permission & KMemoryPermission.Write) == KMemoryPermission.Write)
             {
                 output |= MemoryPermission.Write;
             }
 
-            if (permission.HasFlag(KMemoryPermission.Execute))
+            if ((permission & KMemoryPermission.Execute) == KMemoryPermission.Execute)
             {
                 output |= MemoryPermission.Execute;
             }

src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs

@@ -1006,7 +1006,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             }
             else
             {
-                return new KMemoryInfo(
+                return KMemoryInfo.Pool.Allocate().Set(
                     AddressSpaceEnd,
                     ~AddressSpaceEnd + 1,
                     MemoryState.Reserved,
@@ -2558,10 +2558,10 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             KMemoryPermission firstPermission = info.Permission;
             MemoryAttribute firstAttribute = info.Attribute;
 
-            do
+            info = currBlock.GetInfo(info);
+            
+            while (info.Address + info.Size - 1 < endAddr - 1 && (currBlock = currBlock.Successor) != null)
             {
-                info = currBlock.GetInfo();
-
                 // Check if the block state matches what we expect.
                 if (firstState != info.State ||
                      firstPermission != info.Permission ||
@@ -2573,11 +2573,16 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                     outState = MemoryState.Unmapped;
                     outPermission = KMemoryPermission.None;
                     outAttribute = MemoryAttribute.None;
+                    
+                    KMemoryInfo.Pool.Release(info);
 
                     return false;
                 }
+                
+                info = currBlock.GetInfo(info);
             }
-            while (info.Address + info.Size - 1 < endAddr - 1 && (currBlock = currBlock.Successor) != null);
+            
+            KMemoryInfo.Pool.Release(info);
 
             outState = firstState;
             outPermission = firstPermission;
@@ -2596,16 +2601,26 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             MemoryAttribute attributeMask,
             MemoryAttribute attributeExpected)
         {
-            foreach (KMemoryInfo info in IterateOverRange(address, address + size))
+            KMemoryBlock currBlock = _blockManager.FindBlock(address);
+
+            KMemoryInfo info = currBlock.GetInfo();
+
+            while (info.Address + info.Size - 1 < address + size - 1 && (currBlock = currBlock.Successor) != null)
             {
                 // Check if the block state matches what we expect.
                 if ((info.State & stateMask) != stateExpected ||
                     (info.Permission & permissionMask) != permissionExpected ||
                     (info.Attribute & attributeMask) != attributeExpected)
                 {
+                    KMemoryInfo.Pool.Release(info);
+                    
                     return false;
                 }
+                
+                info = currBlock.GetInfo(info);
             }
+            
+            KMemoryInfo.Pool.Release(info);
 
             return true;
         }
@@ -2655,6 +2670,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
                     ulong currBaseAddr = info.Address + reservedPagesCount * PageSize;
                     ulong currEndAddr = info.Address + info.Size;
+                    
+                    KMemoryInfo.Pool.Release(info);
 
                     if (aslrAddress >= regionStart &&
                         aslrAddress >= currBaseAddr &&
@@ -2735,6 +2752,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                         allocationEndAddr <= regionEndAddr &&
                         allocationEndAddr <= currEndAddr)
                     {
+                        KMemoryInfo.Pool.Release(info);
                         return address;
                     }
                 }
@@ -2745,9 +2763,11 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                 {
                     break;
                 }
-
-                info = currBlock.GetInfo();
+                
+                info = currBlock.GetInfo(info);
             }
+            
+            KMemoryInfo.Pool.Release(info);
 
             return 0;
         }

src/Ryujinx.HLE/HOS/Kernel/Process/HleProcessDebugger.cs

@@ -262,6 +262,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
             }
 
             var rwdataStart = roInfo.Address + roInfo.Size;
+            
+            KMemoryInfo.Pool.Release(roInfo);
 
             try
             {

src/Ryujinx.HLE/HOS/Kernel/Threading/KAddressArbiter.cs

@@ -1,3 +1,4 @@
+using Ryujinx.Common;
 using Ryujinx.HLE.HOS.Kernel.Common;
 using Ryujinx.HLE.HOS.Kernel.Process;
 using Ryujinx.Horizon.Common;
@@ -11,6 +12,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
     class KAddressArbiter
     {
         private const int HasListenersMask = 0x40000000;
+        private static readonly ObjectPool<KThread[]> _threadArrayPool = new(() => []);
 
         private readonly KernelContext _context;
 
@@ -198,9 +200,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
         {
             _context.CriticalSection.Enter();
 
-            WakeThreads(_condVarThreads, count, TryAcquireMutex, x => x.CondVarAddress == address);
+            static bool SignalProcessWideKeyPredicate(KThread thread, ulong address)
+            {
+                return thread.CondVarAddress == address;
+            }
 
-            if (!_condVarThreads.Any(x => x.CondVarAddress == address))
+            int validThreads = WakeThreads(_condVarThreads, count, TryAcquireMutex, SignalProcessWideKeyPredicate, address);
+
+            if (validThreads == 0)
             {
                 KernelTransfer.KernelToUser(address, 0);
             }
@@ -480,9 +487,10 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             // or negative. It is incremented if there are no threads waiting.
             int waitingCount = 0;
 
-            foreach (KThread thread in _arbiterThreads.Where(x => x.MutexAddress == address))
+            foreach (KThread thread in _arbiterThreads)
             {
-                if (++waitingCount >= count)
+                if (thread.MutexAddress == address && 
+                    ++waitingCount >= count)
                 {
                     break;
                 }
@@ -553,23 +561,55 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
                 thread.WaitingInArbitration = false;
             }
 
-            WakeThreads(_arbiterThreads, count, RemoveArbiterThread, x => x.MutexAddress == address);
+            static bool ArbiterThreadPredecate(KThread thread, ulong address)
+            {
+                return thread.MutexAddress == address;
+            }
+
+            WakeThreads(_arbiterThreads, count, RemoveArbiterThread, ArbiterThreadPredecate, address);
         }
 
-        private static void WakeThreads(
+        private static int WakeThreads(
             List<KThread> threads,
             int count,
             Action<KThread> removeCallback,
-            Func<KThread, bool> predicate)
+            Func<KThread, ulong, bool> predicate,
+            ulong address = 0)
         {
-            var candidates = threads.Where(predicate).OrderBy(x => x.DynamicPriority);
-            var toSignal = (count > 0 ? candidates.Take(count) : candidates).ToArray();
+            KThread[] candidates = _threadArrayPool.Allocate();
+            if (candidates.Length < threads.Count)
+            {
+                Array.Resize(ref candidates, threads.Count);
+            }
+            
+            int validCount = 0;
+            
+            for (int i = 0; i < threads.Count; i++)
+            {
+                if (predicate(threads[i], address))
+                {
+                    candidates[validCount++] = threads[i];
+                }
+            }
+            
+            Span<KThread> candidatesSpan = candidates.AsSpan(..validCount);
+            
+            candidatesSpan.Sort((x, y) => (x.DynamicPriority.CompareTo(y.DynamicPriority)));
 
-            foreach (KThread thread in toSignal)
+            if (count > 0)
+            {
+                candidatesSpan = candidatesSpan[..Math.Min(count, candidatesSpan.Length)];
+            }
+
+            foreach (KThread thread in candidatesSpan)
             {
                 removeCallback(thread);
                 threads.Remove(thread);
             }
+            
+            _threadArrayPool.Release(candidates);
+            
+            return validCount;
         }
     }
 }

src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs

@@ -29,8 +29,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
         public KThreadContext ThreadContext { get; private set; }
 
-        public int DynamicPriority { get; set; }
-        public ulong AffinityMask { get; set; }
+        public int DynamicPriority { get; private set; }
+        public ulong AffinityMask { get; private set; }
 
         public ulong ThreadUid { get; private set; }
         
@@ -64,18 +64,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
         public long LastScheduledTime { get; set; }
 
-        public LinkedListNode<KThread>[] SiblingsPerCore { get; private set; }
+        public readonly LinkedListNode<KThread>[] SiblingsPerCore;
 
         public LinkedList<KThread> Withholder { get; set; }
-        public LinkedListNode<KThread> WithholderNode { get; set; }
+        public readonly LinkedListNode<KThread> WithholderNode;
 
-        public LinkedListNode<KThread> ProcessListNode { get; set; }
+        public readonly LinkedListNode<KThread> ProcessListNode;
 
         private readonly LinkedList<KThread> _mutexWaiters;
-        private LinkedListNode<KThread> _mutexWaiterNode;
+        private readonly LinkedListNode<KThread> _mutexWaiterNode;
 
         private readonly LinkedList<KThread> _pinnedWaiters;
-        private LinkedListNode<KThread> _pinnedWaiterNode;
+        private readonly LinkedListNode<KThread> _pinnedWaiterNode;
 
         public KThread MutexOwner { get; private set; }
 
@@ -1045,11 +1045,11 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
             if (nextPrio != null)
             {
-                thread._mutexWaiterNode = _mutexWaiters.AddBefore(nextPrio, thread);
+                _mutexWaiters.AddBefore(nextPrio, thread._mutexWaiterNode);
             }
             else
             {
-                thread._mutexWaiterNode = _mutexWaiters.AddLast(thread);
+                _mutexWaiters.AddLast(thread._mutexWaiterNode);
             }
         }
 

src/Ryujinx.HLE/HOS/Services/Nv/INvDrvServices.cs

@@ -14,6 +14,7 @@ using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvMap;
 using Ryujinx.HLE.HOS.Services.Nv.Types;
 using Ryujinx.Memory;
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Reflection;
 
@@ -46,6 +47,8 @@ namespace Ryujinx.HLE.HOS.Services.Nv
             { "/dev/nvhost-dbg-gpu",  typeof(NvHostDbgGpuDeviceFile)  },
             { "/dev/nvhost-prof-gpu", typeof(NvHostProfGpuDeviceFile) },
         };
+        
+        private static readonly ArrayPool<byte> _byteArrayPool = ArrayPool<byte>.Create();
 
         public static IdDictionary DeviceFileIdRegistry = new();
 
@@ -471,10 +474,13 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
                 errorCode = GetIoctlArgument(context, ioctlCommand, out Span<byte> arguments);
 
-                if (!context.Memory.TryReadUnsafe(inlineInBufferPosition, (int)inlineInBufferSize, out Span<byte> inlineInBuffer))
+                byte[] inlineInBuffer = null;
+                
+                if (!context.Memory.TryReadUnsafe(inlineInBufferPosition, (int)inlineInBufferSize, out Span<byte> inlineInBufferSpan))
                 {
-                    inlineInBuffer = new byte[inlineInBufferSize];
-                    context.Memory.Read(inlineInBufferPosition, inlineInBuffer);
+                    inlineInBuffer = _byteArrayPool.Rent((int)inlineInBufferSize);
+                    inlineInBufferSpan = inlineInBuffer;
+                    context.Memory.Read(inlineInBufferPosition, inlineInBufferSpan[..(int)inlineInBufferSize]);
                 }
 
                 if (errorCode == NvResult.Success)
@@ -483,7 +489,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
                     if (errorCode == NvResult.Success)
                     {
-                        NvInternalResult internalResult = deviceFile.Ioctl2(ioctlCommand, arguments, inlineInBuffer);
+                        NvInternalResult internalResult = deviceFile.Ioctl2(ioctlCommand, arguments, inlineInBufferSpan[..(int)inlineInBufferSize]);
 
                         if (internalResult == NvInternalResult.NotImplemented)
                         {
@@ -498,6 +504,11 @@ namespace Ryujinx.HLE.HOS.Services.Nv
                         }
                     }
                 }
+                
+                if (inlineInBuffer is not null)
+                {
+                    _byteArrayPool.Return(inlineInBuffer);
+                }
             }
 
             context.ResponseData.Write((uint)errorCode);
@@ -520,10 +531,13 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
                 errorCode = GetIoctlArgument(context, ioctlCommand, out Span<byte> arguments);
 
-                if (!context.Memory.TryReadUnsafe(inlineOutBufferPosition, (int)inlineOutBufferSize, out Span<byte> inlineOutBuffer))
+                byte[] inlineOutBuffer = null;
+                
+                if (!context.Memory.TryReadUnsafe(inlineOutBufferPosition, (int)inlineOutBufferSize, out Span<byte> inlineOutBufferSpan))
                 {
-                    inlineOutBuffer = new byte[inlineOutBufferSize];
-                    context.Memory.Read(inlineOutBufferPosition, inlineOutBuffer);
+                    inlineOutBuffer = _byteArrayPool.Rent((int)inlineOutBufferSize);
+                    inlineOutBufferSpan = inlineOutBuffer;
+                    context.Memory.Read(inlineOutBufferPosition, inlineOutBufferSpan[..(int)inlineOutBufferSize]);
                 }
 
                 if (errorCode == NvResult.Success)
@@ -532,7 +546,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
                     if (errorCode == NvResult.Success)
                     {
-                        NvInternalResult internalResult = deviceFile.Ioctl3(ioctlCommand, arguments, inlineOutBuffer);
+                        NvInternalResult internalResult = deviceFile.Ioctl3(ioctlCommand, arguments, inlineOutBufferSpan[..(int)inlineOutBufferSize]);
 
                         if (internalResult == NvInternalResult.NotImplemented)
                         {
@@ -544,10 +558,15 @@ namespace Ryujinx.HLE.HOS.Services.Nv
                         if ((ioctlCommand.DirectionValue & NvIoctl.Direction.Write) != 0)
                         {
                             context.Memory.Write(context.Request.GetBufferType0x22(0).Position, arguments.ToArray());
-                            context.Memory.Write(inlineOutBufferPosition, inlineOutBuffer.ToArray());
+                            context.Memory.Write(inlineOutBufferPosition, inlineOutBufferSpan[..(int)inlineOutBufferSize].ToArray());
                         }
                     }
                 }
+                
+                if (inlineOutBuffer is not null)
+                {
+                    _byteArrayPool.Return(inlineOutBuffer);
+                }
             }
 
             context.ResponseData.Write((uint)errorCode);