Linux: Fix file picker not launching from disabling core dumps

Core dumps are disabled by default on Linux, but this prevents access to the file picker due to security hardening. To work around this, core dumps are selectively enabled and disabled around the file picker tasks.

src/Ryujinx.Common/Utilities/OsUtils.cs

@@ -22,10 +22,11 @@ namespace Ryujinx.Common.Utilities
         }
 
         // "dumpable" attribute of the calling process
+        private const int PR_GET_DUMPABLE = 3;
         private const int PR_SET_DUMPABLE = 4;
 
-        [DllImport("libc", SetLastError = true)]
+        [LibraryImport("libc", SetLastError = true)]
-        private static extern int prctl(int option, int arg2);
+        private static partial int prctl(int option, int arg2);
 
         public static void SetCoreDumpable(bool dumpable)
         {
@@ -36,5 +37,13 @@ namespace Ryujinx.Common.Utilities
                 Debug.Assert(result == 0);
             }
         }
+
+        // Use the below line to display dumpable status in the console:
+        // Console.WriteLine($"{OsUtils.IsCoreDumpable()}");
+        public static bool IsCoreDumpable()
+        {
+            int result = prctl(PR_GET_DUMPABLE, 0);
+            return result == 1;
+        }
     }
 }

src/Ryujinx/Common/ApplicationHelper.cs

@@ -16,6 +16,7 @@ using Ryujinx.Ava.Common.Locale;
 using Ryujinx.Ava.UI.Controls;
 using Ryujinx.Ava.UI.Helpers;
 using Ryujinx.Common.Logging;
+using Ryujinx.Common.Utilities;
 using Ryujinx.HLE.FileSystem;
 using Ryujinx.HLE.HOS.Services.Account.Acc;
 using Ryujinx.HLE.Loaders.Processes.Extensions;
@@ -419,6 +420,8 @@ namespace Ryujinx.Ava.Common
 
         public static async Task ExtractAoc(IStorageProvider storageProvider, string updateFilePath, string updateName)
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await storageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions
             {
                 Title = LocaleManager.Instance[LocaleKeys.FolderDialogExtractTitle],
@@ -431,11 +434,18 @@ namespace Ryujinx.Ava.Common
             }
 
             ExtractAoc(result[0].Path.LocalPath, updateFilePath, updateName);
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
 
         public static async Task ExtractSection(IStorageProvider storageProvider, NcaSectionType ncaSectionType, string titleFilePath, string titleName, int programIndex = 0)
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await storageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions
             {
                 Title = LocaleManager.Instance[LocaleKeys.FolderDialogExtractTitle],
@@ -448,6 +458,11 @@ namespace Ryujinx.Ava.Common
             }
 
             ExtractSection(result[0].Path.LocalPath, ncaSectionType, titleFilePath, titleName, programIndex);
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public static (Result? result, bool canceled) CopyDirectory(FileSystemClient fs, string sourcePath, string destPath, CancellationToken token)

src/Ryujinx/Program.cs

@@ -33,6 +33,7 @@ namespace Ryujinx.Ava
         public static string ConfigurationPath { get; private set; }
         public static bool PreviewerDetached { get; private set; }
         public static bool UseHardwareAcceleration { get; private set; }
+        public static bool CoreDumpArg { get; private set; }
 
         [LibraryImport("user32.dll", SetLastError = true)]
         public static partial int MessageBoxA(nint hWnd, [MarshalAs(UnmanagedType.LPStr)] string text, [MarshalAs(UnmanagedType.LPStr)] string caption, uint type);
@@ -51,6 +52,8 @@ namespace Ryujinx.Ava
             bool noGuiArg = ConsumeCommandLineArgument(ref args, "--no-gui") || ConsumeCommandLineArgument(ref args, "nogui");
             bool coreDumpArg = ConsumeCommandLineArgument(ref args, "--core-dumps");
 
+            CoreDumpArg = coreDumpArg;
+
             // TODO: Ryujinx causes core dumps on Linux when it exits "uncleanly", eg. through an unhandled exception.
             //       This is undesirable and causes very odd behavior during development (the process stops responding, 
             //       the .NET debugger freezes or suddenly detaches, /tmp/ gets filled etc.), unless explicitly requested by the user.
@@ -81,13 +84,11 @@ namespace Ryujinx.Ava
                 .UsePlatformDetect()
                 .With(new X11PlatformOptions
                 {
-                    EnableMultiTouch = true,
                     EnableIme = true,
                     EnableInputFocusProxy = Environment.GetEnvironmentVariable("XDG_CURRENT_DESKTOP") == "gamescope",
                     RenderingMode = UseHardwareAcceleration ?
                         new[] { X11RenderingMode.Glx, X11RenderingMode.Software } :
                         new[] { X11RenderingMode.Software },
-                    UseDBusFilePicker = false,
                 })
                 .With(new Win32PlatformOptions
                 {

src/Ryujinx/UI/Controls/ApplicationContextMenu.axaml.cs

@@ -10,6 +10,7 @@ using Ryujinx.Ava.UI.Helpers;
 using Ryujinx.Ava.UI.ViewModels;
 using Ryujinx.Ava.UI.Windows;
 using Ryujinx.Common.Configuration;
+using Ryujinx.Common.Utilities;
 using Ryujinx.HLE.HOS;
 using Ryujinx.UI.App.Common;
 using Ryujinx.UI.Common.Helper;
@@ -391,6 +392,8 @@ namespace Ryujinx.Ava.UI.Controls
 
         public async void ExtractApplicationLogo_Click(object sender, RoutedEventArgs args)
         {
+            OsUtils.SetCoreDumpable(true);
+
             var viewModel = (sender as MenuItem)?.DataContext as MainWindowViewModel;
 
             if (viewModel?.SelectedApplication is { } selectedApp)
@@ -420,6 +423,11 @@ namespace Ryujinx.Ava.UI.Controls
                     fileStream.Write(selectedApp.Icon);
                 }
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public void CreateApplicationShortcut_Click(object sender, RoutedEventArgs args)

src/Ryujinx/UI/ViewModels/DownloadableContentManagerViewModel.cs

@@ -7,6 +7,7 @@ using DynamicData;
 using FluentAvalonia.UI.Controls;
 using Ryujinx.Ava.Common.Locale;
 using Ryujinx.Ava.UI.Helpers;
+using Ryujinx.Common.Utilities;
 using Ryujinx.UI.App.Common;
 using Ryujinx.UI.Common.Models;
 using System.Collections.Generic;
@@ -137,6 +138,8 @@ namespace Ryujinx.Ava.UI.ViewModels
 
         public async void Add()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await _storageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
             {
                 Title = LocaleManager.Instance[LocaleKeys.SelectDlcDialogTitle],
@@ -167,6 +170,11 @@ namespace Ryujinx.Ava.UI.ViewModels
             {
                 await ShowNewDlcAddedDialog(totalDlcAdded);
             }
+            
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         private bool AddDownloadableContent(string path, out int numDlcAdded)

src/Ryujinx/UI/ViewModels/MainWindowViewModel.cs

@@ -1140,6 +1140,8 @@ namespace Ryujinx.Ava.UI.ViewModels
 
         private async Task LoadContentFromFolder(LocaleKeys localeMessageAddedKey, LocaleKeys localeMessageRemovedKey, LoadContentFromFolderDelegate onDirsSelected)
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await StorageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions
             {
                 Title = LocaleManager.Instance[LocaleKeys.OpenFolderDialogTitle],
@@ -1163,6 +1165,11 @@ namespace Ryujinx.Ava.UI.ViewModels
                         msg, "", "", "", LocaleManager.Instance[LocaleKeys.InputDialogOk], (int)Symbol.Checkmark);
                 });
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         #endregion
@@ -1241,6 +1248,8 @@ namespace Ryujinx.Ava.UI.ViewModels
 
         public async Task InstallFirmwareFromFile()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
             {
                 AllowMultiple = false,
@@ -1271,10 +1280,17 @@ namespace Ryujinx.Ava.UI.ViewModels
             {
                 await HandleFirmwareInstallation(result[0].Path.LocalPath);
             }
+            
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public async Task InstallFirmwareFromFolder()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await StorageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions
             {
                 AllowMultiple = false,
@@ -1284,10 +1300,17 @@ namespace Ryujinx.Ava.UI.ViewModels
             {
                 await HandleFirmwareInstallation(result[0].Path.LocalPath);
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public async Task InstallKeysFromFile()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
             {
                 AllowMultiple = false,
@@ -1318,10 +1341,17 @@ namespace Ryujinx.Ava.UI.ViewModels
             {
                 await HandleKeysInstallation(result[0].Path.LocalPath);
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public async Task InstallKeysFromFolder()
         {
+            OsUtils.SetCoreDumpable(true);
+            
             var result = await StorageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions
             {
                 AllowMultiple = false,
@@ -1331,6 +1361,11 @@ namespace Ryujinx.Ava.UI.ViewModels
             {
                 await HandleKeysInstallation(result[0].Path.LocalPath);
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public void OpenRyujinxFolder()
@@ -1432,6 +1467,8 @@ namespace Ryujinx.Ava.UI.ViewModels
 
         public async Task OpenFile()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
             {
                 Title = LocaleManager.Instance[LocaleKeys.OpenFileDialogTitle],
@@ -1503,6 +1540,11 @@ namespace Ryujinx.Ava.UI.ViewModels
                     await ContentDialogHelper.CreateErrorDialog(LocaleManager.Instance[LocaleKeys.MenuBarFileOpenFromFileError]);
                 }
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public async Task LoadDlcFromFolder()
@@ -1523,6 +1565,8 @@ namespace Ryujinx.Ava.UI.ViewModels
 
         public async Task OpenFolder()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await StorageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions
             {
                 Title = LocaleManager.Instance[LocaleKeys.OpenFolderDialogTitle],
@@ -1539,6 +1583,11 @@ namespace Ryujinx.Ava.UI.ViewModels
 
                 await LoadApplication(applicationData);
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public async Task LoadApplication(ApplicationData application, bool startFullscreen = false)
@@ -1706,6 +1755,8 @@ namespace Ryujinx.Ava.UI.ViewModels
         }
         public async Task OpenBinFile()
         {
+            OsUtils.SetCoreDumpable(true);
+
             if (AppHost.Device.System.SearchingForAmiibo(out _) && IsGameRunning)
             {
                 var result = await StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
@@ -1724,6 +1775,11 @@ namespace Ryujinx.Ava.UI.ViewModels
                 {
                     AppHost.Device.System.ScanAmiiboFromBin(result[0].Path.LocalPath);
                 }
+
+                if (!Program.CoreDumpArg)
+                {
+                    OsUtils.SetCoreDumpable(false);
+                }
             }
         }
 

src/Ryujinx/UI/ViewModels/ModManagerViewModel.cs

@@ -296,6 +296,8 @@ namespace Ryujinx.Ava.UI.ViewModels
 
         public async void Add()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await _storageProvider.OpenFolderPickerAsync(new FolderPickerOpenOptions
             {
                 Title = LocaleManager.Instance[LocaleKeys.SelectModDialogTitle],
@@ -306,6 +308,11 @@ namespace Ryujinx.Ava.UI.ViewModels
             {
                 AddMod(new DirectoryInfo(folder.Path.LocalPath));
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public void DeleteAll()

src/Ryujinx/UI/ViewModels/TitleUpdateViewModel.cs

@@ -6,6 +6,7 @@ using CommunityToolkit.Mvvm.ComponentModel;
 using FluentAvalonia.UI.Controls;
 using Ryujinx.Ava.Common.Locale;
 using Ryujinx.Ava.UI.Helpers;
+using Ryujinx.Common.Utilities;
 using Ryujinx.UI.App.Common;
 using Ryujinx.UI.Common.Models;
 using System.Collections.Generic;
@@ -154,6 +155,8 @@ namespace Ryujinx.Ava.UI.ViewModels
 
         public async Task Add()
         {
+            OsUtils.SetCoreDumpable(true);
+
             var result = await StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
             {
                 AllowMultiple = true,
@@ -183,6 +186,11 @@ namespace Ryujinx.Ava.UI.ViewModels
             {
                 await ShowNewUpdatesAddedDialog(totalUpdatesAdded);
             }
+            
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         public void Save()

src/Ryujinx/UI/Views/Settings/SettingsUIView.axaml.cs

@@ -3,6 +3,7 @@ using Avalonia.Interactivity;
 using Avalonia.Platform.Storage;
 using Avalonia.VisualTree;
 using Ryujinx.Ava.UI.ViewModels;
+using Ryujinx.Common.Utilities;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -20,6 +21,8 @@ namespace Ryujinx.Ava.UI.Views.Settings
 
         private async void AddGameDirButton_OnClick(object sender, RoutedEventArgs e)
         {
+            OsUtils.SetCoreDumpable(true);
+
             string path = GameDirPathBox.Text;
 
             if (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path) && !ViewModel.GameDirectories.Contains(path))
@@ -43,6 +46,11 @@ namespace Ryujinx.Ava.UI.Views.Settings
                     }
                 }
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         private void RemoveGameDirButton_OnClick(object sender, RoutedEventArgs e)
@@ -63,6 +71,8 @@ namespace Ryujinx.Ava.UI.Views.Settings
 
         private async void AddAutoloadDirButton_OnClick(object sender, RoutedEventArgs e)
         {
+            OsUtils.SetCoreDumpable(true);
+
             string path = AutoloadDirPathBox.Text;
 
             if (!string.IsNullOrWhiteSpace(path) && Directory.Exists(path) && !ViewModel.AutoloadDirectories.Contains(path))
@@ -86,6 +96,11 @@ namespace Ryujinx.Ava.UI.Views.Settings
                     }
                 }
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         private void RemoveAutoloadDirButton_OnClick(object sender, RoutedEventArgs e)

src/Ryujinx/UI/Views/User/UserProfileImageSelectorView.axaml.cs

@@ -8,6 +8,7 @@ using Ryujinx.Ava.Common.Locale;
 using Ryujinx.Ava.UI.Controls;
 using Ryujinx.Ava.UI.Models;
 using Ryujinx.Ava.UI.ViewModels;
+using Ryujinx.Common.Utilities;
 using Ryujinx.HLE.FileSystem;
 using SkiaSharp;
 using System.Collections.Generic;
@@ -63,6 +64,8 @@ namespace Ryujinx.Ava.UI.Views.User
 
         private async void Import_OnClick(object sender, RoutedEventArgs e)
         {
+            OsUtils.SetCoreDumpable(true);
+            
             if (this.GetVisualRoot() is Window window)
             {
                 var result = await window.StorageProvider.OpenFilePickerAsync(new FilePickerOpenOptions
@@ -85,6 +88,11 @@ namespace Ryujinx.Ava.UI.Views.User
                     _parent.GoBack();
                 }
             }
+
+            if (!Program.CoreDumpArg)
+            {
+                OsUtils.SetCoreDumpable(false);
+            }
         }
 
         private void GoBack(object sender, RoutedEventArgs e)