Linux: Automatically increase vm.max_map_count if it's too low (#4702)

* memory: Check results of pinvoke calls

* Increase vm.max_map_count when running Ryujinx

* Add SupportedOSPlatform attribute for WindowsApiException

* Revert increasing vm.max_map_count via script

* Add LinuxHelper to detect and increase vm.max_map_count

With GUI dialogs, this should be a bit more user-friendly.

* Supply arguments as a list to RunPkExec

* Add error logging in case RunPkExec() fails

* Prevent Gtk from crashing

src/Ryujinx.Ava/Assets/Locales/en_US.json

@@ -74,6 +74,13 @@
   "GameListContextMenuExtractDataLogoToolTip": "Extract the Logo section from Application's current config (including updates)",
   "StatusBarGamesLoaded": "{0}/{1} Games Loaded",
   "StatusBarSystemVersion": "System Version: {0}",
+  "LinuxVmMaxMapCountDialogTitle": "Low limit for memory mappings detected",
+  "LinuxVmMaxMapCountDialogTextPrimary": "Would you like to increase the value of vm.max_map_count to {0}",
+  "LinuxVmMaxMapCountDialogTextSecondary": "Some games might try to create more memory mappings than currently allowed. Ryujinx will crash as soon as this limit gets exceeded.",
+  "LinuxVmMaxMapCountDialogButtonUntilRestart": "Yes, until the next restart",
+  "LinuxVmMaxMapCountDialogButtonPersistent": "Yes, permanently",
+  "LinuxVmMaxMapCountWarningTextPrimary": "Max amount of memory mappings is lower than recommended.",
+  "LinuxVmMaxMapCountWarningTextSecondary": "The current value of vm.max_map_count ({0}) is lower than {1}. Some games might try to create more memory mappings than currently allowed. Ryujinx will crash as soon as this limit gets exceeded.\n\nYou might want to either manually increase the limit or install pkexec, which allows Ryujinx to assist with that.",
   "Settings": "Settings",
   "SettingsTabGeneral": "User Interface",
   "SettingsTabGeneralGeneral": "General",

src/Ryujinx.Ava/Ryujinx.Ava.csproj

@@ -18,6 +18,7 @@
 
   <PropertyGroup Condition="'$(RuntimeIdentifier)' != ''">
     <PublishSingleFile>true</PublishSingleFile>
+    <TrimmerSingleWarn>false</TrimmerSingleWarn>
     <PublishTrimmed>true</PublishTrimmed>
     <TrimMode>partial</TrimMode>
   </PropertyGroup>
@@ -147,4 +148,4 @@
   <ItemGroup>
     <AdditionalFiles Include="Assets\Locales\en_US.json" />
   </ItemGroup>
-</Project>
+</Project>

src/Ryujinx.Ava/UI/Helpers/ContentDialogHelper.cs

@@ -6,7 +6,6 @@ using Avalonia.Threading;
 using FluentAvalonia.Core;
 using FluentAvalonia.UI.Controls;
 using Ryujinx.Ava.Common.Locale;
-using Ryujinx.Ava.UI.Controls;
 using Ryujinx.Ava.UI.Windows;
 using Ryujinx.Common.Logging;
 using System;
@@ -19,7 +18,7 @@ namespace Ryujinx.Ava.UI.Helpers
     {
         private static bool _isChoiceDialogOpen;
 
-        public async static Task<UserResult> ShowContentDialog(
+        private async static Task<UserResult> ShowContentDialog(
              string title,
              object content,
              string primaryButton,
@@ -67,7 +66,7 @@ namespace Ryujinx.Ava.UI.Helpers
             return result;
         }
 
-        private async static Task<UserResult> ShowTextDialog(
+        public async static Task<UserResult> ShowTextDialog(
             string title,
             string primaryText,
             string secondaryText,
@@ -319,7 +318,7 @@ namespace Ryujinx.Ava.UI.Helpers
 
             Window parent = GetMainWindow();
 
-            if (parent != null && parent.IsActive && parent is MainWindow window && window.ViewModel.IsGameRunning)
+            if (parent is { IsActive: true } and MainWindow window && window.ViewModel.IsGameRunning)
             {
                 contentDialogOverlayWindow = new()
                 {

src/Ryujinx.Ava/UI/Windows/MainWindow.axaml.cs

@@ -23,6 +23,7 @@ using Ryujinx.Ui.Common.Helper;
 using System;
 using System.ComponentModel;
 using System.IO;
+using System.Runtime.Versioning;
 using System.Threading.Tasks;
 using InputManager = Ryujinx.Input.HLE.InputManager;
 
@@ -258,7 +259,64 @@ namespace Ryujinx.Ava.UI.Windows
             ApplicationHelper.Initialize(VirtualFileSystem, AccountManager, LibHacHorizonManager.RyujinxClient, this);
         }
 
-        protected void CheckLaunchState()
+        [SupportedOSPlatform("linux")]
+        private static async void ShowVmMaxMapCountWarning()
+        {
+            LocaleManager.Instance.UpdateAndGetDynamicValue(LocaleKeys.LinuxVmMaxMapCountWarningTextSecondary,
+                LinuxHelper.VmMaxMapCount, LinuxHelper.RecommendedVmMaxMapCount);
+
+            await ContentDialogHelper.CreateWarningDialog(
+                LocaleManager.Instance[LocaleKeys.LinuxVmMaxMapCountWarningTextPrimary],
+                LocaleManager.Instance[LocaleKeys.LinuxVmMaxMapCountWarningTextSecondary]
+            );
+        }
+
+        [SupportedOSPlatform("linux")]
+        private static async void ShowVmMaxMapCountDialog()
+        {
+            LocaleManager.Instance.UpdateAndGetDynamicValue(LocaleKeys.LinuxVmMaxMapCountDialogTextPrimary,
+                LinuxHelper.RecommendedVmMaxMapCount);
+
+            UserResult response = await ContentDialogHelper.ShowTextDialog(
+                $"Ryujinx - {LocaleManager.Instance[LocaleKeys.LinuxVmMaxMapCountDialogTitle]}",
+                LocaleManager.Instance[LocaleKeys.LinuxVmMaxMapCountDialogTextPrimary],
+                LocaleManager.Instance[LocaleKeys.LinuxVmMaxMapCountDialogTextSecondary],
+                LocaleManager.Instance[LocaleKeys.LinuxVmMaxMapCountDialogButtonUntilRestart],
+                LocaleManager.Instance[LocaleKeys.LinuxVmMaxMapCountDialogButtonPersistent],
+                LocaleManager.Instance[LocaleKeys.InputDialogNo],
+                (int)Symbol.Help
+            );
+
+            int rc;
+
+            switch (response)
+            {
+                case UserResult.Ok:
+                    rc = LinuxHelper.RunPkExec($"echo {LinuxHelper.RecommendedVmMaxMapCount} > {LinuxHelper.VmMaxMapCountPath}");
+                    if (rc == 0)
+                    {
+                        Logger.Info?.Print(LogClass.Application, $"vm.max_map_count set to {LinuxHelper.VmMaxMapCount} until the next restart.");
+                    }
+                    else
+                    {
+                        Logger.Error?.Print(LogClass.Application, $"Unable to change vm.max_map_count. Process exited with code: {rc}");
+                    }
+                    break;
+                case UserResult.No:
+                    rc = LinuxHelper.RunPkExec($"echo \"vm.max_map_count = {LinuxHelper.RecommendedVmMaxMapCount}\" > {LinuxHelper.SysCtlConfigPath} && sysctl -p {LinuxHelper.SysCtlConfigPath}");
+                    if (rc == 0)
+                    {
+                        Logger.Info?.Print(LogClass.Application, $"vm.max_map_count set to {LinuxHelper.VmMaxMapCount}. Written to config: {LinuxHelper.SysCtlConfigPath}");
+                    }
+                    else
+                    {
+                        Logger.Error?.Print(LogClass.Application, $"Unable to write new value for vm.max_map_count to config. Process exited with code: {rc}");
+                    }
+                    break;
+            }
+        }
+
+        private void CheckLaunchState()
         {
             if (ShowKeyErrorOnLoad)
             {
@@ -268,6 +326,20 @@ namespace Ryujinx.Ava.UI.Windows
                     UserErrorDialog.ShowUserErrorDialog(UserError.NoKeys, this));
             }
 
+            if (OperatingSystem.IsLinux() && LinuxHelper.VmMaxMapCount < LinuxHelper.RecommendedVmMaxMapCount)
+            {
+                Logger.Warning?.Print(LogClass.Application, $"The value of vm.max_map_count is lower than {LinuxHelper.RecommendedVmMaxMapCount}. ({LinuxHelper.VmMaxMapCount})");
+
+                if (LinuxHelper.PkExecPath is not null)
+                {
+                    Dispatcher.UIThread.Post(ShowVmMaxMapCountDialog);
+                }
+                else
+                {
+                    Dispatcher.UIThread.Post(ShowVmMaxMapCountWarning);
+                }
+            }
+
             if (_deferLoad)
             {
                 _deferLoad = false;