Add Vulkan filtering support for AMD VAAPI (Vega/gfx9+)

This requires:
- VK_EXT_image_drm_format_modifier extension
- Linux kernel version >= 5.15
- jellyfin-ffmpeg5 >= 5.0.1-2

Signed-off-by: nyanmisaka <nst799610810@gmail.com>

2 files changed, 59 insertions, 2 deletions

diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
index 9b4b1db94..8c8fc6b0f 100644
--- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
@@ -102,7 +102,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
             "tonemap_vaapi",
             "procamp_vaapi",
             "overlay_vaapi",
-            "hwupload_vaapi"
+            "hwupload_vaapi",
+            // vulkan
+            "libplacebo",
+            "scale_vulkan",
+            "overlay_vulkan"
         };
 
         private static readonly IReadOnlyDictionary<int, string[]> _filterOptionsDict = new Dictionary<int, string[]>
@@ -111,7 +115,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
             { 1, new string[] { "tonemap_cuda", "GPU accelerated HDR to SDR tonemapping" } },
             { 2, new string[] { "tonemap_opencl", "bt2390" } },
             { 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } },
-            { 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } }
+            { 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } },
+            { 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } }
         };
 
         // These are the library versions that corresponds to our minimum ffmpeg version 4.x according to the version table below
@@ -351,6 +356,39 @@ namespace MediaBrowser.MediaEncoding.Encoder
             }
         }
 
+        public bool CheckVulkanDrmDeviceByExtensionName(string renderNodePath, string[] vulkanExtensions)
+        {
+            if (!OperatingSystem.IsLinux())
+            {
+                return false;
+            }
+
+            if (string.IsNullOrEmpty(renderNodePath))
+            {
+                return false;
+            }
+
+            try
+            {
+                var command = "-v verbose -hide_banner -init_hw_device drm=dr:" + renderNodePath + " -init_hw_device vulkan=vk@dr";
+                var output = GetProcessOutput(_encoderPath, command, true, null);
+                foreach (string ext in vulkanExtensions)
+                {
+                    if (!output.Contains(ext, StringComparison.Ordinal))
+                    {
+                        return false;
+                    }
+                }
+
+                return true;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error detecting the given drm render node path");
+                return false;
+            }
+        }
+
         private IEnumerable<string> GetHwaccelTypes()
         {
             string? output = null;
diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
index 757a01715..ec3412f90 100644
--- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
@@ -72,6 +72,16 @@ namespace MediaBrowser.MediaEncoding.Encoder
         private bool _isVaapiDeviceAmd = false;
         private bool _isVaapiDeviceInteliHD = false;
         private bool _isVaapiDeviceInteli965 = false;
+        private bool _isVaapiDeviceSupportVulkanFmtModifier = false;
+
+        private static string[] _vulkanFmtModifierExts = {
+            "VK_KHR_sampler_ycbcr_conversion",
+            "VK_EXT_image_drm_format_modifier",
+            "VK_KHR_external_memory_fd",
+            "VK_EXT_external_memory_dma_buf",
+            "VK_KHR_external_semaphore_fd",
+            "VK_EXT_external_memory_host"
+        };
 
         private Version _ffmpegVersion = null;
         private string _ffmpegPath = string.Empty;
@@ -110,6 +120,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
         public bool IsVaapiDeviceInteli965 => _isVaapiDeviceInteli965;
 
+        public bool IsVaapiDeviceSupportVulkanFmtModifier => _isVaapiDeviceSupportVulkanFmtModifier;
+
         /// <summary>
         /// Run at startup or if the user removes a Custom path from transcode page.
         /// Sets global variables FFmpegPath.
@@ -169,6 +181,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
                     _isVaapiDeviceAmd = validator.CheckVaapiDeviceByDriverName("Mesa Gallium driver", options.VaapiDevice);
                     _isVaapiDeviceInteliHD = validator.CheckVaapiDeviceByDriverName("Intel iHD driver", options.VaapiDevice);
                     _isVaapiDeviceInteli965 = validator.CheckVaapiDeviceByDriverName("Intel i965 driver", options.VaapiDevice);
+                    _isVaapiDeviceSupportVulkanFmtModifier = validator.CheckVulkanDrmDeviceByExtensionName(options.VaapiDevice, _vulkanFmtModifierExts);
+
                     if (_isVaapiDeviceAmd)
                     {
                         _logger.LogInformation("VAAPI device {RenderNodePath} is AMD GPU", options.VaapiDevice);
@@ -181,6 +195,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
                     {
                         _logger.LogInformation("VAAPI device {RenderNodePath} is Intel GPU (i965)", options.VaapiDevice);
                     }
+
+                    if (_isVaapiDeviceSupportVulkanFmtModifier)
+                    {
+                        _logger.LogInformation("VAAPI device {RenderNodePath} supports Vulkan DRM format modifier", options.VaapiDevice);
+                    }
                 }
             }