Adapting AMD VAAPI-Vulkan pipeline to FFmpeg 7.0 (#12577)

3 files changed, 52 insertions, 11 deletions

diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index 4936a296f..0530e2af0 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -67,6 +67,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         private readonly Version _minFFmpegWorkingVtHwSurface = new Version(7, 0, 1);
         private readonly Version _minFFmpegDisplayRotationOption = new Version(6, 0);
         private readonly Version _minFFmpegAdvancedTonemapMode = new Version(7, 0, 1);
+        private readonly Version _minFFmpegAlteredVaVkInterop = new Version(7, 0, 1);
 
         private static readonly Regex _validationRegex = new(ValidationRegex, RegexOptions.Compiled);
 
@@ -3367,15 +3368,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                     algorithm = "clip";
                 }
 
-                tonemapArg = ":tonemapping=" + algorithm;
-
-                if (string.Equals(mode, "max", StringComparison.OrdinalIgnoreCase)
-                    || string.Equals(mode, "rgb", StringComparison.OrdinalIgnoreCase))
-                {
-                    tonemapArg += ":tonemapping_mode=" + mode;
-                }
-
-                tonemapArg += ":peak_detect=0:color_primaries=bt709:color_trc=bt709:colorspace=bt709";
+                tonemapArg = ":tonemapping=" + algorithm + ":peak_detect=0:color_primaries=bt709:color_trc=bt709:colorspace=bt709";
 
                 if (string.Equals(range, "tv", StringComparison.OrdinalIgnoreCase)
                     || string.Equals(range, "pc", StringComparison.OrdinalIgnoreCase))
@@ -4804,8 +4797,34 @@ namespace MediaBrowser.Controller.MediaEncoding
                 if (doVkTranspose || doVkTonemap || hasSubs)
                 {
                     // map from vaapi to vulkan/drm via interop (Polaris/gfx8+).
-                    mainFilters.Add("hwmap=derive_device=vulkan");
-                    mainFilters.Add("format=vulkan");
+                    if (_mediaEncoder.EncoderVersion >= _minFFmpegAlteredVaVkInterop)
+                    {
+                        if (doVkTranspose || !_mediaEncoder.IsVaapiDeviceSupportVulkanDrmModifier)
+                        {
+                            // disable the indirect va-drm-vk mapping since it's no longer reliable.
+                            mainFilters.Add("hwmap=derive_device=drm");
+                            mainFilters.Add("format=drm_prime");
+                            mainFilters.Add("hwmap=derive_device=vulkan");
+                            mainFilters.Add("format=vulkan");
+
+                            // workaround for libplacebo using the imported vulkan frame on gfx8.
+                            if (!_mediaEncoder.IsVaapiDeviceSupportVulkanDrmModifier)
+                            {
+                                mainFilters.Add("scale_vulkan");
+                            }
+                        }
+                        else if (doVkTonemap || hasSubs)
+                        {
+                            // non ad-hoc libplacebo also accepts drm_prime direct input.
+                            mainFilters.Add("hwmap=derive_device=drm");
+                            mainFilters.Add("format=drm_prime");
+                        }
+                    }
+                    else // legacy va-vk mapping that works only in jellyfin-ffmpeg6
+                    {
+                        mainFilters.Add("hwmap=derive_device=vulkan");
+                        mainFilters.Add("format=vulkan");
+                    }
                 }
                 else
                 {
@@ -4840,6 +4859,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 var libplaceboFilter = GetLibplaceboFilter(options, "bgra", doVkTonemap, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH);
                 mainFilters.Add(libplaceboFilter);
+                mainFilters.Add("format=vulkan");
             }
 
             if (doVkTonemap && !hasSubs)
diff --git a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs
index e36106e52..c767b4a51 100644
--- a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs
+++ b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs
@@ -66,6 +66,12 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <summary>
         /// Gets a value indicating whether the configured Vaapi device supports vulkan drm format modifier.
         /// </summary>
+        /// <value><c>true</c> if the Vaapi device supports vulkan drm format modifier, <c>false</c> otherwise.</value>
+        bool IsVaapiDeviceSupportVulkanDrmModifier { get; }
+
+        /// <summary>
+        /// Gets a value indicating whether the configured Vaapi device supports vulkan drm interop via dma-buf.
+        /// </summary>
         /// <value><c>true</c> if the Vaapi device supports vulkan drm interop, <c>false</c> otherwise.</value>
         bool IsVaapiDeviceSupportVulkanDrmInterop { get; }
 
diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
index 6fb6a9552..764230feb 100644
--- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
@@ -80,8 +80,14 @@ namespace MediaBrowser.MediaEncoding.Encoder
         private bool _isVaapiDeviceAmd = false;
         private bool _isVaapiDeviceInteliHD = false;
         private bool _isVaapiDeviceInteli965 = false;
+        private bool _isVaapiDeviceSupportVulkanDrmModifier = false;
         private bool _isVaapiDeviceSupportVulkanDrmInterop = false;
 
+        private static string[] _vulkanImageDrmFmtModifierExts =
+        {
+            "VK_EXT_image_drm_format_modifier",
+        };
+
         private static string[] _vulkanExternalMemoryDmaBufExts =
         {
             "VK_KHR_external_memory_fd",
@@ -142,6 +148,9 @@ namespace MediaBrowser.MediaEncoding.Encoder
         public bool IsVaapiDeviceInteli965 => _isVaapiDeviceInteli965;
 
         /// <inheritdoc />
+        public bool IsVaapiDeviceSupportVulkanDrmModifier => _isVaapiDeviceSupportVulkanDrmModifier;
+
+        /// <inheritdoc />
         public bool IsVaapiDeviceSupportVulkanDrmInterop => _isVaapiDeviceSupportVulkanDrmInterop;
 
         [GeneratedRegex(@"[^\/\\]+?(\.[^\/\\\n.]+)?$")]
@@ -220,6 +229,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
                     _isVaapiDeviceAmd = validator.CheckVaapiDeviceByDriverName("Mesa Gallium driver", options.VaapiDevice);
                     _isVaapiDeviceInteliHD = validator.CheckVaapiDeviceByDriverName("Intel iHD driver", options.VaapiDevice);
                     _isVaapiDeviceInteli965 = validator.CheckVaapiDeviceByDriverName("Intel i965 driver", options.VaapiDevice);
+                    _isVaapiDeviceSupportVulkanDrmModifier = validator.CheckVulkanDrmDeviceByExtensionName(options.VaapiDevice, _vulkanImageDrmFmtModifierExts);
                     _isVaapiDeviceSupportVulkanDrmInterop = validator.CheckVulkanDrmDeviceByExtensionName(options.VaapiDevice, _vulkanExternalMemoryDmaBufExts);
 
                     if (_isVaapiDeviceAmd)
@@ -235,6 +245,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
                         _logger.LogInformation("VAAPI device {RenderNodePath} is Intel GPU (i965)", options.VaapiDevice);
                     }
 
+                    if (_isVaapiDeviceSupportVulkanDrmModifier)
+                    {
+                        _logger.LogInformation("VAAPI device {RenderNodePath} supports Vulkan DRM modifier", options.VaapiDevice);
+                    }
+
                     if (_isVaapiDeviceSupportVulkanDrmInterop)
                     {
                         _logger.LogInformation("VAAPI device {RenderNodePath} supports Vulkan DRM interop", options.VaapiDevice);