Fix the transparency issue of ASS subtitle rendering in HWA (#14024)

4 files changed, 38 insertions, 35 deletions

diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index 7c3138002..0bc922cbb 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -3947,6 +3947,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 if (hasSubs)
                 {
+                    var alphaFormatOpt = string.Empty;
                     if (hasGraphicalSubs)
                     {
                         var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH);
@@ -3964,10 +3965,13 @@ namespace MediaBrowser.Controller.MediaEncoding
                         subFilters.Add(alphaSrcFilter);
                         subFilters.Add("format=yuva420p");
                         subFilters.Add(subTextSubtitlesFilter);
+
+                        alphaFormatOpt = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayCudaAlphaFormat)
+                            ? ":alpha_format=premultiplied" : string.Empty;
                     }
 
                     subFilters.Add("hwupload=derive_device=cuda");
-                    overlayFilters.Add("overlay_cuda=eof_action=pass:repeatlast=0");
+                    overlayFilters.Add($"overlay_cuda=eof_action=pass:repeatlast=0{alphaFormatOpt}");
                 }
             }
             else
@@ -4164,6 +4168,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 if (hasSubs)
                 {
+                    var alphaFormatOpt = string.Empty;
                     if (hasGraphicalSubs)
                     {
                         var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH);
@@ -4181,10 +4186,13 @@ namespace MediaBrowser.Controller.MediaEncoding
                         subFilters.Add(alphaSrcFilter);
                         subFilters.Add("format=yuva420p");
                         subFilters.Add(subTextSubtitlesFilter);
+
+                        alphaFormatOpt = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayOpenclAlphaFormat)
+                            ? ":alpha_format=premultiplied" : string.Empty;
                     }
 
                     subFilters.Add("hwupload=derive_device=opencl");
-                    overlayFilters.Add("overlay_opencl=eof_action=pass:repeatlast=0");
+                    overlayFilters.Add($"overlay_opencl=eof_action=pass:repeatlast=0{alphaFormatOpt}");
                     overlayFilters.Add("hwmap=derive_device=d3d11va:mode=write:reverse=1");
                     overlayFilters.Add("format=d3d11");
                 }
@@ -6947,7 +6955,8 @@ namespace MediaBrowser.Controller.MediaEncoding
 
                 if (string.Equals(videoStream.Codec, "av1", StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "av1", bitDepth, hwSurface);
+                    var accelType = GetHwaccelType(state, options, "av1", bitDepth, hwSurface);
+                    return accelType + ((!string.IsNullOrEmpty(accelType) && isAfbcSupported) ? " -afbc rga" : string.Empty);
                 }
             }
 
diff --git a/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs b/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs
index a2b6e1d73..6ad953023 100644
--- a/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs
+++ b/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs
@@ -38,6 +38,16 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <summary>
         /// The transpose_opencl_reversal.
         /// </summary>
-        TransposeOpenclReversal = 6
+        TransposeOpenclReversal = 6,
+
+        /// <summary>
+        /// The overlay_opencl_alpha_format.
+        /// </summary>
+        OverlayOpenclAlphaFormat = 7,
+
+        /// <summary>
+        /// The overlay_cuda_alpha_format.
+        /// </summary>
+        OverlayCudaAlphaFormat = 8
     }
 }
diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
index 5683de169..77fd1bcd2 100644
--- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
@@ -150,15 +150,17 @@ namespace MediaBrowser.MediaEncoding.Encoder
             "overlay_rkrga"
         ];
 
-        private static readonly Dictionary<int, string[]> _filterOptionsDict = new Dictionary<int, string[]>
+        private static readonly Dictionary<FilterOptionType, (string, string)> _filterOptionsDict = new Dictionary<FilterOptionType, (string, string)>
         {
-            { 0, new string[] { "scale_cuda", "format" } },
-            { 1, new string[] { "tonemap_cuda", "GPU accelerated HDR to SDR tonemapping" } },
-            { 2, new string[] { "tonemap_opencl", "bt2390" } },
-            { 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } },
-            { 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } },
-            { 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } },
-            { 6, new string[] { "transpose_opencl", "rotate by half-turn" } }
+            { FilterOptionType.ScaleCudaFormat, ("scale_cuda", "format") },
+            { FilterOptionType.TonemapCudaName, ("tonemap_cuda", "GPU accelerated HDR to SDR tonemapping") },
+            { FilterOptionType.TonemapOpenclBt2390, ("tonemap_opencl", "bt2390") },
+            { FilterOptionType.OverlayOpenclFrameSync, ("overlay_opencl", "Action to take when encountering EOF from secondary input") },
+            { FilterOptionType.OverlayVaapiFrameSync, ("overlay_vaapi", "Action to take when encountering EOF from secondary input") },
+            { FilterOptionType.OverlayVulkanFrameSync, ("overlay_vulkan", "Action to take when encountering EOF from secondary input") },
+            { FilterOptionType.TransposeOpenclReversal, ("transpose_opencl", "rotate by half-turn") },
+            { FilterOptionType.OverlayOpenclAlphaFormat, ("overlay_opencl", "alpha_format") },
+            { FilterOptionType.OverlayCudaAlphaFormat, ("overlay_cuda", "alpha_format") }
         };
 
         private static readonly Dictionary<BitStreamFilterOptionType, (string, string)> _bsfOptionsDict = new Dictionary<BitStreamFilterOptionType, (string, string)>
@@ -294,7 +296,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
         public IEnumerable<string> GetFilters() => GetFFmpegFilters();
 
-        public IDictionary<int, bool> GetFiltersWithOption() => GetFFmpegFiltersWithOption();
+        public IDictionary<FilterOptionType, bool> GetFiltersWithOption() => _filterOptionsDict
+            .ToDictionary(item => item.Key, item => CheckFilterWithOption(item.Value.Item1, item.Value.Item2));
 
         public IDictionary<BitStreamFilterOptionType, bool> GetBitStreamFiltersWithOption() => _bsfOptionsDict
             .ToDictionary(item => item.Key, item => CheckBitStreamFilterWithOption(item.Value.Item1, item.Value.Item2));
@@ -628,20 +631,6 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return found;
         }
 
-        private Dictionary<int, bool> GetFFmpegFiltersWithOption()
-        {
-            Dictionary<int, bool> dict = new Dictionary<int, bool>();
-            for (int i = 0; i < _filterOptionsDict.Count; i++)
-            {
-                if (_filterOptionsDict.TryGetValue(i, out var val) && val.Length == 2)
-                {
-                    dict.Add(i, CheckFilterWithOption(val[0], val[1]));
-                }
-            }
-
-            return dict;
-        }
-
         private string GetProcessOutput(string path, string arguments, bool readStdErr, string? testKey)
         {
             var redirectStandardIn = !string.IsNullOrEmpty(testKey);
diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
index bf574a06f..897652fcd 100644
--- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
@@ -72,7 +72,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
         private List<string> _decoders = new List<string>();
         private List<string> _hwaccels = new List<string>();
         private List<string> _filters = new List<string>();
-        private IDictionary<int, bool> _filtersWithOption = new Dictionary<int, bool>();
+        private IDictionary<FilterOptionType, bool> _filtersWithOption = new Dictionary<FilterOptionType, bool>();
         private IDictionary<BitStreamFilterOptionType, bool> _bitStreamFiltersWithOption = new Dictionary<BitStreamFilterOptionType, bool>();
 
         private bool _isPkeyPauseSupported = false;
@@ -341,7 +341,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             _filters = list.ToList();
         }
 
-        public void SetAvailableFiltersWithOption(IDictionary<int, bool> dict)
+        public void SetAvailableFiltersWithOption(IDictionary<FilterOptionType, bool> dict)
         {
             _filtersWithOption = dict;
         }
@@ -383,12 +383,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
         /// <inheritdoc />
         public bool SupportsFilterWithOption(FilterOptionType option)
         {
-            if (_filtersWithOption.TryGetValue((int)option, out var val))
-            {
-                return val;
-            }
-
-            return false;
+            return _filtersWithOption.TryGetValue(option, out var val) && val;
         }
 
         public bool SupportsBitStreamFilterWithOption(BitStreamFilterOptionType option)