Backport pull request #18261 from jellyfin/release-12.z

Fix video quality at low bitrates when reducing bit depth

Original-merge: addc6110560966f63f3a4fafb8c0f359d6afa10b

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Cody Robibero <cody@robibe.ro>

1 files changed, 51 insertions, 9 deletions

diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index 58d7495fa4..ebfa78176a 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -88,6 +88,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         private readonly Version _minFFmpegRkmppHevcDecDoviRpu = new Version(7, 1, 1);
         private readonly Version _minFFmpegReadrateCatchupOption = new Version(8, 0);
         private readonly Version _minFFmpegNoiseBsfDrop = new Version(5, 0);
+        private readonly Version _minFFmpegAmfAv1EncCAQ = new Version(7, 1);
 
         private static readonly string[] _videoProfilesH264 =
         [
@@ -1668,14 +1669,6 @@ namespace MediaBrowser.Controller.MediaEncoding
                 // TODO: probe QSV encoders' capabilities and enable more tuning options
                 // See also https://github.com/intel/media-delivery/blob/master/doc/quality.rst
 
-                // Enable MacroBlock level bitrate control for better subjective visual quality
-                var mbbrcOpt = string.Empty;
-                if (string.Equals(videoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase)
-                    || string.Equals(videoCodec, "hevc_qsv", StringComparison.OrdinalIgnoreCase))
-                {
-                    mbbrcOpt = " -mbbrc 1";
-                }
-
                 // Some less powerful H.264 HW decoders require strict CPB size
                 // So bufsize optimizations should not be applied to them
                 int factor = 2;
@@ -1696,7 +1689,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                 int qsvInitOcc = (int)Math.Min((long)bitrate * 1 * factor, int.MaxValue);
                 int qsvBufsize = (int)Math.Min((long)bitrate * 2 * factor, int.MaxValue);
 
-                return FormattableString.Invariant($"{mbbrcOpt} -b:v {bitrate} -maxrate {qsvMaxrate} -rc_init_occupancy {qsvInitOcc} -bufsize {qsvBufsize}");
+                return string.Create(CultureInfo.InvariantCulture, $" -b:v {bitrate} -maxrate {qsvMaxrate} -rc_init_occupancy {qsvInitOcc} -bufsize {qsvBufsize}");
             }
 
             if (string.Equals(videoCodec, "h264_amf", StringComparison.OrdinalIgnoreCase)
@@ -1813,6 +1806,13 @@ namespace MediaBrowser.Controller.MediaEncoding
                 EncoderPreset[] valid_presets = [EncoderPreset.veryslow, EncoderPreset.slower, EncoderPreset.slow, EncoderPreset.medium, EncoderPreset.fast, EncoderPreset.faster, EncoderPreset.veryfast];
 
                 param += " -preset " + (valid_presets.Contains(encoderPreset) ? encoderPreset : EncoderPreset.veryfast).ToString().ToLowerInvariant();
+
+                // Enable MacroBlock level bitrate control for better subjective visual quality
+                if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase))
+                {
+                    param += " -mbbrc 1";
+                }
             }
             else if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) // h264 (h264_nvenc)
                         || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) // hevc (hevc_nvenc)
@@ -1829,6 +1829,13 @@ namespace MediaBrowser.Controller.MediaEncoding
                     EncoderPreset.faster => " -preset p2",
                     _ => " -preset p1"
                 };
+
+                // Enable Spatial Adaptive Quantization for better subjective visual quality
+                // H.264 is skipped for now because Maxwell 1 does not support it
+                if (!string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase))
+                {
+                    param += " -spatial-aq 1 -aq-strength 15";
+                }
             }
             else if (string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) // h264 (h264_amf)
                         || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) // hevc (hevc_amf)
@@ -1844,6 +1851,18 @@ namespace MediaBrowser.Controller.MediaEncoding
                     _ => " -quality speed"
                 };
 
+                // Enable Variance Based/Context Adaptive Quantization for better subjective visual quality
+                if (string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase))
+                {
+                    param += " -vbaq 1";
+                }
+                else if (string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase)
+                         && _mediaEncoder.EncoderVersion >= _minFFmpegAmfAv1EncCAQ)
+                {
+                    param += " -aq_mode caq";
+                }
+
                 if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)
                     || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase))
                 {
@@ -2693,6 +2712,28 @@ namespace MediaBrowser.Controller.MediaEncoding
             return reasons;
         }
 
+        private static int GetBitDepthCompensatedBitrate(int bitrate, MediaStream videoStream, int targetBitDepth)
+        {
+            if (targetBitDepth >= 10
+                || (videoStream.BitDepth is int bitDepth && bitDepth <= targetBitDepth)
+                || videoStream.BitRate is not int sourceBitrate)
+            {
+                return bitrate;
+            }
+
+            // Bit depth reduction requires bit rate compensation because
+            // 10-bit encoded video is significantly more efficient.
+            var factor = sourceBitrate switch
+            {
+                <= 4_000_000 => 1.5,
+                <= 8_000_000 => 1.2,
+                <= 12_000_000 => 1.1,
+                _ => 1.05
+            };
+
+            return Convert.ToInt32(bitrate * factor);
+        }
+
         public int GetVideoBitrateParamValue(BaseEncodingJobOptions request, MediaStream videoStream, string outputVideoCodec)
         {
             var bitrate = request.VideoBitRate;
@@ -2715,6 +2756,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                 if (bitrate.HasValue)
                 {
                     var inputVideoCodec = videoStream.Codec;
+                    bitrate = GetBitDepthCompensatedBitrate(bitrate.Value, videoStream, 8);
                     bitrate = ScaleBitrate(bitrate.Value, inputVideoCodec, outputVideoCodec);
 
                     // If a max bitrate was requested, don't let the scaled bitrate exceed it