Merge pull request #7828 from nyanmisaka/fix-dovi-tonemap

Fix Dolby Vision profile 5 and 8 to SDR HW tone-mapping

(cherry picked from commit 8595a979a872c05d0d06f9b87ed1b56c693c267d)
Signed-off-by: crobibero <cody@robibe.ro>

4 files changed, 78 insertions, 33 deletions

diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs
index 6347b908c..9173487dd 100644
--- a/Jellyfin.Api/Controllers/DynamicHlsController.cs
+++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs
@@ -1711,20 +1711,30 @@ namespace Jellyfin.Api.Controllers
                 return audioTranscodeParams;
             }
 
+            // flac and opus are experimental in mp4 muxer
+            var strictArgs = string.Empty;
+
+            if (string.Equals(state.ActualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase)
+                || string.Equals(state.ActualOutputAudioCodec, "opus", StringComparison.OrdinalIgnoreCase))
+            {
+                strictArgs = " -strict -2";
+            }
+
             if (EncodingHelper.IsCopyCodec(audioCodec))
             {
                 var videoCodec = _encodingHelper.GetVideoEncoder(state, _encodingOptions);
                 var bitStreamArgs = EncodingHelper.GetAudioBitStreamArguments(state, state.Request.SegmentContainer, state.MediaSource.Container);
+                var copyArgs = "-codec:a:0 copy" + bitStreamArgs + strictArgs;
 
                 if (EncodingHelper.IsCopyCodec(videoCodec) && state.EnableBreakOnNonKeyFrames(videoCodec))
                 {
-                    return "-codec:a:0 copy -strict -2 -copypriorss:a:0 0" + bitStreamArgs;
+                    return copyArgs + " -copypriorss:a:0 0";
                 }
 
-                return "-codec:a:0 copy -strict -2" + bitStreamArgs;
+                return copyArgs;
             }
 
-            var args = "-codec:a:0 " + audioCodec;
+            var args = "-codec:a:0 " + audioCodec + strictArgs;
 
             var channels = state.OutputAudioChannels;
 
@@ -1779,11 +1789,12 @@ namespace Jellyfin.Api.Controllers
                 || string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase))
             {
                 if (EncodingHelper.IsCopyCodec(codec)
-                    && (string.Equals(state.VideoStream.CodecTag, "dvh1", StringComparison.OrdinalIgnoreCase)
+                    && (string.Equals(state.VideoStream.CodecTag, "dovi", StringComparison.OrdinalIgnoreCase)
+                        || string.Equals(state.VideoStream.CodecTag, "dvh1", StringComparison.OrdinalIgnoreCase)
                         || string.Equals(state.VideoStream.CodecTag, "dvhe", StringComparison.OrdinalIgnoreCase)))
                 {
                     // Prefer dvh1 to dvhe
-                    args += " -tag:v:0 dvh1";
+                    args += " -tag:v:0 dvh1 -strict -2";
                 }
                 else
                 {
diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index bffd3a1ca..eb344b528 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -13,11 +13,13 @@ using System.Threading;
 using Jellyfin.Data.Enums;
 using Jellyfin.Extensions;
 using MediaBrowser.Common.Configuration;
+using MediaBrowser.Controller.Extensions;
 using MediaBrowser.Model.Configuration;
 using MediaBrowser.Model.Dlna;
 using MediaBrowser.Model.Dto;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.MediaInfo;
+using Microsoft.Extensions.Configuration;
 
 namespace MediaBrowser.Controller.MediaEncoding
 {
@@ -32,6 +34,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         private readonly IApplicationPaths _appPaths;
         private readonly IMediaEncoder _mediaEncoder;
         private readonly ISubtitleEncoder _subtitleEncoder;
+        private readonly IConfiguration _config;
 
         private static readonly string[] _videoProfilesH264 = new[]
         {
@@ -54,11 +57,13 @@ namespace MediaBrowser.Controller.MediaEncoding
         public EncodingHelper(
             IApplicationPaths appPaths,
             IMediaEncoder mediaEncoder,
-            ISubtitleEncoder subtitleEncoder)
+            ISubtitleEncoder subtitleEncoder,
+            IConfiguration config)
         {
             _appPaths = appPaths;
             _mediaEncoder = mediaEncoder;
             _subtitleEncoder = subtitleEncoder;
+            _config = config;
         }
 
         public string GetH264Encoder(EncodingJobInfo state, EncodingOptions encodingOptions)
@@ -144,15 +149,28 @@ namespace MediaBrowser.Controller.MediaEncoding
 
         private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
         {
-            if (state.VideoStream == null)
+            if (state.VideoStream == null
+                || !options.EnableTonemapping
+                || GetVideoColorBitDepth(state) != 10)
             {
                 return false;
             }
 
-            return options.EnableTonemapping
-                   && (string.Equals(state.VideoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase)
-                       || string.Equals(state.VideoStream.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase))
-                   && GetVideoColorBitDepth(state) == 10;
+            if (string.Equals(state.VideoStream.CodecTag, "dovi", StringComparison.OrdinalIgnoreCase)
+                || string.Equals(state.VideoStream.CodecTag, "dvh1", StringComparison.OrdinalIgnoreCase)
+                || string.Equals(state.VideoStream.CodecTag, "dvhe", StringComparison.OrdinalIgnoreCase))
+            {
+                // Only native SW decoder and HW accelerator can parse dovi rpu.
+                var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty;
+                var isSwDecoder = string.IsNullOrEmpty(vidDecoder);
+                var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase);
+                var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase);
+                var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase);
+                return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder;
+            }
+
+            return string.Equals(state.VideoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase)
+                   || string.Equals(state.VideoStream.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase);
         }
 
         private bool IsVaapiVppTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
@@ -516,8 +534,7 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             if (string.Equals(codec, "flac", StringComparison.OrdinalIgnoreCase))
             {
-                // flac is experimental in mp4 muxer
-                return "flac -strict -2";
+                return "flac";
             }
 
             return codec.ToLowerInvariant();
@@ -1024,7 +1041,8 @@ namespace MediaBrowser.Controller.MediaEncoding
             if (string.Equals(videoCodec, "h264_amf", StringComparison.OrdinalIgnoreCase)
                 || string.Equals(videoCodec, "hevc_amf", StringComparison.OrdinalIgnoreCase))
             {
-                return FormattableString.Invariant($" -qmin 18 -qmax 32 -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}");
+                // Override the too high default qmin 18 in transcoding preset
+                return FormattableString.Invariant($" -rc cbr -qmin 0 -qmax 32 -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}");
             }
 
             if (string.Equals(videoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase)
@@ -1222,10 +1240,9 @@ namespace MediaBrowser.Controller.MediaEncoding
                 // Example: we encoded half of desired length, then codec detected
                 // scene cut and inserted a keyframe; next forced keyframe would
                 // be created outside of segment, which breaks seeking.
-                // -sc_threshold 0 is used to prevent the hardware encoder from post processing to break the set keyframe.
                 gopArg = string.Format(
                     CultureInfo.InvariantCulture,
-                    " -g:v:0 {0} -keyint_min:v:0 {0} -sc_threshold:v:0 0",
+                    " -g:v:0 {0} -keyint_min:v:0 {0}",
                     Math.Ceiling(segmentLength * framerate.Value));
             }
 
@@ -1245,6 +1262,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                      || string.Equals(codec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase))
             {
                 args += keyFrameArg;
+
+                // prevent the libx264 from post processing to break the set keyframe.
+                if (string.Equals(codec, "libx264", StringComparison.OrdinalIgnoreCase))
+                {
+                    args += " -sc_threshold:v:0 0";
+                }
             }
             else
             {
@@ -4865,21 +4888,20 @@ namespace MediaBrowser.Controller.MediaEncoding
         public string GetInputModifier(EncodingJobInfo state, EncodingOptions encodingOptions, string segmentContainer)
         {
             var inputModifier = string.Empty;
-            var probeSizeArgument = string.Empty;
+            var analyzeDurationArgument = string.Empty;
 
-            string analyzeDurationArgument;
-            if (state.MediaSource.AnalyzeDurationMs.HasValue)
-            {
-                analyzeDurationArgument = "-analyzeduration " + (state.MediaSource.AnalyzeDurationMs.Value * 1000).ToString(CultureInfo.InvariantCulture);
-            }
-            else
+            // Apply -analyzeduration as per the environment variable,
+            // otherwise ffmpeg will break on certain files due to default value is 0.
+            // The default value of -probesize is more than enough, so leave it as is.
+            var ffmpegAnalyzeDuration = _config.GetFFmpegAnalyzeDuration() ?? string.Empty;
+
+            if (!string.IsNullOrEmpty(ffmpegAnalyzeDuration))
             {
-                analyzeDurationArgument = string.Empty;
+                analyzeDurationArgument = "-analyzeduration " + ffmpegAnalyzeDuration;
             }
-
-            if (!string.IsNullOrEmpty(probeSizeArgument))
+            else if (state.MediaSource.AnalyzeDurationMs.HasValue)
             {
-                inputModifier += " " + probeSizeArgument;
+                analyzeDurationArgument = "-analyzeduration " + (state.MediaSource.AnalyzeDurationMs.Value * 1000).ToString(CultureInfo.InvariantCulture);
             }
 
             if (!string.IsNullOrEmpty(analyzeDurationArgument))
diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
index 1bac4b187..77b97c9b4 100644
--- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
@@ -16,6 +16,7 @@ using MediaBrowser.Common;
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.Extensions;
 using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Extensions;
 using MediaBrowser.Controller.MediaEncoding;
 using MediaBrowser.MediaEncoding.Probing;
 using MediaBrowser.Model.Dlna;
@@ -49,6 +50,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
         private readonly IServerConfigurationManager _configurationManager;
         private readonly IFileSystem _fileSystem;
         private readonly ILocalizationManager _localization;
+        private readonly IConfiguration _config;
         private readonly string _startupOptionFFmpegPath;
 
         private readonly SemaphoreSlim _thumbnailResourcePool = new SemaphoreSlim(2, 2);
@@ -85,6 +87,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             _configurationManager = configurationManager;
             _fileSystem = fileSystem;
             _localization = localization;
+            _config = config;
             _startupOptionFFmpegPath = config.GetValue<string>(Controller.Extensions.ConfigurationExtensions.FfmpegPathKey) ?? string.Empty;
             _jsonSerializerOptions = JsonDefaults.Options;
         }
@@ -371,8 +374,13 @@ namespace MediaBrowser.MediaEncoding.Encoder
             var inputFile = request.MediaSource.Path;
 
             string analyzeDuration = string.Empty;
+            string ffmpegAnalyzeDuration = _config.GetFFmpegAnalyzeDuration() ?? string.Empty;
 
-            if (request.MediaSource.AnalyzeDurationMs > 0)
+            if (!string.IsNullOrEmpty(ffmpegAnalyzeDuration))
+            {
+                analyzeDuration = "-analyzeduration " + ffmpegAnalyzeDuration;
+            }
+            else if (request.MediaSource.AnalyzeDurationMs > 0)
             {
                 analyzeDuration = "-analyzeduration " +
                                   (request.MediaSource.AnalyzeDurationMs * 1000).ToString();
@@ -629,10 +637,15 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 filters.Add("thumbnail=n=" + (useLargerBatchSize ? "50" : "24"));
             }
 
-            // Use SW tonemap on HDR video stream only when the zscale filter is available.
-            var enableHdrExtraction = string.Equals(videoStream?.VideoRange, "HDR", StringComparison.OrdinalIgnoreCase) && SupportsFilter("zscale");
-            if (enableHdrExtraction)
+            // Use SW tonemap on HDR10/HLG video stream only when the zscale filter is available.
+            var enableHdrExtraction = false;
+
+            if ((string.Equals(videoStream?.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase)
+                || string.Equals(videoStream?.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase))
+                && SupportsFilter("zscale"))
             {
+                enableHdrExtraction = true;
+
                 filters.Add("zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0:peak=100,zscale=t=bt709:m=bt709,format=yuv420p");
             }
 
diff --git a/MediaBrowser.Model/Entities/MediaStream.cs b/MediaBrowser.Model/Entities/MediaStream.cs
index 4742d21e9..96b48ca52 100644
--- a/MediaBrowser.Model/Entities/MediaStream.cs
+++ b/MediaBrowser.Model/Entities/MediaStream.cs
@@ -121,8 +121,7 @@ namespace MediaBrowser.Model.Entities
 
                 var codecTag = CodecTag;
 
-                if (string.Equals(codecTag, "dva1", StringComparison.OrdinalIgnoreCase)
-                    || string.Equals(codecTag, "dvav", StringComparison.OrdinalIgnoreCase)
+                if (string.Equals(codecTag, "dovi", StringComparison.OrdinalIgnoreCase)
                     || string.Equals(codecTag, "dvh1", StringComparison.OrdinalIgnoreCase)
                     || string.Equals(codecTag, "dvhe", StringComparison.OrdinalIgnoreCase)
                     || string.Equals(codecTag, "dav1", StringComparison.OrdinalIgnoreCase))