add Tonemapping relaying on nvdec and ocl

5 files changed, 166 insertions, 14 deletions

diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index 0242338b7..4a33f22e6 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -454,6 +454,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1;
             var isQsvDecoder = videoDecoder.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1;
             var isQsvEncoder = outputVideoCodec.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1;
+            var isNvencHevcDecoder = videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1;
             var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
             var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
 
@@ -511,6 +512,25 @@ namespace MediaBrowser.Controller.MediaEncoding
                         }
                     }
                 }
+
+                if (state.IsVideoRequest
+                    && string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase))
+                {
+                    var codec = state.VideoStream.Codec.ToLowerInvariant();
+                    var isColorDepth10 = IsColorDepth10(state);
+
+                    if (isNvencHevcDecoder && isColorDepth10
+                        && _mediaEncoder.SupportsHwaccel("opencl")
+                        && encodingOptions.EnableTonemapping
+                        && !string.IsNullOrEmpty(state.VideoStream.VideoRange)
+                        && state.VideoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
+                    {
+                        arg.Append("-init_hw_device opencl=ocl:")
+                            .Append(encodingOptions.OpenclDevice)
+                            .Append(' ')
+                            .Append("-filter_hw_device ocl ");
+                    }
+                }
             }
 
             arg.Append("-i ")
@@ -994,11 +1014,33 @@ namespace MediaBrowser.Controller.MediaEncoding
             if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase)
                 && !string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase)
                 && !string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase)
+                && !string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase)
                 && !string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase))
             {
                 param = "-pix_fmt yuv420p " + param;
             }
 
+            if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase))
+            {
+                var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions) ?? string.Empty;
+                var videoStream = state.VideoStream;
+                var isColorDepth10 = IsColorDepth10(state);
+
+                if (videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1
+                    && isColorDepth10
+                    && _mediaEncoder.SupportsHwaccel("opencl")
+                    && encodingOptions.EnableTonemapping
+                    && !string.IsNullOrEmpty(videoStream.VideoRange)
+                    && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
+                {
+                    param = "-pix_fmt nv12 " + param;
+                }
+                else
+                {
+                    param = "-pix_fmt yuv420p " + param;
+                }
+            }
+
             if (string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase))
             {
                 param = "-pix_fmt nv21 " + param;
@@ -1599,46 +1641,54 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"');
 
-                var index = outputSizeParam.IndexOf("hwdownload", StringComparison.OrdinalIgnoreCase);
+                var index = outputSizeParam.IndexOf("hwupload,tonemap_opencl", StringComparison.OrdinalIgnoreCase);
                 if (index != -1)
                 {
                     outputSizeParam = outputSizeParam.Substring(index);
                 }
                 else
                 {
-                    index = outputSizeParam.IndexOf("hwupload=extra_hw_frames", StringComparison.OrdinalIgnoreCase);
+                    index = outputSizeParam.IndexOf("hwdownload", StringComparison.OrdinalIgnoreCase);
                     if (index != -1)
                     {
                         outputSizeParam = outputSizeParam.Substring(index);
                     }
                     else
                     {
-                        index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase);
+                        index = outputSizeParam.IndexOf("hwupload=extra_hw_frames", StringComparison.OrdinalIgnoreCase);
                         if (index != -1)
                         {
                             outputSizeParam = outputSizeParam.Substring(index);
                         }
                         else
                         {
-                            index = outputSizeParam.IndexOf("yadif", StringComparison.OrdinalIgnoreCase);
+                            index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase);
                             if (index != -1)
                             {
                                 outputSizeParam = outputSizeParam.Substring(index);
                             }
                             else
                             {
-                                index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase);
+                                index = outputSizeParam.IndexOf("yadif", StringComparison.OrdinalIgnoreCase);
                                 if (index != -1)
                                 {
                                     outputSizeParam = outputSizeParam.Substring(index);
                                 }
                                 else
                                 {
-                                    index = outputSizeParam.IndexOf("vpp", StringComparison.OrdinalIgnoreCase);
+                                    index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase);
                                     if (index != -1)
                                     {
                                         outputSizeParam = outputSizeParam.Substring(index);
                                     }
+                                    else
+                                    {
+                                        index = outputSizeParam.IndexOf("vpp", StringComparison.OrdinalIgnoreCase);
+                                        if (index != -1)
+                                        {
+                                            outputSizeParam = outputSizeParam.Substring(index);
+                                        }
+                                    }
                                 }
                             }
                         }
@@ -2058,12 +2108,58 @@ namespace MediaBrowser.Controller.MediaEncoding
             var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1;
             var isQsvH264Encoder = outputVideoCodec.IndexOf("h264_qsv", StringComparison.OrdinalIgnoreCase) != -1;
             var isNvdecH264Decoder = videoDecoder.IndexOf("h264_cuvid", StringComparison.OrdinalIgnoreCase) != -1;
+            var isNvdecHevcDecoder = videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1;
             var isLibX264Encoder = outputVideoCodec.IndexOf("libx264", StringComparison.OrdinalIgnoreCase) != -1;
             var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
+            var isColorDepth10 = IsColorDepth10(state);
 
             var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
             var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
 
+            // Currently only with the use of NVENC decoder can we get a decent performance.
+            // Currently only the HEVC/H265 format is supported.
+            // NVIDIA Pascal and Turing or higher are recommended.
+            if (isNvdecHevcDecoder && isColorDepth10
+                && _mediaEncoder.SupportsHwaccel("opencl")
+                && options.EnableTonemapping
+                && !string.IsNullOrEmpty(videoStream.VideoRange)
+                && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
+            {
+                var parameters = "tonemap_opencl=format=nv12:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:desat={1}:threshold={2}:peak={3}";
+
+                if (options.TonemappingParam != 0)
+                {
+                    parameters += ":param={4}";
+                }
+
+                if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase))
+                {
+                    parameters += ":range={5}";
+                }
+
+                // Upload the HDR10 or HLG data to the OpenCL device,
+                // use tonemap_opencl filter for tone mapping,
+                // and then download the SDR data to memory.
+                filters.Add("hwupload");
+                filters.Add(
+                    string.Format(
+                        CultureInfo.InvariantCulture,
+                        parameters,
+                        options.TonemappingAlgorithm,
+                        options.TonemappingDesat,
+                        options.TonemappingThreshold,
+                        options.TonemappingPeak,
+                        options.TonemappingParam,
+                        options.TonemappingRange));
+                filters.Add("hwdownload");
+
+                if (hasGraphicalSubs || state.DeInterlace("h265", true) || state.DeInterlace("hevc", true)
+                    || string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase))
+                {
+                    filters.Add("format=nv12");
+                }
+            }
+
             // When the input may or may not be hardware VAAPI decodable
             if (isVaapiH264Encoder)
             {
@@ -2081,7 +2177,6 @@ namespace MediaBrowser.Controller.MediaEncoding
             else if (IsVaapiSupported(state) && isVaapiDecoder && isLibX264Encoder)
             {
                 var codec = videoStream.Codec.ToLowerInvariant();
-                var isColorDepth10 = IsColorDepth10(state);
 
                 // Assert 10-bit hardware VAAPI decodable
                 if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)
diff --git a/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs b/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs
index a2ea0766a..d9658cba2 100644
--- a/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs
+++ b/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs
@@ -280,6 +280,20 @@ namespace MediaBrowser.MediaEncoding.Probing
         public IReadOnlyDictionary<string, int> Disposition { get; set; }
 
         /// <summary>
+        /// Gets or sets the color range.
+        /// </summary>
+        /// <value>The color range.</value>
+        [JsonPropertyName("color_range")]
+        public string ColorRange { get; set; }
+
+        /// <summary>
+        /// Gets or sets the color space.
+        /// </summary>
+        /// <value>The color space.</value>
+        [JsonPropertyName("color_space")]
+        public string ColorSpace { get; set; }
+
+        /// <summary>
         /// Gets or sets the color transfer.
         /// </summary>
         /// <value>The color transfer.</value>
diff --git a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
index c85d963ed..3815e00d4 100644
--- a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
+++ b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
@@ -703,6 +703,16 @@ namespace MediaBrowser.MediaEncoding.Probing
                     stream.RefFrames = streamInfo.Refs;
                 }
 
+                if (!string.IsNullOrEmpty(streamInfo.ColorRange))
+                {
+                    stream.ColorRange = streamInfo.ColorRange;
+                }
+
+                if (!string.IsNullOrEmpty(streamInfo.ColorSpace))
+                {
+                    stream.ColorSpace = streamInfo.ColorSpace;
+                }
+
                 if (!string.IsNullOrEmpty(streamInfo.ColorTransfer))
                 {
                     stream.ColorTransfer = streamInfo.ColorTransfer;
diff --git a/MediaBrowser.Model/Configuration/EncodingOptions.cs b/MediaBrowser.Model/Configuration/EncodingOptions.cs
index 9a30f7e9f..843ff3ff9 100644
--- a/MediaBrowser.Model/Configuration/EncodingOptions.cs
+++ b/MediaBrowser.Model/Configuration/EncodingOptions.cs
@@ -29,6 +29,22 @@ namespace MediaBrowser.Model.Configuration
 
         public string VaapiDevice { get; set; }
 
+        public string OpenclDevice { get; set; }
+
+        public bool EnableTonemapping { get; set; }
+
+        public string TonemappingAlgorithm { get; set; }
+
+        public string TonemappingRange { get; set; }
+
+        public double TonemappingDesat { get; set; }
+
+        public double TonemappingThreshold { get; set; }
+
+        public double TonemappingPeak { get; set; }
+
+        public double TonemappingParam { get; set; }
+
         public int H264Crf { get; set; }
 
         public int H265Crf { get; set; }
@@ -53,8 +69,19 @@ namespace MediaBrowser.Model.Configuration
             EnableThrottling = false;
             ThrottleDelaySeconds = 180;
             EncodingThreadCount = -1;
-            // This is a DRM device that is almost guaranteed to be there on every intel platform, plus it's the default one in ffmpeg if you don't specify anything
+            // This is a DRM device that is almost guaranteed to be there on every intel platform,
+            // plus it's the default one in ffmpeg if you don't specify anything
             VaapiDevice = "/dev/dri/renderD128";
+            // This is the OpenCL device that is used for tonemapping.
+            // The left side of the dot is the platform number, and the right side is the device number on the platform.
+            OpenclDevice = "0.0";
+            EnableTonemapping = false;
+            TonemappingAlgorithm = "reinhard";
+            TonemappingRange = "auto";
+            TonemappingDesat = 0;
+            TonemappingThreshold = 0.8;
+            TonemappingPeak = 0;
+            TonemappingParam = 0;
             H264Crf = 23;
             H265Crf = 28;
             DeinterlaceMethod = "yadif";
diff --git a/MediaBrowser.Model/Entities/MediaStream.cs b/MediaBrowser.Model/Entities/MediaStream.cs
index 1b37cfc93..19eb79acc 100644
--- a/MediaBrowser.Model/Entities/MediaStream.cs
+++ b/MediaBrowser.Model/Entities/MediaStream.cs
@@ -36,6 +36,18 @@ namespace MediaBrowser.Model.Entities
         public string Language { get; set; }
 
         /// <summary>
+        /// Gets or sets the color range.
+        /// </summary>
+        /// <value>The color range.</value>
+        public string ColorRange { get; set; }
+
+        /// <summary>
+        /// Gets or sets the color space.
+        /// </summary>
+        /// <value>The color space.</value>
+        public string ColorSpace { get; set; }
+
+        /// <summary>
         /// Gets or sets the color transfer.
         /// </summary>
         /// <value>The color transfer.</value>
@@ -48,12 +60,6 @@ namespace MediaBrowser.Model.Entities
         public string ColorPrimaries { get; set; }
 
         /// <summary>
-        /// Gets or sets the color space.
-        /// </summary>
-        /// <value>The color space.</value>
-        public string ColorSpace { get; set; }
-
-        /// <summary>
         /// Gets or sets the comment.
         /// </summary>
         /// <value>The comment.</value>