Improve dynamic HDR metadata handling (#13277)

* Add support for bitstream filter to remove dynamic hdr metadata

* Add support for ffprobe's only_first_vframe for HDR10+ detection

* Add BitStreamFilterOptionType for metadata removal check

* Map HDR10+ metadata to VideoRangeType.cs

Current implementation uses a hack that abuses the EL flag to avoid database schema changes. Should add proper field once EFCore migration is merged.

* Add more Dolby Vision Range types

Out of spec ones are problematic and should be marked as a dedicated invalid type and handled by the server to not crash the player.

Profile 7 videos should not be treated as normal HDR10 videos at all and should remove the metadata before serving.

* Remove dynamic hdr metadata when necessary

* Allow direct playback of HDR10+ videos on HDR10 clients

* Only use dovi codec tag when dovi metadata is not removed

* Handle DV Profile 7 Videos better

* Fix HDR10+ with new bitmask

* Indicate the presence of HDR10+ in HLS SUPPLEMENTAL-CODECS

* Fix Dovi 8.4 not labeled as HLG in HLS

* Fallback to dovi_rpu bsf for av1 when possible

* Fix dovi_rpu cli for av1

* Use correct EFCore db column for HDR10+

* Undo outdated migration

* Add proper hdr10+ migration

* Remove outdated migration

* Rebase to new db code

* Add migrations for Hdr10PlusPresentFlag

* Directly use bsf enum

* Add xmldocs for SupportsBitStreamFilterWithOption

* Make `VideoRangeType.Unknown` explicitly default on api models.

* Unset default for non-api model class

* Use tuples for bsf dictionary for now

6 files changed, 286 insertions, 2 deletions

diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
index 54d0eb4b5..d28cd70ef 100644
--- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
@@ -7,6 +7,7 @@ using System.Globalization;
 using System.Linq;
 using System.Runtime.Versioning;
 using System.Text.RegularExpressions;
+using MediaBrowser.Controller.MediaEncoding;
 using Microsoft.Extensions.Logging;
 
 namespace MediaBrowser.MediaEncoding.Encoder
@@ -160,6 +161,15 @@ namespace MediaBrowser.MediaEncoding.Encoder
             { 6, new string[] { "transpose_opencl", "rotate by half-turn" } }
         };
 
+        private static readonly Dictionary<BitStreamFilterOptionType, (string, string)> _bsfOptionsDict = new Dictionary<BitStreamFilterOptionType, (string, string)>
+        {
+            { BitStreamFilterOptionType.HevcMetadataRemoveDovi, ("hevc_metadata", "remove_dovi") },
+            { BitStreamFilterOptionType.HevcMetadataRemoveHdr10Plus, ("hevc_metadata", "remove_hdr10plus") },
+            { BitStreamFilterOptionType.Av1MetadataRemoveDovi, ("av1_metadata", "remove_dovi") },
+            { BitStreamFilterOptionType.Av1MetadataRemoveHdr10Plus, ("av1_metadata", "remove_hdr10plus") },
+            { BitStreamFilterOptionType.DoviRpuStrip, ("dovi_rpu", "strip") }
+        };
+
         // These are the library versions that corresponds to our minimum ffmpeg version 4.4 according to the version table below
         // Refers to the versions in https://ffmpeg.org/download.html
         private static readonly Dictionary<string, Version> _ffmpegMinimumLibraryVersions = new Dictionary<string, Version>
@@ -286,6 +296,9 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
         public IDictionary<int, bool> GetFiltersWithOption() => GetFFmpegFiltersWithOption();
 
+        public IDictionary<BitStreamFilterOptionType, bool> GetBitStreamFiltersWithOption() => _bsfOptionsDict
+            .ToDictionary(item => item.Key, item => CheckBitStreamFilterWithOption(item.Value.Item1, item.Value.Item2));
+
         public Version? GetFFmpegVersion()
         {
             string output;
@@ -495,6 +508,34 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return false;
         }
 
+        public bool CheckBitStreamFilterWithOption(string filter, string option)
+        {
+            if (string.IsNullOrEmpty(filter) || string.IsNullOrEmpty(option))
+            {
+                return false;
+            }
+
+            string output;
+            try
+            {
+                output = GetProcessOutput(_encoderPath, "-h bsf=" + filter, false, null);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error detecting the given bit stream filter");
+                return false;
+            }
+
+            if (output.Contains("Bit stream filter " + filter, StringComparison.Ordinal))
+            {
+                return output.Contains(option, StringComparison.Ordinal);
+            }
+
+            _logger.LogWarning("Bit stream filter: {Name} with option {Option} is not available", filter, option);
+
+            return false;
+        }
+
         public bool CheckSupportedRuntimeKey(string keyDesc, Version? ffmpegVersion)
         {
             if (string.IsNullOrEmpty(keyDesc))
@@ -523,6 +564,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return !string.IsNullOrEmpty(flag) && GetProcessExitCode(_encoderPath, $"-loglevel quiet -hwaccel_flags +{flag} -hide_banner -f lavfi -i nullsrc=s=1x1:d=100 -f null -");
         }
 
+        public bool CheckSupportedProberOption(string option, string proberPath)
+        {
+            return !string.IsNullOrEmpty(option) && GetProcessExitCode(proberPath, $"-loglevel quiet -f lavfi -i nullsrc=s=1x1:d=1 -{option}");
+        }
+
         private IEnumerable<string> GetCodecs(Codec codec)
         {
             string codecstr = codec == Codec.Encoder ? "encoders" : "decoders";
diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
index e96040506..9a759ba41 100644
--- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
@@ -73,9 +73,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
         private List<string> _hwaccels = new List<string>();
         private List<string> _filters = new List<string>();
         private IDictionary<int, bool> _filtersWithOption = new Dictionary<int, bool>();
+        private IDictionary<BitStreamFilterOptionType, bool> _bitStreamFiltersWithOption = new Dictionary<BitStreamFilterOptionType, bool>();
 
         private bool _isPkeyPauseSupported = false;
         private bool _isLowPriorityHwDecodeSupported = false;
+        private bool _proberSupportsFirstVideoFrame = false;
 
         private bool _isVaapiDeviceAmd = false;
         private bool _isVaapiDeviceInteliHD = false;
@@ -222,6 +224,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 SetAvailableEncoders(validator.GetEncoders());
                 SetAvailableFilters(validator.GetFilters());
                 SetAvailableFiltersWithOption(validator.GetFiltersWithOption());
+                SetAvailableBitStreamFiltersWithOption(validator.GetBitStreamFiltersWithOption());
                 SetAvailableHwaccels(validator.GetHwaccels());
                 SetMediaEncoderVersion(validator);
 
@@ -229,6 +232,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
                 _isPkeyPauseSupported = validator.CheckSupportedRuntimeKey("p      pause transcoding", _ffmpegVersion);
                 _isLowPriorityHwDecodeSupported = validator.CheckSupportedHwaccelFlag("low_priority");
+                _proberSupportsFirstVideoFrame = validator.CheckSupportedProberOption("only_first_vframe", _ffprobePath);
 
                 // Check the Vaapi device vendor
                 if (OperatingSystem.IsLinux()
@@ -342,6 +346,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
             _filtersWithOption = dict;
         }
 
+        public void SetAvailableBitStreamFiltersWithOption(IDictionary<BitStreamFilterOptionType, bool> dict)
+        {
+            _bitStreamFiltersWithOption = dict;
+        }
+
         public void SetMediaEncoderVersion(EncoderValidator validator)
         {
             _ffmpegVersion = validator.GetFFmpegVersion();
@@ -382,6 +391,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return false;
         }
 
+        public bool SupportsBitStreamFilterWithOption(BitStreamFilterOptionType option)
+        {
+            return _bitStreamFiltersWithOption.TryGetValue(option, out var val) && val;
+        }
+
         public bool CanEncodeToAudioCodec(string codec)
         {
             if (string.Equals(codec, "opus", StringComparison.OrdinalIgnoreCase))
@@ -501,6 +515,12 @@ namespace MediaBrowser.MediaEncoding.Encoder
             var args = extractChapters
                 ? "{0} -i {1} -threads {2} -v warning -print_format json -show_streams -show_chapters -show_format"
                 : "{0} -i {1} -threads {2} -v warning -print_format json -show_streams -show_format";
+
+            if (_proberSupportsFirstVideoFrame)
+            {
+                args += " -show_frames -only_first_vframe";
+            }
+
             args = string.Format(CultureInfo.InvariantCulture, args, probeSizeArgument, inputPath, _threads).Trim();
 
             var process = new Process
diff --git a/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs b/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs
index d4d153b08..53eea64db 100644
--- a/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs
+++ b/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs
@@ -30,5 +30,12 @@ namespace MediaBrowser.MediaEncoding.Probing
         /// <value>The chapters.</value>
         [JsonPropertyName("chapters")]
         public IReadOnlyList<MediaChapter> Chapters { get; set; }
+
+        /// <summary>
+        /// Gets or sets the frames.
+        /// </summary>
+        /// <value>The streams.</value>
+        [JsonPropertyName("frames")]
+        public IReadOnlyList<MediaFrameInfo> Frames { get; set; }
     }
 }
diff --git a/MediaBrowser.MediaEncoding/Probing/MediaFrameInfo.cs b/MediaBrowser.MediaEncoding/Probing/MediaFrameInfo.cs
new file mode 100644
index 000000000..bed4368ed
--- /dev/null
+++ b/MediaBrowser.MediaEncoding/Probing/MediaFrameInfo.cs
@@ -0,0 +1,184 @@
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace MediaBrowser.MediaEncoding.Probing;
+
+/// <summary>
+/// Class MediaFrameInfo.
+/// </summary>
+public class MediaFrameInfo
+{
+    /// <summary>
+    /// Gets or sets the media type.
+    /// </summary>
+    [JsonPropertyName("media_type")]
+    public string? MediaType { get; set; }
+
+    /// <summary>
+    /// Gets or sets the StreamIndex.
+    /// </summary>
+    [JsonPropertyName("stream_index")]
+    public int? StreamIndex { get; set; }
+
+    /// <summary>
+    /// Gets or sets the KeyFrame.
+    /// </summary>
+    [JsonPropertyName("key_frame")]
+    public int? KeyFrame { get; set; }
+
+    /// <summary>
+    /// Gets or sets the Pts.
+    /// </summary>
+    [JsonPropertyName("pts")]
+    public long? Pts { get; set; }
+
+    /// <summary>
+    /// Gets or sets the PtsTime.
+    /// </summary>
+    [JsonPropertyName("pts_time")]
+    public string? PtsTime { get; set; }
+
+    /// <summary>
+    /// Gets or sets the BestEffortTimestamp.
+    /// </summary>
+    [JsonPropertyName("best_effort_timestamp")]
+    public long BestEffortTimestamp { get; set; }
+
+    /// <summary>
+    /// Gets or sets the BestEffortTimestampTime.
+    /// </summary>
+    [JsonPropertyName("best_effort_timestamp_time")]
+    public string? BestEffortTimestampTime { get; set; }
+
+    /// <summary>
+    /// Gets or sets the Duration.
+    /// </summary>
+    [JsonPropertyName("duration")]
+    public int Duration { get; set; }
+
+    /// <summary>
+    /// Gets or sets the DurationTime.
+    /// </summary>
+    [JsonPropertyName("duration_time")]
+    public string? DurationTime { get; set; }
+
+    /// <summary>
+    /// Gets or sets the PktPos.
+    /// </summary>
+    [JsonPropertyName("pkt_pos")]
+    public string? PktPos { get; set; }
+
+    /// <summary>
+    /// Gets or sets the PktSize.
+    /// </summary>
+    [JsonPropertyName("pkt_size")]
+    public string? PktSize { get; set; }
+
+    /// <summary>
+    /// Gets or sets the Width.
+    /// </summary>
+    [JsonPropertyName("width")]
+    public int? Width { get; set; }
+
+    /// <summary>
+    /// Gets or sets the Height.
+    /// </summary>
+    [JsonPropertyName("height")]
+    public int? Height { get; set; }
+
+    /// <summary>
+    /// Gets or sets the CropTop.
+    /// </summary>
+    [JsonPropertyName("crop_top")]
+    public int? CropTop { get; set; }
+
+    /// <summary>
+    /// Gets or sets the CropBottom.
+    /// </summary>
+    [JsonPropertyName("crop_bottom")]
+    public int? CropBottom { get; set; }
+
+    /// <summary>
+    /// Gets or sets the CropLeft.
+    /// </summary>
+    [JsonPropertyName("crop_left")]
+    public int? CropLeft { get; set; }
+
+    /// <summary>
+    /// Gets or sets the CropRight.
+    /// </summary>
+    [JsonPropertyName("crop_right")]
+    public int? CropRight { get; set; }
+
+    /// <summary>
+    /// Gets or sets the PixFmt.
+    /// </summary>
+    [JsonPropertyName("pix_fmt")]
+    public string? PixFmt { get; set; }
+
+    /// <summary>
+    /// Gets or sets the SampleAspectRatio.
+    /// </summary>
+    [JsonPropertyName("sample_aspect_ratio")]
+    public string? SampleAspectRatio { get; set; }
+
+    /// <summary>
+    /// Gets or sets the PictType.
+    /// </summary>
+    [JsonPropertyName("pict_type")]
+    public string? PictType { get; set; }
+
+    /// <summary>
+    /// Gets or sets the InterlacedFrame.
+    /// </summary>
+    [JsonPropertyName("interlaced_frame")]
+    public int? InterlacedFrame { get; set; }
+
+    /// <summary>
+    /// Gets or sets the TopFieldFirst.
+    /// </summary>
+    [JsonPropertyName("top_field_first")]
+    public int? TopFieldFirst { get; set; }
+
+    /// <summary>
+    /// Gets or sets the RepeatPict.
+    /// </summary>
+    [JsonPropertyName("repeat_pict")]
+    public int? RepeatPict { get; set; }
+
+    /// <summary>
+    /// Gets or sets the ColorRange.
+    /// </summary>
+    [JsonPropertyName("color_range")]
+    public string? ColorRange { get; set; }
+
+    /// <summary>
+    /// Gets or sets the ColorSpace.
+    /// </summary>
+    [JsonPropertyName("color_space")]
+    public string? ColorSpace { get; set; }
+
+    /// <summary>
+    /// Gets or sets the ColorPrimaries.
+    /// </summary>
+    [JsonPropertyName("color_primaries")]
+    public string? ColorPrimaries { get; set; }
+
+    /// <summary>
+    /// Gets or sets the ColorTransfer.
+    /// </summary>
+    [JsonPropertyName("color_transfer")]
+    public string? ColorTransfer { get; set; }
+
+    /// <summary>
+    /// Gets or sets the ChromaLocation.
+    /// </summary>
+    [JsonPropertyName("chroma_location")]
+    public string? ChromaLocation { get; set; }
+
+    /// <summary>
+    /// Gets or sets the SideDataList.
+    /// </summary>
+    [JsonPropertyName("side_data_list")]
+    public IReadOnlyList<MediaFrameSideDataInfo>? SideDataList { get; set; }
+}
diff --git a/MediaBrowser.MediaEncoding/Probing/MediaFrameSideDataInfo.cs b/MediaBrowser.MediaEncoding/Probing/MediaFrameSideDataInfo.cs
new file mode 100644
index 000000000..3f7dd9a69
--- /dev/null
+++ b/MediaBrowser.MediaEncoding/Probing/MediaFrameSideDataInfo.cs
@@ -0,0 +1,16 @@
+using System.Text.Json.Serialization;
+
+namespace MediaBrowser.MediaEncoding.Probing;
+
+/// <summary>
+/// Class MediaFrameSideDataInfo.
+/// Currently only records the SideDataType for HDR10+ detection.
+/// </summary>
+public class MediaFrameSideDataInfo
+{
+    /// <summary>
+    /// Gets or sets the SideDataType.
+    /// </summary>
+    [JsonPropertyName("side_data_type")]
+    public string? SideDataType { get; set; }
+}
diff --git a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
index 6b0fd9a14..a98dbe597 100644
--- a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
+++ b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs
@@ -105,8 +105,9 @@ namespace MediaBrowser.MediaEncoding.Probing
             SetSize(data, info);
 
             var internalStreams = data.Streams ?? Array.Empty<MediaStreamInfo>();
+            var internalFrames = data.Frames ?? Array.Empty<MediaFrameInfo>();
 
-            info.MediaStreams = internalStreams.Select(s => GetMediaStream(isAudio, s, data.Format))
+            info.MediaStreams = internalStreams.Select(s => GetMediaStream(isAudio, s, data.Format, internalFrames))
                 .Where(i => i is not null)
                 // Drop subtitle streams if we don't know the codec because it will just cause failures if we don't know how to handle them
                 .Where(i => i.Type != MediaStreamType.Subtitle || !string.IsNullOrWhiteSpace(i.Codec))
@@ -685,8 +686,9 @@ namespace MediaBrowser.MediaEncoding.Probing
         /// <param name="isAudio">if set to <c>true</c> [is info].</param>
         /// <param name="streamInfo">The stream info.</param>
         /// <param name="formatInfo">The format info.</param>
+        /// <param name="frameInfoList">The frame info.</param>
         /// <returns>MediaStream.</returns>
-        private MediaStream GetMediaStream(bool isAudio, MediaStreamInfo streamInfo, MediaFormatInfo formatInfo)
+        private MediaStream GetMediaStream(bool isAudio, MediaStreamInfo streamInfo, MediaFormatInfo formatInfo, IReadOnlyList<MediaFrameInfo> frameInfoList)
         {
             // These are mp4 chapters
             if (string.Equals(streamInfo.CodecName, "mov_text", StringComparison.OrdinalIgnoreCase))
@@ -904,6 +906,15 @@ namespace MediaBrowser.MediaEncoding.Probing
                         }
                     }
                 }
+
+                var frameInfo = frameInfoList?.FirstOrDefault(i => i.StreamIndex == stream.Index);
+                if (frameInfo?.SideDataList != null)
+                {
+                    if (frameInfo.SideDataList.Any(data => string.Equals(data.SideDataType, "HDR Dynamic Metadata SMPTE2094-40 (HDR10+)", StringComparison.OrdinalIgnoreCase)))
+                    {
+                        stream.Hdr10PlusPresentFlag = true;
+                    }
+                }
             }
             else if (streamInfo.CodecType == CodecType.Data)
             {