Improve dynamic HDR metadata handling (#13277)

* Add support for bitstream filter to remove dynamic hdr metadata

* Add support for ffprobe's only_first_vframe for HDR10+ detection

* Add BitStreamFilterOptionType for metadata removal check

* Map HDR10+ metadata to VideoRangeType.cs

Current implementation uses a hack that abuses the EL flag to avoid database schema changes. Should add proper field once EFCore migration is merged.

* Add more Dolby Vision Range types

Out of spec ones are problematic and should be marked as a dedicated invalid type and handled by the server to not crash the player.

Profile 7 videos should not be treated as normal HDR10 videos at all and should remove the metadata before serving.

* Remove dynamic hdr metadata when necessary

* Allow direct playback of HDR10+ videos on HDR10 clients

* Only use dovi codec tag when dovi metadata is not removed

* Handle DV Profile 7 Videos better

* Fix HDR10+ with new bitmask

* Indicate the presence of HDR10+ in HLS SUPPLEMENTAL-CODECS

* Fix Dovi 8.4 not labeled as HLG in HLS

* Fallback to dovi_rpu bsf for av1 when possible

* Fix dovi_rpu cli for av1

* Use correct EFCore db column for HDR10+

* Undo outdated migration

* Add proper hdr10+ migration

* Remove outdated migration

* Rebase to new db code

* Add migrations for Hdr10PlusPresentFlag

* Directly use bsf enum

* Add xmldocs for SupportsBitStreamFilterWithOption

* Make `VideoRangeType.Unknown` explicitly default on api models.

* Unset default for non-api model class

* Use tuples for bsf dictionary for now

2 files changed, 64 insertions, 30 deletions

diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs
index 51291ec62..31b96972e 100644
--- a/Jellyfin.Api/Controllers/DynamicHlsController.cs
+++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs
@@ -1675,7 +1675,7 @@ public class DynamicHlsController : BaseJellyfinApiController
         }
 
         var audioCodec = _encodingHelper.GetAudioEncoder(state);
-        var bitStreamArgs = EncodingHelper.GetAudioBitStreamArguments(state, state.Request.SegmentContainer, state.MediaSource.Container);
+        var bitStreamArgs = _encodingHelper.GetAudioBitStreamArguments(state, state.Request.SegmentContainer, state.MediaSource.Container);
 
         // opus, dts, truehd and flac (in FFmpeg 5 and older) are experimental in mp4 muxer
         var strictArgs = string.Empty;
@@ -1822,10 +1822,11 @@ public class DynamicHlsController : BaseJellyfinApiController
             // Clients reporting Dolby Vision capabilities with fallbacks may only support the fallback layer.
             // Only enable Dolby Vision remuxing if the client explicitly declares support for profiles without fallbacks.
             var clientSupportsDoVi = requestedRange.Contains(VideoRangeType.DOVI.ToString(), StringComparison.OrdinalIgnoreCase);
-            var videoIsDoVi = state.VideoStream.VideoRangeType is VideoRangeType.DOVI or VideoRangeType.DOVIWithHDR10 or VideoRangeType.DOVIWithHLG or VideoRangeType.DOVIWithSDR;
+            var videoIsDoVi = EncodingHelper.IsDovi(state.VideoStream);
 
             if (EncodingHelper.IsCopyCodec(codec)
-                && (videoIsDoVi && clientSupportsDoVi))
+                && (videoIsDoVi && clientSupportsDoVi)
+                && !_encodingHelper.IsDoviRemoved(state))
             {
                 if (isActualOutputVideoCodecHevc)
                 {
@@ -1855,7 +1856,7 @@ public class DynamicHlsController : BaseJellyfinApiController
             // If h264_mp4toannexb is ever added, do not use it for live tv.
             if (state.VideoStream is not null && !string.Equals(state.VideoStream.NalLengthSize, "0", StringComparison.OrdinalIgnoreCase))
             {
-                string bitStreamArgs = EncodingHelper.GetBitStreamArgs(state.VideoStream);
+                string bitStreamArgs = _encodingHelper.GetBitStreamArgs(state, MediaStreamType.Video);
                 if (!string.IsNullOrEmpty(bitStreamArgs))
                 {
                     args += " " + bitStreamArgs;
diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs
index ebd0288ca..a38ad379c 100644
--- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs
+++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs
@@ -345,13 +345,15 @@ public class DynamicHlsHelper
 
                 if (videoRange == VideoRange.HDR)
                 {
-                    if (videoRangeType == VideoRangeType.HLG)
+                    switch (videoRangeType)
                     {
-                        builder.Append(",VIDEO-RANGE=HLG");
-                    }
-                    else
-                    {
-                        builder.Append(",VIDEO-RANGE=PQ");
+                        case VideoRangeType.HLG:
+                        case VideoRangeType.DOVIWithHLG:
+                            builder.Append(",VIDEO-RANGE=HLG");
+                            break;
+                        default:
+                            builder.Append(",VIDEO-RANGE=PQ");
+                            break;
                     }
                 }
             }
@@ -418,36 +420,67 @@ public class DynamicHlsHelper
     /// <param name="state">StreamState of the current stream.</param>
     private void AppendPlaylistSupplementalCodecsField(StringBuilder builder, StreamState state)
     {
-        // Dolby Vision currently cannot exist when transcoding
+        // HDR dynamic metadata currently cannot exist when transcoding
         if (!EncodingHelper.IsCopyCodec(state.OutputVideoCodec))
         {
             return;
         }
 
-        var dvProfile = state.VideoStream.DvProfile;
-        var dvLevel = state.VideoStream.DvLevel;
-        var dvRangeString = state.VideoStream.VideoRangeType switch
+        if (EncodingHelper.IsDovi(state.VideoStream) && !_encodingHelper.IsDoviRemoved(state))
         {
-            VideoRangeType.DOVIWithHDR10 => "db1p",
-            VideoRangeType.DOVIWithHLG => "db4h",
-            _ => string.Empty
-        };
+            AppendDvString();
+        }
+        else if (EncodingHelper.IsHdr10Plus(state.VideoStream) && !_encodingHelper.IsHdr10PlusRemoved(state))
+        {
+            AppendHdr10PlusString();
+        }
 
-        if (dvProfile is null || dvLevel is null || string.IsNullOrEmpty(dvRangeString))
+        return;
+
+        void AppendDvString()
         {
-            return;
+            var dvProfile = state.VideoStream.DvProfile;
+            var dvLevel = state.VideoStream.DvLevel;
+            var dvRangeString = state.VideoStream.VideoRangeType switch
+            {
+                VideoRangeType.DOVIWithHDR10 => "db1p",
+                VideoRangeType.DOVIWithHLG => "db4h",
+                VideoRangeType.DOVIWithHDR10Plus => "db1p", // The HDR10+ metadata would be removed if Dovi metadata is not removed
+                _ => string.Empty // Don't label Dovi with EL and SDR due to compatability issues, ignore invalid configurations
+            };
+
+            if (dvProfile is null || dvLevel is null || string.IsNullOrEmpty(dvRangeString))
+            {
+                return;
+            }
+
+            var dvFourCc = string.Equals(state.ActualOutputVideoCodec, "av1", StringComparison.OrdinalIgnoreCase) ? "dav1" : "dvh1";
+            builder.Append(",SUPPLEMENTAL-CODECS=\"")
+                .Append(dvFourCc)
+                .Append('.')
+                .Append(dvProfile.Value.ToString("D2", CultureInfo.InvariantCulture))
+                .Append('.')
+                .Append(dvLevel.Value.ToString("D2", CultureInfo.InvariantCulture))
+                .Append('/')
+                .Append(dvRangeString)
+                .Append('"');
         }
 
-        var dvFourCc = string.Equals(state.ActualOutputVideoCodec, "av1", StringComparison.OrdinalIgnoreCase) ? "dav1" : "dvh1";
-        builder.Append(",SUPPLEMENTAL-CODECS=\"")
-            .Append(dvFourCc)
-            .Append('.')
-            .Append(dvProfile.Value.ToString("D2", CultureInfo.InvariantCulture))
-            .Append('.')
-            .Append(dvLevel.Value.ToString("D2", CultureInfo.InvariantCulture))
-            .Append('/')
-            .Append(dvRangeString)
-            .Append('"');
+        void AppendHdr10PlusString()
+        {
+            var videoCodecLevel = GetOutputVideoCodecLevel(state);
+            if (string.IsNullOrEmpty(state.ActualOutputVideoCodec) || videoCodecLevel is null)
+            {
+                return;
+            }
+
+            var videoCodecString = GetPlaylistVideoCodecs(state, state.ActualOutputVideoCodec, videoCodecLevel.Value);
+            builder.Append(",SUPPLEMENTAL-CODECS=\"")
+                .Append(videoCodecString)
+                .Append('/')
+                .Append("cdm4")
+                .Append('"');
+        }
     }
 
     /// <summary>