Merge pull request #6934 from nyanmisaka/hwa

HWA pipeline refactor, AMD/Intel/Nvidia full hardware filtering support, AV1 hwdec

1 files changed, 287 insertions, 42 deletions

diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs
index caa3d2368..6ef3a2ff9 100644
--- a/Jellyfin.Api/Controllers/DynamicHlsController.cs
+++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs
@@ -39,7 +39,8 @@ namespace Jellyfin.Api.Controllers
     [Authorize(Policy = Policies.DefaultAuthorization)]
     public class DynamicHlsController : BaseJellyfinApiController
     {
-        private const string DefaultEncoderPreset = "veryfast";
+        private const string DefaultVodEncoderPreset = "veryfast";
+        private const string DefaultEventEncoderPreset = "superfast";
         private const TranscodingJobType TranscodingJobType = MediaBrowser.Controller.MediaEncoding.TranscodingJobType.Hls;
 
         private readonly ILibraryManager _libraryManager;
@@ -106,6 +107,253 @@ namespace Jellyfin.Api.Controllers
         }
 
         /// <summary>
+        /// Gets a hls live stream.
+        /// </summary>
+        /// <param name="itemId">The item id.</param>
+        /// <param name="container">The audio container.</param>
+        /// <param name="static">Optional. If true, the original file will be streamed statically without any encoding. Use either no url extension or the original file extension. true/false.</param>
+        /// <param name="params">The streaming parameters.</param>
+        /// <param name="tag">The tag.</param>
+        /// <param name="deviceProfileId">Optional. The dlna device profile id to utilize.</param>
+        /// <param name="playSessionId">The play session id.</param>
+        /// <param name="segmentContainer">The segment container.</param>
+        /// <param name="segmentLength">The segment lenght.</param>
+        /// <param name="minSegments">The minimum number of segments.</param>
+        /// <param name="mediaSourceId">The media version id, if playing an alternate version.</param>
+        /// <param name="deviceId">The device id of the client requesting. Used to stop encoding processes when needed.</param>
+        /// <param name="audioCodec">Optional. Specify a audio codec to encode to, e.g. mp3. If omitted the server will auto-select using the url's extension. Options: aac, mp3, vorbis, wma.</param>
+        /// <param name="enableAutoStreamCopy">Whether or not to allow automatic stream copy if requested values match the original source. Defaults to true.</param>
+        /// <param name="allowVideoStreamCopy">Whether or not to allow copying of the video stream url.</param>
+        /// <param name="allowAudioStreamCopy">Whether or not to allow copying of the audio stream url.</param>
+        /// <param name="breakOnNonKeyFrames">Optional. Whether to break on non key frames.</param>
+        /// <param name="audioSampleRate">Optional. Specify a specific audio sample rate, e.g. 44100.</param>
+        /// <param name="maxAudioBitDepth">Optional. The maximum audio bit depth.</param>
+        /// <param name="audioBitRate">Optional. Specify an audio bitrate to encode to, e.g. 128000. If omitted this will be left to encoder defaults.</param>
+        /// <param name="audioChannels">Optional. Specify a specific number of audio channels to encode to, e.g. 2.</param>
+        /// <param name="maxAudioChannels">Optional. Specify a maximum number of audio channels to encode to, e.g. 2.</param>
+        /// <param name="profile">Optional. Specify a specific an encoder profile (varies by encoder), e.g. main, baseline, high.</param>
+        /// <param name="level">Optional. Specify a level for the encoder profile (varies by encoder), e.g. 3, 3.1.</param>
+        /// <param name="framerate">Optional. A specific video framerate to encode to, e.g. 23.976. Generally this should be omitted unless the device has specific requirements.</param>
+        /// <param name="maxFramerate">Optional. A specific maximum video framerate to encode to, e.g. 23.976. Generally this should be omitted unless the device has specific requirements.</param>
+        /// <param name="copyTimestamps">Whether or not to copy timestamps when transcoding with an offset. Defaults to false.</param>
+        /// <param name="startTimeTicks">Optional. Specify a starting offset, in ticks. 1 tick = 10000 ms.</param>
+        /// <param name="width">Optional. The fixed horizontal resolution of the encoded video.</param>
+        /// <param name="height">Optional. The fixed vertical resolution of the encoded video.</param>
+        /// <param name="videoBitRate">Optional. Specify a video bitrate to encode to, e.g. 500000. If omitted this will be left to encoder defaults.</param>
+        /// <param name="subtitleStreamIndex">Optional. The index of the subtitle stream to use. If omitted no subtitles will be used.</param>
+        /// <param name="subtitleMethod">Optional. Specify the subtitle delivery method.</param>
+        /// <param name="maxRefFrames">Optional.</param>
+        /// <param name="maxVideoBitDepth">Optional. The maximum video bit depth.</param>
+        /// <param name="requireAvc">Optional. Whether to require avc.</param>
+        /// <param name="deInterlace">Optional. Whether to deinterlace the video.</param>
+        /// <param name="requireNonAnamorphic">Optional. Whether to require a non anamorphic stream.</param>
+        /// <param name="transcodingMaxAudioChannels">Optional. The maximum number of audio channels to transcode.</param>
+        /// <param name="cpuCoreLimit">Optional. The limit of how many cpu cores to use.</param>
+        /// <param name="liveStreamId">The live stream id.</param>
+        /// <param name="enableMpegtsM2TsMode">Optional. Whether to enable the MpegtsM2Ts mode.</param>
+        /// <param name="videoCodec">Optional. Specify a video codec to encode to, e.g. h264. If omitted the server will auto-select using the url's extension. Options: h265, h264, mpeg4, theora, vp8, vp9, vpx (deprecated), wmv.</param>
+        /// <param name="subtitleCodec">Optional. Specify a subtitle codec to encode to.</param>
+        /// <param name="transcodeReasons">Optional. The transcoding reason.</param>
+        /// <param name="audioStreamIndex">Optional. The index of the audio stream to use. If omitted the first audio stream will be used.</param>
+        /// <param name="videoStreamIndex">Optional. The index of the video stream to use. If omitted the first video stream will be used.</param>
+        /// <param name="context">Optional. The <see cref="EncodingContext"/>.</param>
+        /// <param name="streamOptions">Optional. The streaming options.</param>
+        /// <param name="maxWidth">Optional. The max width.</param>
+        /// <param name="maxHeight">Optional. The max height.</param>
+        /// <param name="enableSubtitlesInManifest">Optional. Whether to enable subtitles in the manifest.</param>
+        /// <response code="200">Hls live stream retrieved.</response>
+        /// <returns>A <see cref="FileResult"/> containing the hls file.</returns>
+        [HttpGet("Videos/{itemId}/live.m3u8")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        [ProducesPlaylistFile]
+        public async Task<ActionResult> GetLiveHlsStream(
+            [FromRoute, Required] Guid itemId,
+            [FromQuery] string? container,
+            [FromQuery] bool? @static,
+            [FromQuery] string? @params,
+            [FromQuery] string? tag,
+            [FromQuery] string? deviceProfileId,
+            [FromQuery] string? playSessionId,
+            [FromQuery] string? segmentContainer,
+            [FromQuery] int? segmentLength,
+            [FromQuery] int? minSegments,
+            [FromQuery] string? mediaSourceId,
+            [FromQuery] string? deviceId,
+            [FromQuery] string? audioCodec,
+            [FromQuery] bool? enableAutoStreamCopy,
+            [FromQuery] bool? allowVideoStreamCopy,
+            [FromQuery] bool? allowAudioStreamCopy,
+            [FromQuery] bool? breakOnNonKeyFrames,
+            [FromQuery] int? audioSampleRate,
+            [FromQuery] int? maxAudioBitDepth,
+            [FromQuery] int? audioBitRate,
+            [FromQuery] int? audioChannels,
+            [FromQuery] int? maxAudioChannels,
+            [FromQuery] string? profile,
+            [FromQuery] string? level,
+            [FromQuery] float? framerate,
+            [FromQuery] float? maxFramerate,
+            [FromQuery] bool? copyTimestamps,
+            [FromQuery] long? startTimeTicks,
+            [FromQuery] int? width,
+            [FromQuery] int? height,
+            [FromQuery] int? videoBitRate,
+            [FromQuery] int? subtitleStreamIndex,
+            [FromQuery] SubtitleDeliveryMethod? subtitleMethod,
+            [FromQuery] int? maxRefFrames,
+            [FromQuery] int? maxVideoBitDepth,
+            [FromQuery] bool? requireAvc,
+            [FromQuery] bool? deInterlace,
+            [FromQuery] bool? requireNonAnamorphic,
+            [FromQuery] int? transcodingMaxAudioChannels,
+            [FromQuery] int? cpuCoreLimit,
+            [FromQuery] string? liveStreamId,
+            [FromQuery] bool? enableMpegtsM2TsMode,
+            [FromQuery] string? videoCodec,
+            [FromQuery] string? subtitleCodec,
+            [FromQuery] string? transcodeReasons,
+            [FromQuery] int? audioStreamIndex,
+            [FromQuery] int? videoStreamIndex,
+            [FromQuery] EncodingContext? context,
+            [FromQuery] Dictionary<string, string> streamOptions,
+            [FromQuery] int? maxWidth,
+            [FromQuery] int? maxHeight,
+            [FromQuery] bool? enableSubtitlesInManifest)
+        {
+            VideoRequestDto streamingRequest = new VideoRequestDto
+            {
+                Id = itemId,
+                Container = container,
+                Static = @static ?? false,
+                Params = @params,
+                Tag = tag,
+                DeviceProfileId = deviceProfileId,
+                PlaySessionId = playSessionId,
+                SegmentContainer = segmentContainer,
+                SegmentLength = segmentLength,
+                MinSegments = minSegments,
+                MediaSourceId = mediaSourceId,
+                DeviceId = deviceId,
+                AudioCodec = audioCodec,
+                EnableAutoStreamCopy = enableAutoStreamCopy ?? true,
+                AllowAudioStreamCopy = allowAudioStreamCopy ?? true,
+                AllowVideoStreamCopy = allowVideoStreamCopy ?? true,
+                BreakOnNonKeyFrames = breakOnNonKeyFrames ?? false,
+                AudioSampleRate = audioSampleRate,
+                MaxAudioChannels = maxAudioChannels,
+                AudioBitRate = audioBitRate,
+                MaxAudioBitDepth = maxAudioBitDepth,
+                AudioChannels = audioChannels,
+                Profile = profile,
+                Level = level,
+                Framerate = framerate,
+                MaxFramerate = maxFramerate,
+                CopyTimestamps = copyTimestamps ?? false,
+                StartTimeTicks = startTimeTicks,
+                Width = width,
+                Height = height,
+                VideoBitRate = videoBitRate,
+                SubtitleStreamIndex = subtitleStreamIndex,
+                SubtitleMethod = subtitleMethod ?? SubtitleDeliveryMethod.Encode,
+                MaxRefFrames = maxRefFrames,
+                MaxVideoBitDepth = maxVideoBitDepth,
+                RequireAvc = requireAvc ?? false,
+                DeInterlace = deInterlace ?? false,
+                RequireNonAnamorphic = requireNonAnamorphic ?? false,
+                TranscodingMaxAudioChannels = transcodingMaxAudioChannels,
+                CpuCoreLimit = cpuCoreLimit,
+                LiveStreamId = liveStreamId,
+                EnableMpegtsM2TsMode = enableMpegtsM2TsMode ?? false,
+                VideoCodec = videoCodec,
+                SubtitleCodec = subtitleCodec,
+                TranscodeReasons = transcodeReasons,
+                AudioStreamIndex = audioStreamIndex,
+                VideoStreamIndex = videoStreamIndex,
+                Context = context ?? EncodingContext.Streaming,
+                StreamOptions = streamOptions,
+                MaxHeight = maxHeight,
+                MaxWidth = maxWidth,
+                EnableSubtitlesInManifest = enableSubtitlesInManifest ?? true
+            };
+
+            // CTS lifecycle is managed internally.
+            var cancellationTokenSource = new CancellationTokenSource();
+            // Due to CTS.Token calling ThrowIfDisposed (https://github.com/dotnet/runtime/issues/29970) we have to "cache" the token
+            // since it gets disposed when ffmpeg exits
+            var cancellationToken = cancellationTokenSource.Token;
+            using var state = await StreamingHelpers.GetStreamingState(
+                    streamingRequest,
+                    Request,
+                    _authContext,
+                    _mediaSourceManager,
+                    _userManager,
+                    _libraryManager,
+                    _serverConfigurationManager,
+                    _mediaEncoder,
+                    _encodingHelper,
+                    _dlnaManager,
+                    _deviceManager,
+                    _transcodingJobHelper,
+                    TranscodingJobType,
+                    cancellationToken)
+                .ConfigureAwait(false);
+
+            TranscodingJobDto? job = null;
+            var playlistPath = Path.ChangeExtension(state.OutputFilePath, ".m3u8");
+
+            if (!System.IO.File.Exists(playlistPath))
+            {
+                var transcodingLock = _transcodingJobHelper.GetTranscodingLock(playlistPath);
+                await transcodingLock.WaitAsync(cancellationToken).ConfigureAwait(false);
+                try
+                {
+                    if (!System.IO.File.Exists(playlistPath))
+                    {
+                        // If the playlist doesn't already exist, startup ffmpeg
+                        try
+                        {
+                            job = await _transcodingJobHelper.StartFfMpeg(
+                                    state,
+                                    playlistPath,
+                                    GetCommandLineArguments(playlistPath, state, true, 0),
+                                    Request,
+                                    TranscodingJobType,
+                                    cancellationTokenSource)
+                                .ConfigureAwait(false);
+                            job.IsLiveOutput = true;
+                        }
+                        catch
+                        {
+                            state.Dispose();
+                            throw;
+                        }
+
+                        minSegments = state.MinSegments;
+                        if (minSegments > 0)
+                        {
+                            await HlsHelpers.WaitForMinimumSegmentCount(playlistPath, minSegments, _logger, cancellationToken).ConfigureAwait(false);
+                        }
+                    }
+                }
+                finally
+                {
+                    transcodingLock.Release();
+                }
+            }
+
+            job ??= _transcodingJobHelper.OnTranscodeBeginRequest(playlistPath, TranscodingJobType);
+
+            if (job != null)
+            {
+                _transcodingJobHelper.OnTranscodeEndRequest(job);
+            }
+
+            var playlistText = HlsHelpers.GetLivePlaylistText(playlistPath, state);
+
+            return Content(playlistText, MimeTypes.GetMimeType("playlist.m3u8"));
+        }
+
+        /// <summary>
         /// Gets a video hls playlist stream.
         /// </summary>
         /// <param name="itemId">The item id.</param>
@@ -1149,7 +1397,7 @@ namespace Jellyfin.Api.Controllers
                 .AppendLine("#EXT-X-MEDIA-SEQUENCE:0");
 
             var index = 0;
-            var segmentExtension = GetSegmentFileExtension(streamingRequest.SegmentContainer);
+            var segmentExtension = EncodingHelper.GetSegmentFileExtension(streamingRequest.SegmentContainer);
             var queryString = Request.QueryString;
 
             if (isHlsInFmp4)
@@ -1214,7 +1462,7 @@ namespace Jellyfin.Api.Controllers
 
             var segmentPath = GetSegmentPath(state, playlistPath, segmentId);
 
-            var segmentExtension = GetSegmentFileExtension(state.Request.SegmentContainer);
+            var segmentExtension = EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer);
 
             TranscodingJobDto? job;
 
@@ -1286,7 +1534,7 @@ namespace Jellyfin.Api.Controllers
                             job = await _transcodingJobHelper.StartFfMpeg(
                                 state,
                                 playlistPath,
-                                GetCommandLineArguments(playlistPath, state, true, segmentId),
+                                GetCommandLineArguments(playlistPath, state, false, segmentId),
                                 Request,
                                 TranscodingJobType,
                                 cancellationTokenSource).ConfigureAwait(false);
@@ -1346,7 +1594,7 @@ namespace Jellyfin.Api.Controllers
             return segments;
         }
 
-        private string GetCommandLineArguments(string outputPath, StreamState state, bool isEncoding, int startNumber)
+        private string GetCommandLineArguments(string outputPath, StreamState state, bool isEventPlaylist, int startNumber)
         {
             var videoCodec = _encodingHelper.GetVideoEncoder(state, _encodingOptions);
             var threads = EncodingHelper.GetNumberOfThreads(state, _encodingOptions, videoCodec);
@@ -1361,15 +1609,13 @@ namespace Jellyfin.Api.Controllers
                 state.BaseRequest.BreakOnNonKeyFrames = false;
             }
 
-            // If isEncoding is true we're actually starting ffmpeg
-            var startNumberParam = isEncoding ? startNumber.ToString(CultureInfo.InvariantCulture) : "0";
             var inputModifier = _encodingHelper.GetInputModifier(state, _encodingOptions);
             var mapArgs = state.IsOutputVideo ? _encodingHelper.GetMapArgs(state) : string.Empty;
 
             var directory = Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath));
             var outputFileNameWithoutExtension = Path.GetFileNameWithoutExtension(outputPath);
             var outputPrefix = Path.Combine(directory, outputFileNameWithoutExtension);
-            var outputExtension = GetSegmentFileExtension(state.Request.SegmentContainer);
+            var outputExtension = EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer);
             var outputTsArg = outputPrefix + "%d" + outputExtension;
 
             var segmentFormat = outputExtension.TrimStart('.');
@@ -1398,19 +1644,30 @@ namespace Jellyfin.Api.Controllers
                 ? _encodingOptions.MaxMuxingQueueSize.ToString(CultureInfo.InvariantCulture)
                 : "128";
 
+            var baseUrlParam = string.Empty;
+            if (isEventPlaylist)
+            {
+                baseUrlParam = string.Format(
+                    CultureInfo.InvariantCulture,
+                    " -hls_base_url \"hls/{0}/\"",
+                    Path.GetFileNameWithoutExtension(outputPath));
+            }
+
             return string.Format(
                 CultureInfo.InvariantCulture,
-                "{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -copyts -avoid_negative_ts disabled -max_muxing_queue_size {6} -f hls -max_delay 5000000 -hls_time {7} -hls_segment_type {8} -start_number {9} -hls_segment_filename \"{10}\" -hls_playlist_type vod -hls_list_size 0 -y \"{11}\"",
+                "{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -copyts -avoid_negative_ts disabled -max_muxing_queue_size {6} -f hls -max_delay 5000000 -hls_time {7} -hls_segment_type {8} -start_number {9}{10} -hls_segment_filename \"{12}\" -hls_playlist_type {11} -hls_list_size 0 -y \"{13}\"",
                 inputModifier,
                 _encodingHelper.GetInputArgument(state, _encodingOptions),
                 threads,
                 mapArgs,
-                GetVideoArguments(state, startNumber),
+                GetVideoArguments(state, startNumber, isEventPlaylist),
                 GetAudioArguments(state),
                 maxMuxingQueueSize,
                 state.SegmentLength.ToString(CultureInfo.InvariantCulture),
                 segmentFormat,
-                startNumberParam,
+                startNumber.ToString(CultureInfo.InvariantCulture),
+                baseUrlParam,
+                isEventPlaylist ? "event" : "vod",
                 outputTsArg,
                 outputPath).Trim();
         }
@@ -1505,8 +1762,9 @@ namespace Jellyfin.Api.Controllers
         /// </summary>
         /// <param name="state">The <see cref="StreamState"/>.</param>
         /// <param name="startNumber">The first number in the hls sequence.</param>
+        /// <param name="isEventPlaylist">Whether the playlist is EVENT or VOD.</param>
         /// <returns>The command line arguments for video transcoding.</returns>
-        private string GetVideoArguments(StreamState state, int startNumber)
+        private string GetVideoArguments(StreamState state, int startNumber, bool isEventPlaylist)
         {
             if (state.VideoStream == null)
             {
@@ -1539,6 +1797,7 @@ namespace Jellyfin.Api.Controllers
             // See if we can save come cpu cycles by avoiding encoding.
             if (EncodingHelper.IsCopyCodec(codec))
             {
+                // If h264_mp4toannexb is ever added, do not use it for live tv.
                 if (state.VideoStream != null && !string.Equals(state.VideoStream.NalLengthSize, "0", StringComparison.OrdinalIgnoreCase))
                 {
                     string bitStreamArgs = EncodingHelper.GetBitStreamArgs(state.VideoStream);
@@ -1549,15 +1808,13 @@ namespace Jellyfin.Api.Controllers
                 }
 
                 args += " -start_at_zero";
-
-                // args += " -flags -global_header";
             }
             else
             {
-                args += _encodingHelper.GetVideoQualityParam(state, codec, _encodingOptions, DefaultEncoderPreset);
+                args += _encodingHelper.GetVideoQualityParam(state, codec, _encodingOptions, isEventPlaylist ? DefaultEventEncoderPreset : DefaultVodEncoderPreset);
 
                 // Set the key frame params for video encoding to match the hls segment time.
-                args += _encodingHelper.GetHlsVideoKeyFrameArguments(state, codec, state.SegmentLength, false, startNumber);
+                args += _encodingHelper.GetHlsVideoKeyFrameArguments(state, codec, state.SegmentLength, isEventPlaylist, startNumber);
 
                 // Currenly b-frames in libx265 breaks the FMP4-HLS playback on iOS, disable it for now.
                 if (string.Equals(codec, "libx265", StringComparison.OrdinalIgnoreCase))
@@ -1567,27 +1824,25 @@ namespace Jellyfin.Api.Controllers
 
                 // args += " -mixed-refs 0 -refs 3 -x264opts b_pyramid=0:weightb=0:weightp=0";
 
-                var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
-
-                if (hasGraphicalSubs)
-                {
-                    // Graphical subs overlay and resolution params.
-                    args += _encodingHelper.GetGraphicalSubtitleParam(state, _encodingOptions, codec);
-                }
-                else
-                {
-                    // Resolution params.
-                    args += _encodingHelper.GetOutputSizeParam(state, _encodingOptions, codec);
-                }
+                // video processing filters.
+                args += _encodingHelper.GetVideoProcessingFilterParam(state, _encodingOptions, codec);
 
                 // -start_at_zero is necessary to use with -ss when seeking,
                 // otherwise the target position cannot be determined.
-                if (!(state.SubtitleStream != null && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream))
+                if (state.SubtitleStream != null)
                 {
-                    args += " -start_at_zero";
+                    // Disable start_at_zero for external graphical subs
+                    if (!(state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream))
+                    {
+                        args += " -start_at_zero";
+                    }
                 }
+            }
 
-                // args += " -flags -global_header";
+            // TODO why was this not enabled for VOD?
+            if (isEventPlaylist)
+            {
+                args += " -flags -global_header";
             }
 
             if (!string.IsNullOrEmpty(state.OutputVideoSync))
@@ -1600,22 +1855,12 @@ namespace Jellyfin.Api.Controllers
             return args;
         }
 
-        private string GetSegmentFileExtension(string? segmentContainer)
-        {
-            if (!string.IsNullOrWhiteSpace(segmentContainer))
-            {
-                return "." + segmentContainer;
-            }
-
-            return ".ts";
-        }
-
         private string GetSegmentPath(StreamState state, string playlist, int index)
         {
             var folder = Path.GetDirectoryName(playlist) ?? throw new ArgumentException($"Provided path ({playlist}) is not valid.", nameof(playlist));
             var filename = Path.GetFileNameWithoutExtension(playlist);
 
-            return Path.Combine(folder, filename + index.ToString(CultureInfo.InvariantCulture) + GetSegmentFileExtension(state.Request.SegmentContainer));
+            return Path.Combine(folder, filename + index.ToString(CultureInfo.InvariantCulture) + EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer));
         }
 
         private async Task<ActionResult> GetSegmentResult(