diff options
Diffstat (limited to 'MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs')
| -rw-r--r-- | MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs | 312 |
1 files changed, 258 insertions, 54 deletions
diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index b6738e7cc..eb375c8a2 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -1,6 +1,8 @@ #nullable disable #pragma warning disable CS1591 +// We need lowercase normalized string for ffmpeg +#pragma warning disable CA1308 using System; using System.Collections.Generic; @@ -26,6 +28,14 @@ namespace MediaBrowser.Controller.MediaEncoding { public partial class EncodingHelper { + /// <summary> + /// The codec validation regex. + /// This regular expression matches strings that consist of alphanumeric characters, hyphens, + /// periods, underscores, commas, and vertical bars, with a length between 0 and 40 characters. + /// This should matches all common valid codecs. + /// </summary> + public const string ValidationRegex = @"^[a-zA-Z0-9\-\._,|]{0,40}$"; + private const string QsvAlias = "qs"; private const string VaapiAlias = "va"; private const string D3d11vaAlias = "dx11"; @@ -51,6 +61,9 @@ namespace MediaBrowser.Controller.MediaEncoding private readonly Version _minFFmpegOclCuTonemapMode = new Version(5, 1, 3); private readonly Version _minFFmpegSvtAv1Params = new Version(5, 1); private readonly Version _minFFmpegVaapiH26xEncA53CcSei = new Version(6, 0); + private readonly Version _minFFmpegReadrateOption = new Version(5, 0); + + private static readonly Regex _validationRegex = new(ValidationRegex, RegexOptions.Compiled); private static readonly string[] _videoProfilesH264 = new[] { @@ -94,7 +107,6 @@ namespace MediaBrowser.Controller.MediaEncoding { "wmav2", 2 }, { "libmp3lame", 2 }, { "libfdk_aac", 6 }, - { "aac_at", 6 }, { "ac3", 6 }, { "eac3", 6 }, { "dca", 6 }, @@ -253,6 +265,15 @@ namespace MediaBrowser.Controller.MediaEncoding && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync); } + private bool IsVideoToolboxFullSupported() + { + return _mediaEncoder.SupportsHwaccel("videotoolbox") + && _mediaEncoder.SupportsFilter("yadif_videotoolbox") + && _mediaEncoder.SupportsFilter("overlay_videotoolbox") + && _mediaEncoder.SupportsFilter("tonemap_videotoolbox") + && _mediaEncoder.SupportsFilter("scale_vt"); + } + private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { if (state.VideoStream is null @@ -272,12 +293,15 @@ namespace MediaBrowser.Controller.MediaEncoding var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); - return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder; + var isVideoToolBoxDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder || isVideoToolBoxDecoder; } return state.VideoStream.VideoRange == VideoRange.HDR && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 - || state.VideoStream.VideoRangeType == VideoRangeType.HLG); + || state.VideoStream.VideoRangeType == VideoRangeType.HLG + || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10 + || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHLG); } private bool IsVulkanHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) @@ -305,7 +329,23 @@ namespace MediaBrowser.Controller.MediaEncoding // Native VPP tonemapping may come to QSV in the future. return state.VideoStream.VideoRange == VideoRange.HDR - && state.VideoStream.VideoRangeType == VideoRangeType.HDR10; + && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 + || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10); + } + + private bool IsVideoToolboxTonemapAvailable(EncodingJobInfo state, EncodingOptions options) + { + if (state.VideoStream is null + || !options.EnableVideoToolboxTonemapping + || GetVideoColorBitDepth(state) != 10) + { + return false; + } + + // Certain DV profile 5 video works in Safari with direct playing, but the VideoToolBox does not produce correct mapping results with transcoding. + // All other HDR formats working. + return state.VideoStream.VideoRange == VideoRange.HDR + && state.VideoStream.VideoRangeType is VideoRangeType.HDR10 or VideoRangeType.HLG or VideoRangeType.HDR10Plus or VideoRangeType.DOVIWithHDR10 or VideoRangeType.DOVIWithHLG; } /// <summary> @@ -362,7 +402,10 @@ namespace MediaBrowser.Controller.MediaEncoding return "libtheora"; } - return codec.ToLowerInvariant(); + if (_validationRegex.IsMatch(codec)) + { + return codec.ToLowerInvariant(); + } } return "copy"; @@ -400,7 +443,7 @@ namespace MediaBrowser.Controller.MediaEncoding public static string GetInputFormat(string container) { - if (string.IsNullOrEmpty(container)) + if (string.IsNullOrEmpty(container) || !_validationRegex.IsMatch(container)) { return null; } @@ -656,6 +699,11 @@ namespace MediaBrowser.Controller.MediaEncoding { var codec = state.OutputAudioCodec; + if (!_validationRegex.IsMatch(codec)) + { + codec = "aac"; + } + if (string.Equals(codec, "aac", StringComparison.OrdinalIgnoreCase)) { // Use Apple's aac encoder if available as it provides best audio quality @@ -703,6 +751,15 @@ namespace MediaBrowser.Controller.MediaEncoding return "dca"; } + if (string.Equals(codec, "alac", StringComparison.OrdinalIgnoreCase)) + { + // The ffmpeg upstream breaks the AudioToolbox ALAC encoder in version 6.1 but fixes it in version 7.0. + // Since ALAC is lossless in quality and the AudioToolbox encoder is not faster, + // its only benefit is a smaller file size. + // To prevent problems, use the ffmpeg native encoder instead. + return "alac"; + } + return codec.ToLowerInvariant(); } @@ -1071,7 +1128,7 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Empty; } - // no videotoolbox hw filter. + // videotoolbox hw filter does not require device selection args.Append(GetVideoToolboxDeviceArgs(VideotoolboxAlias)); } else if (string.Equals(optHwaccelType, "rkmpp", StringComparison.OrdinalIgnoreCase)) @@ -1197,7 +1254,7 @@ namespace MediaBrowser.Controller.MediaEncoding // Disable auto inserted SW scaler for HW decoders in case of changed resolution. var isSwDecoder = string.IsNullOrEmpty(GetHardwareVideoDecoder(state, options)); - if (!isSwDecoder && _mediaEncoder.EncoderVersion >= new Version(4, 4)) + if (!isSwDecoder) { arg.Append(" -noautoscale"); } @@ -1214,23 +1271,23 @@ namespace MediaBrowser.Controller.MediaEncoding { var codec = stream.Codec ?? string.Empty; - return codec.IndexOf("264", StringComparison.OrdinalIgnoreCase) != -1 - || codec.IndexOf("avc", StringComparison.OrdinalIgnoreCase) != -1; + return codec.Contains("264", StringComparison.OrdinalIgnoreCase) + || codec.Contains("avc", StringComparison.OrdinalIgnoreCase); } public static bool IsH265(MediaStream stream) { var codec = stream.Codec ?? string.Empty; - return codec.IndexOf("265", StringComparison.OrdinalIgnoreCase) != -1 - || codec.IndexOf("hevc", StringComparison.OrdinalIgnoreCase) != -1; + return codec.Contains("265", StringComparison.OrdinalIgnoreCase) + || codec.Contains("hevc", StringComparison.OrdinalIgnoreCase); } public static bool IsAAC(MediaStream stream) { var codec = stream.Codec ?? string.Empty; - return codec.IndexOf("aac", StringComparison.OrdinalIgnoreCase) != -1; + return codec.Contains("aac", StringComparison.OrdinalIgnoreCase); } public static string GetBitStreamArgs(MediaStream stream) @@ -1284,7 +1341,7 @@ namespace MediaBrowser.Controller.MediaEncoding return ".ts"; } - public string GetVideoBitrateParam(EncodingJobInfo state, string videoCodec) + private string GetVideoBitrateParam(EncodingJobInfo state, string videoCodec) { if (state.OutputVideoBitrate is null) { @@ -1348,6 +1405,14 @@ namespace MediaBrowser.Controller.MediaEncoding return FormattableString.Invariant($" -rc_mode VBR -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); } + if (string.Equals(videoCodec, "h264_videotoolbox", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "hevc_videotoolbox", StringComparison.OrdinalIgnoreCase)) + { + // The `maxrate` and `bufsize` options can potentially lead to performance regression + // and even encoder hangs, especially when the value is very high. + return FormattableString.Invariant($" -b:v {bitrate} -qmin -1 -qmax -1"); + } + return FormattableString.Invariant($" -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); } @@ -1818,6 +1883,31 @@ namespace MediaBrowser.Controller.MediaEncoding param += " -gops_per_idr 1"; } } + else if (string.Equals(videoEncoder, "h264_videotoolbox", StringComparison.OrdinalIgnoreCase) // h264 (h264_videotoolbox) + || string.Equals(videoEncoder, "hevc_videotoolbox", StringComparison.OrdinalIgnoreCase)) // hevc (hevc_videotoolbox) + { + switch (encodingOptions.EncoderPreset) + { + case "veryslow": + case "slower": + case "slow": + case "medium": + param += " -prio_speed 0"; + break; + + case "fast": + case "faster": + case "veryfast": + case "superfast": + case "ultrafast": + param += " -prio_speed 1"; + break; + + default: + param += " -prio_speed 1"; + break; + } + } else if (string.Equals(videoEncoder, "libvpx", StringComparison.OrdinalIgnoreCase)) // vp8 { // Values 0-3, 0 being highest quality but slower @@ -2181,7 +2271,16 @@ namespace MediaBrowser.Controller.MediaEncoding return false; } - if (!requestedRangeTypes.Contains(videoStream.VideoRangeType.ToString(), StringComparison.OrdinalIgnoreCase)) + // DOVIWithHDR10 should be compatible with HDR10 supporting players. Same goes with HLG and of course SDR. So allow copy of those formats + + var requestHasHDR10 = requestedRangeTypes.Contains(VideoRangeType.HDR10.ToString(), StringComparison.OrdinalIgnoreCase); + var requestHasHLG = requestedRangeTypes.Contains(VideoRangeType.HLG.ToString(), StringComparison.OrdinalIgnoreCase); + var requestHasSDR = requestedRangeTypes.Contains(VideoRangeType.SDR.ToString(), StringComparison.OrdinalIgnoreCase); + + if (!requestedRangeTypes.Contains(videoStream.VideoRangeType.ToString(), StringComparison.OrdinalIgnoreCase) + && !((requestHasHDR10 && videoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10) + || (requestHasHLG && videoStream.VideoRangeType == VideoRangeType.DOVIWithHLG) + || (requestHasSDR && videoStream.VideoRangeType == VideoRangeType.DOVIWithSDR))) { return false; } @@ -4954,22 +5053,29 @@ namespace MediaBrowser.Controller.MediaEncoding return (null, null, null); } - var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder); + var isMacOS = OperatingSystem.IsMacOS(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported(); - if (!options.EnableHardwareEncoding) + // legacy videotoolbox pipeline (disable hw filters) + if (!isVtEncoder + || !isVtFullSupported + || !_mediaEncoder.SupportsFilter("alphasrc")) { - return swFilterChain; + return GetSwVidFilterChain(state, options, vidEncoder); } - if (_mediaEncoder.EncoderVersion.CompareTo(new Version("5.0.0")) < 0) - { - // All features used here requires ffmpeg 5.0 or later, fallback to software filters if using an old ffmpeg - return swFilterChain; - } + // preferred videotoolbox + metal filters pipeline + return GetAppleVidFiltersPreferred(state, options, vidDecoder, vidEncoder); + } - var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); - var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); - var doDeintH2645 = doDeintH264 || doDeintHevc; + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAppleVidFiltersPreferred( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { var inW = state.VideoStream?.Width; var inH = state.VideoStream?.Height; var reqW = state.BaseRequest.Width; @@ -4977,33 +5083,121 @@ namespace MediaBrowser.Controller.MediaEncoding var reqMaxW = state.BaseRequest.MaxWidth; var reqMaxH = state.BaseRequest.MaxHeight; var threeDFormat = state.MediaSource.Video3DFormat; - var newfilters = new List<string>(); - var noOverlay = swFilterChain.OverlayFilters.Count == 0; - var supportsHwDeint = _mediaEncoder.SupportsFilter("yadif_videotoolbox"); - // fallback to software filters if we are using filters not supported by hardware yet. - var useHardwareFilters = noOverlay && (!doDeintH2645 || supportsHwDeint); - if (!useHardwareFilters) + var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options); + var doMetalTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options); + + var scaleFormat = string.Empty; + // Use P010 for Metal tone mapping, otherwise force an 8bit output. + if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase)) { - return swFilterChain; + if (doMetalTonemap) + { + if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p10le", StringComparison.OrdinalIgnoreCase)) + { + scaleFormat = "p010le"; + } + } + else + { + scaleFormat = "nv12"; + } } - // ffmpeg cannot use videotoolbox to scale - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); - newfilters.Add(swScaleFilter); + var hwScaleFilter = GetHwScaleFilter("vt", scaleFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + if (!isVtEncoder) + { + // should not happen. + return (null, null, null); + } - // hwupload on videotoolbox encoders can automatically convert AVFrame into its CVPixelBuffer equivalent - // videotoolbox will automatically convert the CVPixelBuffer to a pixel format the encoder supports, so we don't have to set a pixel format explicitly here - // This will reduce CPU usage significantly on UHD videos with 10 bit colors because we bypassed the ffmpeg pixel format conversion - newfilters.Add("hwupload"); + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + // hw deint if (doDeintH2645) { var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox"); - newfilters.Add(deintFilter); + mainFilters.Add(deintFilter); + } + + if (doVtTonemap) + { + const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709"; + + // scale_vt can handle scaling & tonemapping in one shot, just like vpp_qsv. + hwScaleFilter = string.IsNullOrEmpty(hwScaleFilter) + ? "scale_vt=" + VtTonemapArgs + : hwScaleFilter + ":" + VtTonemapArgs; + } + + // hw scale & vt tonemap + mainFilters.Add(hwScaleFilter); + + // Metal tonemap + if (doMetalTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "videotoolbox", "nv12"); + mainFilters.Add(tonemapFilter); + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + + if (hasSubs) + { + if (hasGraphicalSubs) + { + var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subPreProcFilters); + subFilters.Add("format=bgra"); + } + else if (hasTextSubs) + { + var framerate = state.VideoStream?.RealFrameRate; + var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; + + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); + } + + subFilters.Add("hwupload=derive_device=videotoolbox"); + overlayFilters.Add("overlay_videotoolbox=eof_action=pass:repeatlast=0"); } - return (newfilters, swFilterChain.SubFilters, swFilterChain.OverlayFilters); + var needFiltering = mainFilters.Any(f => !string.IsNullOrEmpty(f)) || + subFilters.Any(f => !string.IsNullOrEmpty(f)) || + overlayFilters.Any(f => !string.IsNullOrEmpty(f)); + + // This is a workaround for ffmpeg's hwupload implementation + // For VideoToolbox encoders, a hwupload without a valid filter actually consuming its frame + // will cause the encoder to produce incorrect frames. + if (needFiltering) + { + // INPUT videotoolbox/memory surface(vram/uma) + // this will pass-through automatically if in/out format matches. + mainFilters.Insert(0, "format=nv12|p010le|videotoolbox_vld"); + mainFilters.Insert(0, "hwupload=derive_device=videotoolbox"); + } + + return (mainFilters, subFilters, overlayFilters); } /// <summary> @@ -5995,22 +6189,22 @@ namespace MediaBrowser.Controller.MediaEncoding || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases. + // For example: https://trac.ffmpeg.org/ticket/10884 + // Disable it for now. + const bool UseHwSurface = false; + if (is8bitSwFormatsVt) { if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "h264", bitDepth, false); - } - - if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) - { - return GetHwaccelType(state, options, "mpeg2video", bitDepth, false); + return GetHwaccelType(state, options, "h264", bitDepth, UseHwSurface); } - if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "mpeg4", bitDepth, false); + return GetHwaccelType(state, options, "vp8", bitDepth, UseHwSurface); } } @@ -6019,12 +6213,12 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "hevc", bitDepth, false); + return GetHwaccelType(state, options, "hevc", bitDepth, UseHwSurface); } if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "vp9", bitDepth, false); + return GetHwaccelType(state, options, "vp9", bitDepth, UseHwSurface); } } @@ -6265,6 +6459,16 @@ namespace MediaBrowser.Controller.MediaEncoding { inputModifier += " -re"; } + else if (encodingOptions.EnableSegmentDeletion + && state.VideoStream is not null + && state.TranscodingType == TranscodingJobType.Hls + && IsCopyCodec(state.OutputVideoCodec) + && _mediaEncoder.EncoderVersion >= _minFFmpegReadrateOption) + { + // Set an input read rate limit 10x for using SegmentDeletion with stream-copy + // to prevent ffmpeg from exiting prematurely (due to fast drive) + inputModifier += " -readrate 10"; + } var flags = new List<string>(); if (state.IgnoreInputDts) @@ -6464,7 +6668,7 @@ namespace MediaBrowser.Controller.MediaEncoding while (shiftAudioCodecs.Contains(audioCodecs[0], StringComparison.OrdinalIgnoreCase)) { - var removed = shiftAudioCodecs[0]; + var removed = audioCodecs[0]; audioCodecs.RemoveAt(0); audioCodecs.Add(removed); } @@ -6498,7 +6702,7 @@ namespace MediaBrowser.Controller.MediaEncoding while (shiftVideoCodecs.Contains(videoCodecs[0], StringComparison.OrdinalIgnoreCase)) { - var removed = shiftVideoCodecs[0]; + var removed = videoCodecs[0]; videoCodecs.RemoveAt(0); videoCodecs.Add(removed); } |