diff options
Diffstat (limited to 'MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs')
| -rw-r--r-- | MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs | 5706 |
1 files changed, 4046 insertions, 1660 deletions
diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 26b0bc3de..e619e690d 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -7,29 +7,49 @@ using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; -using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; using System.Threading; using Jellyfin.Data.Enums; +using Jellyfin.Extensions; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Extensions; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; -using MediaBrowser.Model.IO; using MediaBrowser.Model.MediaInfo; using Microsoft.Extensions.Configuration; namespace MediaBrowser.Controller.MediaEncoding { - public class EncodingHelper + public partial class EncodingHelper { - private static readonly CultureInfo _usCulture = new CultureInfo("en-US"); - + private const string QsvAlias = "qs"; + private const string VaapiAlias = "va"; + private const string D3d11vaAlias = "dx11"; + private const string VideotoolboxAlias = "vt"; + private const string OpenclAlias = "ocl"; + private const string CudaAlias = "cu"; + private const string DrmAlias = "dr"; + private const string VulkanAlias = "vk"; + private readonly IApplicationPaths _appPaths; private readonly IMediaEncoder _mediaEncoder; private readonly ISubtitleEncoder _subtitleEncoder; + private readonly IConfiguration _config; + private readonly IConfigurationManager _configurationManager; + + // i915 hang was fixed by linux 6.2 (3f882f2) + private readonly Version _minKerneli915Hang = new Version(5, 18); + private readonly Version _maxKerneli915Hang = new Version(6, 1, 3); + private readonly Version _minFixedKernel60i915Hang = new Version(6, 0, 18); - private static readonly string[] _videoProfiles = new[] + private readonly Version _minFFmpegImplictHwaccel = new Version(6, 0); + private readonly Version _minFFmpegHwaUnsafeOutput = new Version(6, 0); + private readonly Version _minFFmpegOclCuTonemapMode = new Version(5, 1, 3); + private readonly Version _minFFmpegSvtAv1Params = new Version(5, 1); + + private static readonly string[] _videoProfilesH264 = new[] { "ConstrainedBaseline", "Baseline", @@ -37,24 +57,85 @@ namespace MediaBrowser.Controller.MediaEncoding "Main", "High", "ProgressiveHigh", - "ConstrainedHigh" + "ConstrainedHigh", + "High10" + }; + + private static readonly string[] _videoProfilesH265 = new[] + { + "Main", + "Main10" + }; + + private static readonly string[] _videoProfilesAv1 = new[] + { + "Main", + "High", + "Professional", + }; + + private static readonly HashSet<string> _mp4ContainerNames = new(StringComparer.OrdinalIgnoreCase) + { + "mp4", + "m4a", + "m4p", + "m4b", + "m4r", + "m4v", + }; + + // Set max transcoding channels for encoders that can't handle more than a set amount of channels + // AAC, FLAC, ALAC, libopus, libvorbis encoders all support at least 8 channels + private static readonly Dictionary<string, int> _audioTranscodeChannelLookup = new(StringComparer.OrdinalIgnoreCase) + { + { "wmav2", 2 }, + { "libmp3lame", 2 }, + { "libfdk_aac", 6 }, + { "aac_at", 6 }, + { "ac3", 6 }, + { "eac3", 6 }, + { "dca", 6 }, + { "mlp", 6 }, + { "truehd", 6 }, + }; + + public static readonly string[] LosslessAudioCodecs = new string[] + { + "alac", + "ape", + "flac", + "mlp", + "truehd", + "wavpack" }; public EncodingHelper( + IApplicationPaths appPaths, IMediaEncoder mediaEncoder, - ISubtitleEncoder subtitleEncoder) + ISubtitleEncoder subtitleEncoder, + IConfiguration config, + IConfigurationManager configurationManager) { + _appPaths = appPaths; _mediaEncoder = mediaEncoder; _subtitleEncoder = subtitleEncoder; + _config = config; + _configurationManager = configurationManager; } + [GeneratedRegex(@"\s+")] + private static partial Regex WhiteSpaceRegex(); + public string GetH264Encoder(EncodingJobInfo state, EncodingOptions encodingOptions) - => GetH264OrH265Encoder("libx264", "h264", state, encodingOptions); + => GetH26xOrAv1Encoder("libx264", "h264", state, encodingOptions); public string GetH265Encoder(EncodingJobInfo state, EncodingOptions encodingOptions) - => GetH264OrH265Encoder("libx265", "hevc", state, encodingOptions); + => GetH26xOrAv1Encoder("libx265", "hevc", state, encodingOptions); - private string GetH264OrH265Encoder(string defaultEncoder, string hwEncoder, EncodingJobInfo state, EncodingOptions encodingOptions) + public string GetAv1Encoder(EncodingJobInfo state, EncodingOptions encodingOptions) + => GetH26xOrAv1Encoder("libsvtav1", "av1", state, encodingOptions); + + private string GetH26xOrAv1Encoder(string defaultEncoder, string hwEncoder, EncodingJobInfo state, EncodingOptions encodingOptions) { // Only use alternative encoders for video files. // When using concat with folder rips, if the mfx session fails to initialize, ffmpeg will be stuck retrying and will not exit gracefully @@ -65,27 +146,20 @@ namespace MediaBrowser.Controller.MediaEncoding var codecMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) { - { "qsv", hwEncoder + "_qsv" }, - { hwEncoder + "_qsv", hwEncoder + "_qsv" }, - { "nvenc", hwEncoder + "_nvenc" }, { "amf", hwEncoder + "_amf" }, - { "omx", hwEncoder + "_omx" }, - { hwEncoder + "_v4l2m2m", hwEncoder + "_v4l2m2m" }, - { "mediacodec", hwEncoder + "_mediacodec" }, + { "nvenc", hwEncoder + "_nvenc" }, + { "qsv", hwEncoder + "_qsv" }, { "vaapi", hwEncoder + "_vaapi" }, - { "videotoolbox", hwEncoder + "_videotoolbox" } + { "videotoolbox", hwEncoder + "_videotoolbox" }, + { "v4l2m2m", hwEncoder + "_v4l2m2m" }, }; if (!string.IsNullOrEmpty(hwType) && encodingOptions.EnableHardwareEncoding - && codecMap.ContainsKey(hwType)) + && codecMap.TryGetValue(hwType, out var preferredEncoder) + && _mediaEncoder.SupportsEncoder(preferredEncoder)) { - var preferredEncoder = codecMap[hwType]; - - if (_mediaEncoder.SupportsEncoder(preferredEncoder)) - { - return preferredEncoder; - } + return preferredEncoder; } } @@ -94,11 +168,9 @@ namespace MediaBrowser.Controller.MediaEncoding private bool IsVaapiSupported(EncodingJobInfo state) { - var videoStream = state.VideoStream; - // vaapi will throw an error with this input // [vaapi @ 0x7faed8000960] No VAAPI support for codec mpeg4 profile -99. - if (string.Equals(videoStream?.Codec, "mpeg4", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(state.VideoStream?.Codec, "mpeg4", StringComparison.OrdinalIgnoreCase)) { return false; } @@ -106,68 +178,116 @@ namespace MediaBrowser.Controller.MediaEncoding return _mediaEncoder.SupportsHwaccel("vaapi"); } - private bool IsCudaSupported() + private bool IsVaapiFullSupported() + { + return _mediaEncoder.SupportsHwaccel("drm") + && _mediaEncoder.SupportsHwaccel("vaapi") + && _mediaEncoder.SupportsFilter("scale_vaapi") + && _mediaEncoder.SupportsFilter("deinterlace_vaapi") + && _mediaEncoder.SupportsFilter("tonemap_vaapi") + && _mediaEncoder.SupportsFilter("procamp_vaapi") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVaapiFrameSync) + && _mediaEncoder.SupportsFilter("hwupload_vaapi"); + } + + private bool IsOpenclFullSupported() + { + return _mediaEncoder.SupportsHwaccel("opencl") + && _mediaEncoder.SupportsFilter("scale_opencl") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapOpenclBt2390) + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayOpenclFrameSync); + } + + private bool IsCudaFullSupported() { return _mediaEncoder.SupportsHwaccel("cuda") - && _mediaEncoder.SupportsFilter("scale_cuda", null) - && _mediaEncoder.SupportsFilter("yadif_cuda", null); + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.ScaleCudaFormat) + && _mediaEncoder.SupportsFilter("yadif_cuda") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapCudaName) + && _mediaEncoder.SupportsFilter("overlay_cuda") + && _mediaEncoder.SupportsFilter("hwupload_cuda"); } - private bool IsTonemappingSupported(EncodingJobInfo state, EncodingOptions options) + private bool IsVulkanFullSupported() { - var videoStream = state.VideoStream; - return IsColorDepth10(state) - && _mediaEncoder.SupportsHwaccel("opencl") - && options.EnableTonemapping - && string.Equals(videoStream.VideoRange, "HDR", StringComparison.OrdinalIgnoreCase); + return _mediaEncoder.SupportsHwaccel("vulkan") + && _mediaEncoder.SupportsFilter("libplacebo") + && _mediaEncoder.SupportsFilter("scale_vulkan") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync); } - private bool IsVppTonemappingSupported(EncodingJobInfo state, EncodingOptions options) + private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { - var videoStream = state.VideoStream; - if (videoStream == null) + if (state.VideoStream is null + || !options.EnableTonemapping + || GetVideoColorBitDepth(state) != 10) { - // Remote stream doesn't have media info, disable vpp tonemapping. return false; } - var codec = videoStream.Codec; - if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) + && state.VideoStream.VideoRange == VideoRange.HDR + && state.VideoStream.VideoRangeType == VideoRangeType.DOVI) + { + // Only native SW decoder and HW accelerator can parse dovi rpu. + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder; + } + + return state.VideoStream.VideoRange == VideoRange.HDR + && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 + || state.VideoStream.VideoRangeType == VideoRangeType.HLG); + } + + private bool IsVulkanHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) + { + if (state.VideoStream is null) { - // Limited to HEVC for now since the filter doesn't accept master data from VP9. - return IsColorDepth10(state) - && string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - && _mediaEncoder.SupportsHwaccel("vaapi") - && options.EnableVppTonemapping - && string.Equals(videoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase); + return false; } - // Hybrid VPP tonemapping for QSV with VAAPI - var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); - if (isLinux && string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + // libplacebo has partial Dolby Vision to SDR tonemapping support. + return options.EnableTonemapping + && state.VideoStream.VideoRange == VideoRange.HDR + && GetVideoColorBitDepth(state) == 10; + } + + private bool IsVaapiVppTonemapAvailable(EncodingJobInfo state, EncodingOptions options) + { + if (state.VideoStream is null + || !options.EnableVppTonemapping + || GetVideoColorBitDepth(state) != 10) { - // Limited to HEVC for now since the filter doesn't accept master data from VP9. - return IsColorDepth10(state) - && string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - && _mediaEncoder.SupportsHwaccel("vaapi") - && _mediaEncoder.SupportsHwaccel("qsv") - && options.EnableVppTonemapping - && string.Equals(videoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase); + return false; } // Native VPP tonemapping may come to QSV in the future. - return false; + + return state.VideoStream.VideoRange == VideoRange.HDR + && state.VideoStream.VideoRangeType == VideoRangeType.HDR10; } /// <summary> /// Gets the name of the output video codec. /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="encodingOptions">Encoding options.</param> + /// <returns>Encoder string.</returns> public string GetVideoEncoder(EncodingJobInfo state, EncodingOptions encodingOptions) { var codec = state.OutputVideoCodec; if (!string.IsNullOrEmpty(codec)) { + if (string.Equals(codec, "av1", StringComparison.OrdinalIgnoreCase)) + { + return GetAv1Encoder(state, encodingOptions); + } + if (string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)) { @@ -179,11 +299,17 @@ namespace MediaBrowser.Controller.MediaEncoding return GetH264Encoder(state, encodingOptions); } - if (string.Equals(codec, "vpx", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(codec, "vp8", StringComparison.OrdinalIgnoreCase) + || string.Equals(codec, "vpx", StringComparison.OrdinalIgnoreCase)) { return "libvpx"; } + if (string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase)) + { + return "libvpx-vp9"; + } + if (string.Equals(codec, "wmv", StringComparison.OrdinalIgnoreCase)) { return "wmv2"; @@ -215,6 +341,21 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Empty; } + /// <summary> + /// Gets the referer param. + /// </summary> + /// <param name="state">The state.</param> + /// <returns>System.String.</returns> + public string GetRefererParam(EncodingJobInfo state) + { + if (state.RemoteHttpHeaders.TryGetValue("Referer", out string referer)) + { + return "-referer \"" + referer + "\""; + } + + return string.Empty; + } + public static string GetInputFormat(string container) { if (string.IsNullOrEmpty(container)) @@ -316,6 +457,11 @@ namespace MediaBrowser.Controller.MediaEncoding return container; } + /// <summary> + /// Gets decoder from a codec. + /// </summary> + /// <param name="codec">Codec to use.</param> + /// <returns>Decoder string.</returns> public string GetDecoderFromCodec(string codec) { // For these need to find out the ffmpeg names @@ -345,6 +491,8 @@ namespace MediaBrowser.Controller.MediaEncoding /// <summary> /// Infers the audio codec based on the url. /// </summary> + /// <param name="container">Container to use.</param> + /// <returns>Codec string.</returns> public string InferAudioCodec(string container) { var ext = "." + (container ?? string.Empty); @@ -408,7 +556,8 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals(ext, ".webm", StringComparison.OrdinalIgnoreCase)) { - return "vpx"; + // TODO: this may not always mean VP8, as the codec ages + return "vp8"; } if (string.Equals(ext, ".ogg", StringComparison.OrdinalIgnoreCase) || string.Equals(ext, ".ogv", StringComparison.OrdinalIgnoreCase)) @@ -424,18 +573,36 @@ namespace MediaBrowser.Controller.MediaEncoding return "copy"; } - public int GetVideoProfileScore(string profile) + public int GetVideoProfileScore(string videoCodec, string videoProfile) { // strip spaces because they may be stripped out on the query string - profile = profile.Replace(" ", string.Empty, StringComparison.Ordinal); - return Array.FindIndex(_videoProfiles, x => string.Equals(x, profile, StringComparison.OrdinalIgnoreCase)); + string profile = videoProfile.Replace(" ", string.Empty, StringComparison.Ordinal); + if (string.Equals("h264", videoCodec, StringComparison.OrdinalIgnoreCase)) + { + return Array.FindIndex(_videoProfilesH264, x => string.Equals(x, profile, StringComparison.OrdinalIgnoreCase)); + } + + if (string.Equals("hevc", videoCodec, StringComparison.OrdinalIgnoreCase)) + { + return Array.FindIndex(_videoProfilesH265, x => string.Equals(x, profile, StringComparison.OrdinalIgnoreCase)); + } + + if (string.Equals("av1", videoCodec, StringComparison.OrdinalIgnoreCase)) + { + return Array.FindIndex(_videoProfilesAv1, x => string.Equals(x, profile, StringComparison.OrdinalIgnoreCase)); + } + + return -1; } public string GetInputPathArgument(EncodingJobInfo state) { - var mediaPath = state.MediaPath ?? string.Empty; - - return _mediaEncoder.GetInputArgument(mediaPath, state.MediaSource); + return state.MediaSource.VideoType switch + { + VideoType.Dvd => _mediaEncoder.GetInputArgument(_mediaEncoder.GetPrimaryPlaylistVobFiles(state.MediaPath, null).ToList(), state.MediaSource), + VideoType.BluRay => _mediaEncoder.GetInputArgument(_mediaEncoder.GetPrimaryPlaylistM2tsFiles(state.MediaPath).ToList(), state.MediaSource), + _ => _mediaEncoder.GetInputArgument(state.MediaPath, state.MediaSource) + }; } /// <summary> @@ -449,6 +616,12 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals(codec, "aac", StringComparison.OrdinalIgnoreCase)) { + // Use Apple's aac encoder if available as it provides best audio quality + if (_mediaEncoder.SupportsEncoder("aac_at")) + { + return "aac_at"; + } + // Use libfdk_aac for better audio quality if using custom build of FFmpeg which has fdk_aac support if (_mediaEncoder.SupportsEncoder("libfdk_aac")) { @@ -480,158 +653,436 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals(codec, "flac", StringComparison.OrdinalIgnoreCase)) { - // flac is experimental in mp4 muxer - return "flac -strict -2"; + return "flac"; + } + + if (string.Equals(codec, "dts", StringComparison.OrdinalIgnoreCase)) + { + return "dca"; } return codec.ToLowerInvariant(); } + private string GetVideoToolboxDeviceArgs(string alias) + { + alias ??= VideotoolboxAlias; + + // device selection in vt is not supported. + return " -init_hw_device videotoolbox=" + alias; + } + + private string GetCudaDeviceArgs(int deviceIndex, string alias) + { + alias ??= CudaAlias; + deviceIndex = deviceIndex >= 0 + ? deviceIndex + : 0; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device cuda={0}:{1}", + alias, + deviceIndex); + } + + private string GetVulkanDeviceArgs(int deviceIndex, string deviceName, string srcDeviceAlias, string alias) + { + alias ??= VulkanAlias; + deviceIndex = deviceIndex >= 0 + ? deviceIndex + : 0; + var vendorOpts = string.IsNullOrEmpty(deviceName) + ? ":" + deviceIndex + : ":" + "\"" + deviceName + "\""; + var options = string.IsNullOrEmpty(srcDeviceAlias) + ? vendorOpts + : "@" + srcDeviceAlias; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device vulkan={0}{1}", + alias, + options); + } + + private string GetOpenclDeviceArgs(int deviceIndex, string deviceVendorName, string srcDeviceAlias, string alias) + { + alias ??= OpenclAlias; + deviceIndex = deviceIndex >= 0 + ? deviceIndex + : 0; + var vendorOpts = string.IsNullOrEmpty(deviceVendorName) + ? ":0.0" + : ":." + deviceIndex + ",device_vendor=\"" + deviceVendorName + "\""; + var options = string.IsNullOrEmpty(srcDeviceAlias) + ? vendorOpts + : "@" + srcDeviceAlias; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device opencl={0}{1}", + alias, + options); + } + + private string GetD3d11vaDeviceArgs(int deviceIndex, string deviceVendorId, string alias) + { + alias ??= D3d11vaAlias; + deviceIndex = deviceIndex >= 0 ? deviceIndex : 0; + var options = string.IsNullOrEmpty(deviceVendorId) + ? deviceIndex.ToString(CultureInfo.InvariantCulture) + : ",vendor=" + deviceVendorId; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device d3d11va={0}:{1}", + alias, + options); + } + + private string GetVaapiDeviceArgs(string renderNodePath, string driver, string kernelDriver, string srcDeviceAlias, string alias) + { + alias ??= VaapiAlias; + renderNodePath = renderNodePath ?? "/dev/dri/renderD128"; + var driverOpts = string.IsNullOrEmpty(driver) + ? ":" + renderNodePath + : ":,driver=" + driver + (string.IsNullOrEmpty(kernelDriver) ? string.Empty : ",kernel_driver=" + kernelDriver); + var options = string.IsNullOrEmpty(srcDeviceAlias) + ? driverOpts + : "@" + srcDeviceAlias; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device vaapi={0}{1}", + alias, + options); + } + + private string GetDrmDeviceArgs(string renderNodePath, string alias) + { + alias ??= DrmAlias; + renderNodePath = renderNodePath ?? "/dev/dri/renderD128"; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device drm={0}:{1}", + alias, + renderNodePath); + } + + private string GetQsvDeviceArgs(string alias) + { + var arg = " -init_hw_device qsv=" + (alias ?? QsvAlias); + if (OperatingSystem.IsLinux()) + { + // derive qsv from vaapi device + return GetVaapiDeviceArgs(null, "iHD", "i915", null, VaapiAlias) + arg + "@" + VaapiAlias; + } + + if (OperatingSystem.IsWindows()) + { + // derive qsv from d3d11va device + return GetD3d11vaDeviceArgs(0, "0x8086", D3d11vaAlias) + arg + "@" + D3d11vaAlias; + } + + return null; + } + + private string GetFilterHwDeviceArgs(string alias) + { + return string.IsNullOrEmpty(alias) + ? string.Empty + : " -filter_hw_device " + alias; + } + + public string GetGraphicalSubCanvasSize(EncodingJobInfo state) + { + // DVBSUB and DVDSUB use the fixed canvas size 720x576 + if (state.SubtitleStream is not null + && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode + && !state.SubtitleStream.IsTextSubtitleStream + && !string.Equals(state.SubtitleStream.Codec, "DVBSUB", StringComparison.OrdinalIgnoreCase) + && !string.Equals(state.SubtitleStream.Codec, "DVDSUB", StringComparison.OrdinalIgnoreCase)) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + + // setup a relative small canvas_size for overlay_qsv/vaapi to reduce transfer overhead + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, 1080); + + if (overlayW.HasValue && overlayH.HasValue) + { + return string.Format( + CultureInfo.InvariantCulture, + " -canvas_size {0}x{1}", + overlayW.Value, + overlayH.Value); + } + } + + return string.Empty; + } + /// <summary> - /// Gets the input argument. + /// Gets the input video hwaccel argument. /// </summary> - public string GetInputArgument(EncodingJobInfo state, EncodingOptions encodingOptions) + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <returns>Input video hwaccel arguments.</returns> + public string GetInputVideoHwaccelArgs(EncodingJobInfo state, EncodingOptions options) { - var arg = new StringBuilder(); - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions) ?? string.Empty; - var outputVideoCodec = GetVideoEncoder(state, encodingOptions) ?? string.Empty; - var isSwDecoder = string.IsNullOrEmpty(videoDecoder); - var isD3d11vaDecoder = videoDecoder.IndexOf("d3d11va", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiDecoder = videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvDecoder = videoDecoder.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvEncoder = outputVideoCodec.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - var isCuvidHevcDecoder = videoDecoder.Contains("hevc_cuvid", StringComparison.OrdinalIgnoreCase); - var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); - var isMacOS = RuntimeInformation.IsOSPlatform(OSPlatform.OSX); - var isTonemappingSupported = IsTonemappingSupported(state, encodingOptions); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, encodingOptions); - - if (!IsCopyCodec(outputVideoCodec)) - { - if (state.IsVideoRequest - && _mediaEncoder.SupportsHwaccel("vaapi") - && string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) - { - if (isVaapiDecoder) + if (!state.IsVideoRequest) + { + return string.Empty; + } + + var vidEncoder = GetVideoEncoder(state, options) ?? string.Empty; + if (IsCopyCodec(vidEncoder)) + { + return string.Empty; + } + + var args = new StringBuilder(); + var isWindows = OperatingSystem.IsWindows(); + var isLinux = OperatingSystem.IsLinux(); + var isMacOS = OperatingSystem.IsMacOS(); + var optHwaccelType = options.HardwareAccelerationType; + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isHwTonemapAvailable = IsHwTonemapAvailable(state, options); + + if (string.Equals(optHwaccelType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + if (!isLinux || !_mediaEncoder.SupportsHwaccel("vaapi")) + { + return string.Empty; + } + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + if (!isVaapiDecoder && !isVaapiEncoder) + { + return string.Empty; + } + + if (_mediaEncoder.IsVaapiDeviceInteliHD) + { + args.Append(GetVaapiDeviceArgs(null, "iHD", null, null, VaapiAlias)); + } + else if (_mediaEncoder.IsVaapiDeviceInteli965) + { + // Only override i965 since it has lower priority than iHD in libva lookup. + Environment.SetEnvironmentVariable("LIBVA_DRIVER_NAME", "i965"); + Environment.SetEnvironmentVariable("LIBVA_DRIVER_NAME_JELLYFIN", "i965"); + args.Append(GetVaapiDeviceArgs(null, "i965", null, null, VaapiAlias)); + } + + var filterDevArgs = string.Empty; + var doOclTonemap = isHwTonemapAvailable && IsOpenclFullSupported(); + + if (_mediaEncoder.IsVaapiDeviceInteliHD || _mediaEncoder.IsVaapiDeviceInteli965) + { + if (doOclTonemap && !isVaapiDecoder) { - if (isTonemappingSupported && !isVppTonemappingSupported) - { - arg.Append("-init_hw_device vaapi=va:") - .Append(encodingOptions.VaapiDevice) - .Append(' ') - .Append("-init_hw_device opencl=ocl@va ") - .Append("-hwaccel_device va ") - .Append("-hwaccel_output_format vaapi ") - .Append("-filter_hw_device ocl "); - } - else - { - arg.Append("-hwaccel_output_format vaapi ") - .Append("-vaapi_device ") - .Append(encodingOptions.VaapiDevice) - .Append(' '); - } + args.Append(GetOpenclDeviceArgs(0, null, VaapiAlias, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); } - else if (!isVaapiDecoder && isVaapiEncoder) + } + else if (_mediaEncoder.IsVaapiDeviceAmd) + { + // Disable AMD EFC feature since it's still unstable in upstream Mesa. + Environment.SetEnvironmentVariable("AMD_DEBUG", "noefc"); + + if (IsVulkanFullSupported() + && _mediaEncoder.IsVaapiDeviceSupportVulkanDrmInterop) { - arg.Append("-vaapi_device ") - .Append(encodingOptions.VaapiDevice) - .Append(' '); + args.Append(GetDrmDeviceArgs(options.VaapiDevice, DrmAlias)); + args.Append(GetVaapiDeviceArgs(null, null, null, DrmAlias, VaapiAlias)); + args.Append(GetVulkanDeviceArgs(0, null, DrmAlias, VulkanAlias)); + + // libplacebo wants an explicitly set vulkan filter device. + filterDevArgs = GetFilterHwDeviceArgs(VulkanAlias); + } + else + { + args.Append(GetVaapiDeviceArgs(options.VaapiDevice, null, null, null, VaapiAlias)); + filterDevArgs = GetFilterHwDeviceArgs(VaapiAlias); + + if (doOclTonemap) + { + // ROCm/ROCr OpenCL runtime + args.Append(GetOpenclDeviceArgs(0, "Advanced Micro Devices", null, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); + } } + } + else if (doOclTonemap) + { + args.Append(GetOpenclDeviceArgs(0, null, null, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); + } - arg.Append("-autorotate 0 "); + args.Append(filterDevArgs); + } + else if (string.Equals(optHwaccelType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + if ((!isLinux && !isWindows) || !_mediaEncoder.SupportsHwaccel("qsv")) + { + return string.Empty; } - if (state.IsVideoRequest - && string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isQsvDecoder = vidDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvEncoder = vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isQsvDecoder || isVaapiDecoder || isD3d11vaDecoder; + if (!isHwDecoder && !isQsvEncoder) { - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + return string.Empty; + } - if (isQsvEncoder) + args.Append(GetQsvDeviceArgs(QsvAlias)); + var filterDevArgs = GetFilterHwDeviceArgs(QsvAlias); + // child device used by qsv. + if (_mediaEncoder.SupportsHwaccel("vaapi") || _mediaEncoder.SupportsHwaccel("d3d11va")) + { + if (isHwTonemapAvailable && IsOpenclFullSupported()) { - if (isQsvDecoder) + var srcAlias = isLinux ? VaapiAlias : D3d11vaAlias; + args.Append(GetOpenclDeviceArgs(0, null, srcAlias, OpenclAlias)); + if (!isHwDecoder) { - if (isLinux) - { - if (hasGraphicalSubs) - { - arg.Append("-init_hw_device qsv=hw -filter_hw_device hw "); - } - else - { - arg.Append("-hwaccel qsv "); - } - } - - if (isWindows) - { - arg.Append("-hwaccel qsv "); - } + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); } + } + } - // While using SW decoder - else if (isSwDecoder) - { - arg.Append("-init_hw_device qsv=hw -filter_hw_device hw "); - } + args.Append(filterDevArgs); + } + else if (string.Equals(optHwaccelType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + if ((!isLinux && !isWindows) || !IsCudaFullSupported()) + { + return string.Empty; + } - // Hybrid VPP tonemapping with VAAPI - else if (isVaapiDecoder && isVppTonemappingSupported) - { - arg.Append("-init_hw_device vaapi=va:") - .Append(encodingOptions.VaapiDevice) - .Append(' ') - .Append("-init_hw_device qsv@va ") - .Append("-hwaccel_output_format vaapi "); - } + var isCuvidDecoder = vidDecoder.Contains("cuvid", StringComparison.OrdinalIgnoreCase); + var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); + var isNvencEncoder = vidEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isNvdecDecoder || isCuvidDecoder; + if (!isHwDecoder && !isNvencEncoder) + { + return string.Empty; + } - arg.Append("-autorotate 0 "); - } + args.Append(GetCudaDeviceArgs(0, CudaAlias)) + .Append(GetFilterHwDeviceArgs(CudaAlias)); + } + else if (string.Equals(optHwaccelType, "amf", StringComparison.OrdinalIgnoreCase)) + { + if (!isWindows || !_mediaEncoder.SupportsHwaccel("d3d11va")) + { + return string.Empty; } - if (state.IsVideoRequest - && string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) - && isNvdecDecoder) + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isAmfEncoder = vidEncoder.Contains("amf", StringComparison.OrdinalIgnoreCase); + if (!isD3d11vaDecoder && !isAmfEncoder) { - // Fix for 'No decoder surfaces left' error. https://trac.ffmpeg.org/ticket/7562 - arg.Append("-hwaccel_output_format cuda -extra_hw_frames 3 -autorotate 0 "); + return string.Empty; } - if (state.IsVideoRequest - && ((string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) - && (isNvdecDecoder || isCuvidHevcDecoder || isSwDecoder)) - || (string.Equals(encodingOptions.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase) - && (isD3d11vaDecoder || isSwDecoder)))) + // no dxva video processor hw filter. + args.Append(GetD3d11vaDeviceArgs(0, "0x1002", D3d11vaAlias)); + var filterDevArgs = string.Empty; + if (IsOpenclFullSupported()) { - if (isTonemappingSupported) - { - arg.Append("-init_hw_device opencl=ocl:") - .Append(encodingOptions.OpenclDevice) - .Append(' ') - .Append("-filter_hw_device ocl "); - } + args.Append(GetOpenclDeviceArgs(0, null, D3d11vaAlias, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); + } + + args.Append(filterDevArgs); + } + else if (string.Equals(optHwaccelType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + { + if (!isMacOS || !_mediaEncoder.SupportsHwaccel("videotoolbox")) + { + return string.Empty; } - if (state.IsVideoRequest - && string.Equals(encodingOptions.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + var isVideotoolboxDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + var isVideotoolboxEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + if (!isVideotoolboxDecoder && !isVideotoolboxEncoder) { - arg.Append("-hwaccel videotoolbox "); + return string.Empty; } + + // no videotoolbox hw filter. + args.Append(GetVideoToolboxDeviceArgs(VideotoolboxAlias)); + } + + if (!string.IsNullOrEmpty(vidDecoder)) + { + args.Append(vidDecoder); } - arg.Append("-i ") - .Append(GetInputPathArgument(state)); + // hw transpose filters should be added manually. + args.Append(" -autorotate 0"); + + return args.ToString().Trim(); + } - if (state.SubtitleStream != null + /// <summary> + /// Gets the input argument. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="segmentContainer">Segment Container.</param> + /// <returns>Input arguments.</returns> + public string GetInputArgument(EncodingJobInfo state, EncodingOptions options, string segmentContainer) + { + var arg = new StringBuilder(); + var inputVidHwaccelArgs = GetInputVideoHwaccelArgs(state, options); + + if (!string.IsNullOrEmpty(inputVidHwaccelArgs)) + { + arg.Append(inputVidHwaccelArgs); + } + + var canvasArgs = GetGraphicalSubCanvasSize(state); + if (!string.IsNullOrEmpty(canvasArgs)) + { + arg.Append(canvasArgs); + } + + if (state.MediaSource.VideoType == VideoType.Dvd || state.MediaSource.VideoType == VideoType.BluRay) + { + var tmpConcatPath = Path.Join(_configurationManager.GetTranscodePath(), state.MediaSource.Id + ".concat"); + _mediaEncoder.GenerateConcatConfig(state.MediaSource, tmpConcatPath); + arg.Append(" -f concat -safe 0 -i ") + .Append(tmpConcatPath); + } + else + { + arg.Append(" -i ") + .Append(GetInputPathArgument(state)); + } + + // sub2video for external graphical subtitles + if (state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode - && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream) + && !state.SubtitleStream.IsTextSubtitleStream + && state.SubtitleStream.IsExternal) { var subtitlePath = state.SubtitleStream.Path; + var subtitleExtension = Path.GetExtension(subtitlePath); - if (string.Equals(Path.GetExtension(subtitlePath), ".sub", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(subtitleExtension, ".sub", StringComparison.OrdinalIgnoreCase) + || string.Equals(subtitleExtension, ".sup", StringComparison.OrdinalIgnoreCase)) { var idxFile = Path.ChangeExtension(subtitlePath, ".idx"); if (File.Exists(idxFile)) @@ -640,7 +1091,38 @@ namespace MediaBrowser.Controller.MediaEncoding } } - arg.Append(" -i \"").Append(subtitlePath).Append('\"'); + // Also seek the external subtitles stream. + var seekSubParam = GetFastSeekCommandLineParameter(state, options, segmentContainer); + if (!string.IsNullOrEmpty(seekSubParam)) + { + arg.Append(' ').Append(seekSubParam); + } + + if (!string.IsNullOrEmpty(canvasArgs)) + { + arg.Append(canvasArgs); + } + + arg.Append(" -i file:\"").Append(subtitlePath).Append('\"'); + } + + if (state.AudioStream is not null && state.AudioStream.IsExternal) + { + // Also seek the external audio stream. + var seekAudioParam = GetFastSeekCommandLineParameter(state, options, segmentContainer); + if (!string.IsNullOrEmpty(seekAudioParam)) + { + arg.Append(' ').Append(seekAudioParam); + } + + arg.Append(" -i \"").Append(state.AudioStream.Path).Append('"'); + } + + // Disable auto inserted SW scaler for HW decoders in case of changed resolution. + var isSwDecoder = string.IsNullOrEmpty(GetHardwareVideoDecoder(state, options)); + if (!isSwDecoder && _mediaEncoder.EncoderVersion >= new Version(4, 4)) + { + arg.Append(" -autoscale 0"); } return arg.ToString(); @@ -682,19 +1164,19 @@ namespace MediaBrowser.Controller.MediaEncoding { return "-bsf:v h264_mp4toannexb"; } - else if (IsH265(stream)) + + if (IsH265(stream)) { return "-bsf:v hevc_mp4toannexb"; } - else if (IsAAC(stream)) + + if (IsAAC(stream)) { // Convert adts header(mpegts) to asc header(mp4). return "-bsf:a aac_adtstoasc"; } - else - { - return null; - } + + return null; } public static string GetAudioBitStreamArguments(EncodingJobInfo state, string segmentContainer, string mediaSourceContainer) @@ -705,6 +1187,7 @@ namespace MediaBrowser.Controller.MediaEncoding // Apply aac_adtstoasc bitstream filter when media source is in mpegts. if (string.Equals(segmentFormat, "mp4", StringComparison.OrdinalIgnoreCase) && (string.Equals(mediaSourceContainer, "mpegts", StringComparison.OrdinalIgnoreCase) + || string.Equals(mediaSourceContainer, "aac", StringComparison.OrdinalIgnoreCase) || string.Equals(mediaSourceContainer, "hls", StringComparison.OrdinalIgnoreCase))) { bitStreamArgs = GetBitStreamArgs(state.AudioStream); @@ -726,73 +1209,104 @@ namespace MediaBrowser.Controller.MediaEncoding public string GetVideoBitrateParam(EncodingJobInfo state, string videoCodec) { - var bitrate = state.OutputVideoBitrate; + if (state.OutputVideoBitrate is null) + { + return string.Empty; + } + + int bitrate = state.OutputVideoBitrate.Value; - if (bitrate.HasValue) + // Bit rate under 1000k is not allowed in h264_qsv + if (string.Equals(videoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase)) { - if (string.Equals(videoCodec, "libvpx", StringComparison.OrdinalIgnoreCase)) - { - // When crf is used with vpx, b:v becomes a max rate - // https://trac.ffmpeg.org/wiki/Encode/VP9 - return string.Format( - CultureInfo.InvariantCulture, - " -maxrate:v {0} -bufsize:v {1} -b:v {0}", - bitrate.Value, - bitrate.Value * 2); - } + bitrate = Math.Max(bitrate, 1000); + } - if (string.Equals(videoCodec, "msmpeg4", StringComparison.OrdinalIgnoreCase)) - { - return string.Format( - CultureInfo.InvariantCulture, - " -b:v {0}", - bitrate.Value); - } + // Currently use the same buffer size for all encoders + int bufsize = bitrate * 2; + + if (string.Equals(videoCodec, "libvpx", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "libvpx-vp9", StringComparison.OrdinalIgnoreCase)) + { + // When crf is used with vpx, b:v becomes a max rate + // https://trac.ffmpeg.org/wiki/Encode/VP8 + // https://trac.ffmpeg.org/wiki/Encode/VP9 + return FormattableString.Invariant($" -maxrate:v {bitrate} -bufsize:v {bufsize} -b:v {bitrate}"); + } + + if (string.Equals(videoCodec, "msmpeg4", StringComparison.OrdinalIgnoreCase)) + { + return FormattableString.Invariant($" -b:v {bitrate}"); + } + + if (string.Equals(videoCodec, "libsvtav1", StringComparison.OrdinalIgnoreCase)) + { + return FormattableString.Invariant($" -b:v {bitrate} -bufsize {bufsize}"); + } + + if (string.Equals(videoCodec, "libx264", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "libx265", StringComparison.OrdinalIgnoreCase)) + { + return FormattableString.Invariant($" -maxrate {bitrate} -bufsize {bufsize}"); + } + + if (string.Equals(videoCodec, "h264_amf", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "hevc_amf", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "av1_amf", StringComparison.OrdinalIgnoreCase)) + { + // Override the too high default qmin 18 in transcoding preset + return FormattableString.Invariant($" -rc cbr -qmin 0 -qmax 32 -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); + } - if (string.Equals(videoCodec, "libx264", StringComparison.OrdinalIgnoreCase) || - string.Equals(videoCodec, "libx265", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(videoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "av1_vaapi", StringComparison.OrdinalIgnoreCase)) + { + // VBR in i965 driver may result in pixelated output. + if (_mediaEncoder.IsVaapiDeviceInteli965) { - // h264 - return string.Format( - CultureInfo.InvariantCulture, - " -maxrate {0} -bufsize {1}", - bitrate.Value, - bitrate.Value * 2); + return FormattableString.Invariant($" -rc_mode CBR -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); } - // h264 - return string.Format( - CultureInfo.InvariantCulture, - " -b:v {0} -maxrate {0} -bufsize {1}", - bitrate.Value, - bitrate.Value * 2); + return FormattableString.Invariant($" -rc_mode VBR -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); } - return string.Empty; + return FormattableString.Invariant($" -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); } public static string NormalizeTranscodingLevel(EncodingJobInfo state, string level) { - if (double.TryParse(level, NumberStyles.Any, _usCulture, out double requestLevel)) + if (double.TryParse(level, CultureInfo.InvariantCulture, out double requestLevel)) { - if (string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(state.ActualOutputVideoCodec, "h265", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(state.ActualOutputVideoCodec, "av1", StringComparison.OrdinalIgnoreCase)) + { + // Transcode to level 5.3 (15) and lower for maximum compatibility. + // https://en.wikipedia.org/wiki/AV1#Levels + if (requestLevel < 0 || requestLevel >= 15) + { + return "15"; + } + } + else if (string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.ActualOutputVideoCodec, "h265", StringComparison.OrdinalIgnoreCase)) { // Transcode to level 5.0 and lower for maximum compatibility. // Level 5.0 is suitable for up to 4k 30fps hevc encoding, otherwise let the encoder to handle it. // https://en.wikipedia.org/wiki/High_Efficiency_Video_Coding_tiers_and_levels // MaxLumaSampleRate = 3840*2160*30 = 248832000 < 267386880. - if (requestLevel >= 150) + if (requestLevel < 0 || requestLevel >= 150) { return "150"; } } else if (string.Equals(state.ActualOutputVideoCodec, "h264", StringComparison.OrdinalIgnoreCase)) { - // Clients may direct play higher than level 41, but there's no reason to transcode higher. - if (requestLevel >= 41) + // Transcode to level 5.1 and lower for maximum compatibility. + // h264 4k 30fps requires at least level 5.1 otherwise it will break on safari fmp4. + // https://en.wikipedia.org/wiki/Advanced_Video_Coding#Levels + if (requestLevel < 0 || requestLevel >= 51) { - return "41"; + return "51"; } } } @@ -804,8 +1318,10 @@ namespace MediaBrowser.Controller.MediaEncoding /// Gets the text subtitle param. /// </summary> /// <param name="state">The state.</param> + /// <param name="enableAlpha">Enable alpha processing.</param> + /// <param name="enableSub2video">Enable sub2video mode.</param> /// <returns>System.String.</returns> - public string GetTextSubtitleParam(EncodingJobInfo state) + public string GetTextSubtitlesFilter(EncodingJobInfo state, bool enableAlpha, bool enableSub2video) { var seconds = Math.Round(TimeSpan.FromTicks(state.StartTimeTicks ?? 0).TotalSeconds); @@ -814,37 +1330,26 @@ namespace MediaBrowser.Controller.MediaEncoding ? string.Empty : string.Format(CultureInfo.InvariantCulture, ",setpts=PTS -{0}/TB", seconds); - // TODO - // var fallbackFontPath = Path.Combine(_appPaths.ProgramDataPath, "fonts", "DroidSansFallback.ttf"); - // string fallbackFontParam = string.Empty; + var alphaParam = enableAlpha ? ":alpha=1" : string.Empty; + var sub2videoParam = enableSub2video ? ":sub2video=1" : string.Empty; - // if (!File.Exists(fallbackFontPath)) - // { - // _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(fallbackFontPath)); - // using (var stream = _assemblyInfo.GetManifestResourceStream(GetType(), GetType().Namespace + ".DroidSansFallback.ttf")) - // { - // using (var fileStream = new FileStream(fallbackFontPath, FileMode.Create, FileAccess.Write, FileShare.Read)) - // { - // stream.CopyTo(fileStream); - // } - // } - // } - - // fallbackFontParam = string.Format(CultureInfo.InvariantCulture, ":force_style='FontName=Droid Sans Fallback':fontsdir='{0}'", _mediaEncoder.EscapeSubtitleFilterPath(_fileSystem.GetDirectoryName(fallbackFontPath))); + var fontPath = Path.Combine(_appPaths.CachePath, "attachments", state.MediaSource.Id); + var fontParam = string.Format( + CultureInfo.InvariantCulture, + ":fontsdir='{0}'", + _mediaEncoder.EscapeSubtitleFilterPath(fontPath)); if (state.SubtitleStream.IsExternal) { - var subtitlePath = state.SubtitleStream.Path; - var charsetParam = string.Empty; if (!string.IsNullOrEmpty(state.SubtitleStream.Language)) { var charenc = _subtitleEncoder.GetSubtitleFileCharacterSet( - subtitlePath, - state.SubtitleStream.Language, - state.MediaSource.Protocol, - CancellationToken.None).GetAwaiter().GetResult(); + state.SubtitleStream, + state.SubtitleStream.Language, + state.MediaSource, + CancellationToken.None).GetAwaiter().GetResult(); if (!string.IsNullOrEmpty(charenc)) { @@ -855,10 +1360,12 @@ namespace MediaBrowser.Controller.MediaEncoding // TODO: Perhaps also use original_size=1920x800 ?? return string.Format( CultureInfo.InvariantCulture, - "subtitles=filename='{0}'{1}{2}", - _mediaEncoder.EscapeSubtitleFilterPath(subtitlePath), + "subtitles=f='{0}'{1}{2}{3}{4}{5}", + _mediaEncoder.EscapeSubtitleFilterPath(state.SubtitleStream.Path), charsetParam, - // fallbackFontParam, + alphaParam, + sub2videoParam, + fontParam, setPtsParam); } @@ -866,9 +1373,12 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format( CultureInfo.InvariantCulture, - "subtitles='{0}:si={1}'{2}", + "subtitles=f='{0}':si={1}{2}{3}{4}{5}", _mediaEncoder.EscapeSubtitleFilterPath(mediaPath), - state.InternalSubtitleStreamOffset.ToString(_usCulture), + state.InternalSubtitleStreamOffset.ToString(CultureInfo.InvariantCulture), + alphaParam, + sub2videoParam, + fontParam, // fallbackFontParam, setPtsParam); } @@ -884,7 +1394,7 @@ namespace MediaBrowser.Controller.MediaEncoding var maxrate = request.MaxFramerate; - if (maxrate.HasValue && state.VideoStream != null) + if (maxrate.HasValue && state.VideoStream is not null) { var contentRate = state.VideoStream.AverageFrameRate ?? state.VideoStream.RealFrameRate; @@ -906,22 +1416,11 @@ namespace MediaBrowser.Controller.MediaEncoding { var args = string.Empty; var gopArg = string.Empty; - var keyFrameArg = string.Empty; - if (isEventPlaylist) - { - keyFrameArg = string.Format( - CultureInfo.InvariantCulture, - " -force_key_frames:0 \"expr:gte(t,n_forced*{0})\"", - segmentLength); - } - else if (startNumber.HasValue) - { - keyFrameArg = string.Format( - CultureInfo.InvariantCulture, - " -force_key_frames:0 \"expr:gte(t,{0}+n_forced*{1})\"", - startNumber.Value * segmentLength, - segmentLength); - } + + var keyFrameArg = string.Format( + CultureInfo.InvariantCulture, + " -force_key_frames:0 \"expr:gte(t,n_forced*{0})\"", + segmentLength); var framerate = state.VideoStream?.RealFrameRate; if (framerate.HasValue) @@ -931,10 +1430,9 @@ namespace MediaBrowser.Controller.MediaEncoding // Example: we encoded half of desired length, then codec detected // scene cut and inserted a keyframe; next forced keyframe would // be created outside of segment, which breaks seeking. - // -sc_threshold 0 is used to prevent the hardware encoder from post processing to break the set keyframe. gopArg = string.Format( CultureInfo.InvariantCulture, - " -g:v:0 {0} -keyint_min:v:0 {0} -sc_threshold:v:0 0", + " -g:v:0 {0} -keyint_min:v:0 {0}", Math.Ceiling(segmentLength * framerate.Value)); } @@ -944,20 +1442,37 @@ namespace MediaBrowser.Controller.MediaEncoding || string.Equals(codec, "h264_amf", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "hevc_qsv", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "hevc_amf", StringComparison.OrdinalIgnoreCase)) + || string.Equals(codec, "av1_qsv", StringComparison.OrdinalIgnoreCase) + || string.Equals(codec, "av1_nvenc", StringComparison.OrdinalIgnoreCase) + || string.Equals(codec, "av1_amf", StringComparison.OrdinalIgnoreCase) + || string.Equals(codec, "libsvtav1", StringComparison.OrdinalIgnoreCase)) { args += gopArg; } else if (string.Equals(codec, "libx264", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "libx265", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) + || string.Equals(codec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(codec, "av1_vaapi", StringComparison.OrdinalIgnoreCase)) { - args += " " + keyFrameArg; + args += keyFrameArg; + + // prevent the libx264 from post processing to break the set keyframe. + if (string.Equals(codec, "libx264", StringComparison.OrdinalIgnoreCase)) + { + args += " -sc_threshold:v:0 0"; + } } else { - args += " " + keyFrameArg + gopArg; + args += keyFrameArg + gopArg; + } + + // global_header produced by AMD HEVC VA-API encoder causes non-playable fMP4 on iOS + if (string.Equals(codec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) + && _mediaEncoder.IsVaapiDeviceAmd) + { + args += " -flags:v -global_header"; } return args; @@ -966,58 +1481,87 @@ namespace MediaBrowser.Controller.MediaEncoding /// <summary> /// Gets the video bitrate to specify on the command line. /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="videoEncoder">Video encoder to use.</param> + /// <param name="encodingOptions">Encoding options.</param> + /// <param name="defaultPreset">Default present to use for encoding.</param> + /// <returns>Video bitrate.</returns> public string GetVideoQualityParam(EncodingJobInfo state, string videoEncoder, EncodingOptions encodingOptions, string defaultPreset) { var param = string.Empty; - if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) - { - param += " -pix_fmt yuv420p"; - } - - if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) - { - var videoStream = state.VideoStream; - var isColorDepth10 = IsColorDepth10(state); - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions) ?? string.Empty; - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - - if (!isNvdecDecoder) - { - if (isColorDepth10 - && _mediaEncoder.SupportsHwaccel("opencl") - && encodingOptions.EnableTonemapping - && !string.IsNullOrEmpty(videoStream.VideoRange) - && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase)) - { - param += " -pix_fmt nv12"; - } - else + // Tutorials: Enable Intel GuC / HuC firmware loading for Low Power Encoding. + // https://01.org/group/43/downloads/firmware + // https://wiki.archlinux.org/title/intel_graphics#Enable_GuC_/_HuC_firmware_loading + // Intel Low Power Encoding can save unnecessary CPU-GPU synchronization, + // which will reduce overhead in performance intensive tasks such as 4k transcoding and tonemapping. + var intelLowPowerHwEncoding = false; + + // Workaround for linux 5.18 to 6.1.3 i915 hang at cost of performance. + // https://github.com/intel/media-driver/issues/1456 + var enableWaFori915Hang = false; + + if (string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + var isIntelVaapiDriver = _mediaEncoder.IsVaapiDeviceInteliHD || _mediaEncoder.IsVaapiDeviceInteli965; + + if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerH264HwEncoder && isIntelVaapiDriver; + } + else if (string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerHevcHwEncoder && isIntelVaapiDriver; + } + } + else if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + if (OperatingSystem.IsLinux()) + { + var ver = Environment.OSVersion.Version; + var isFixedKernel60 = ver.Major == 6 && ver.Minor == 0 && ver >= _minFixedKernel60i915Hang; + var isUnaffectedKernel = ver < _minKerneli915Hang || ver > _maxKerneli915Hang; + + if (!(isUnaffectedKernel || isFixedKernel60)) { - param += " -pix_fmt yuv420p"; + var vidDecoder = GetHardwareVideoDecoder(state, encodingOptions) ?? string.Empty; + var isIntelDecoder = vidDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase) + || vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var doOclTonemap = _mediaEncoder.SupportsHwaccel("qsv") + && IsVaapiSupported(state) + && IsOpenclFullSupported() + && !IsVaapiVppTonemapAvailable(state, encodingOptions) + && IsHwTonemapAvailable(state, encodingOptions); + + enableWaFori915Hang = isIntelDecoder && doOclTonemap; } } + + if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerH264HwEncoder; + } + else if (string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerHevcHwEncoder; + } + else + { + enableWaFori915Hang = false; + } } - if (string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase)) + if (intelLowPowerHwEncoding) { - param += " -pix_fmt nv21"; + param += " -low_power 1"; } - var isVc1 = state.VideoStream != null && - string.Equals(state.VideoStream.Codec, "vc1", StringComparison.OrdinalIgnoreCase); + if (enableWaFori915Hang) + { + param += " -async_depth 1"; + } + + var isVc1 = string.Equals(state.VideoStream?.Codec, "vc1", StringComparison.OrdinalIgnoreCase); var isLibX265 = string.Equals(videoEncoder, "libx265", StringComparison.OrdinalIgnoreCase); if (string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase) || isLibX265) @@ -1052,62 +1596,118 @@ namespace MediaBrowser.Controller.MediaEncoding param += " -crf " + defaultCrf; } } + else if (string.Equals(videoEncoder, "libsvtav1", StringComparison.OrdinalIgnoreCase)) + { + // Default to use the recommended preset 10. + // Omit presets < 5, which are too slow for on the fly encoding. + // https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Ffmpeg.md + param += encodingOptions.EncoderPreset switch + { + "veryslow" => " -preset 5", + "slower" => " -preset 6", + "slow" => " -preset 7", + "medium" => " -preset 8", + "fast" => " -preset 9", + "faster" => " -preset 10", + "veryfast" => " -preset 11", + "superfast" => " -preset 12", + "ultrafast" => " -preset 13", + _ => " -preset 10" + }; + } + else if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "av1_vaapi", StringComparison.OrdinalIgnoreCase)) + { + // -compression_level is not reliable on AMD. + if (_mediaEncoder.IsVaapiDeviceInteliHD) + { + param += encodingOptions.EncoderPreset switch + { + "veryslow" => " -compression_level 1", + "slower" => " -compression_level 2", + "slow" => " -compression_level 3", + "medium" => " -compression_level 4", + "fast" => " -compression_level 5", + "faster" => " -compression_level 6", + "veryfast" => " -compression_level 7", + "superfast" => " -compression_level 7", + "ultrafast" => " -compression_level 7", + _ => string.Empty + }; + } + } else if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) // h264 (h264_qsv) - || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) // hevc (hevc_qsv) + || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase) // hevc (hevc_qsv) + || string.Equals(videoEncoder, "av1_qsv", StringComparison.OrdinalIgnoreCase)) // av1 (av1_qsv) { - string[] valid_h264_qsv = { "veryslow", "slower", "slow", "medium", "fast", "faster", "veryfast" }; + string[] valid_presets = { "veryslow", "slower", "slow", "medium", "fast", "faster", "veryfast" }; - if (valid_h264_qsv.Contains(encodingOptions.EncoderPreset, StringComparer.OrdinalIgnoreCase)) + if (valid_presets.Contains(encodingOptions.EncoderPreset, StringComparison.OrdinalIgnoreCase)) { param += " -preset " + encodingOptions.EncoderPreset; } else { - param += " -preset 7"; + param += " -preset veryfast"; } - param += " -look_ahead 0"; + // Only h264_qsv has look_ahead option + if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase)) + { + param += " -look_ahead 0"; + } } else if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) // h264 (h264_nvenc) - || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase)) // hevc (hevc_nvenc) + || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) // hevc (hevc_nvenc) + || string.Equals(videoEncoder, "av1_nvenc", StringComparison.OrdinalIgnoreCase)) // av1 (av1_nvenc) { switch (encodingOptions.EncoderPreset) { case "veryslow": + param += " -preset p7"; + break; - param += " -preset slow"; // lossless is only supported on maxwell and newer(2014+) + case "slower": + param += " -preset p6"; break; case "slow": - case "slower": - param += " -preset slow"; + param += " -preset p5"; break; case "medium": - param += " -preset medium"; + param += " -preset p4"; break; case "fast": + param += " -preset p3"; + break; + case "faster": + param += " -preset p2"; + break; + case "veryfast": case "superfast": case "ultrafast": - param += " -preset fast"; + param += " -preset p1"; break; default: - param += " -preset default"; + param += " -preset p1"; break; } } else if (string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) // h264 (h264_amf) - || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) // hevc (hevc_amf) + || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) // hevc (hevc_amf) + || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase)) // av1 (av1_amf) { switch (encodingOptions.EncoderPreset) { case "veryslow": - case "slow": case "slower": + case "slow": param += " -quality quality"; break; @@ -1128,25 +1728,18 @@ namespace MediaBrowser.Controller.MediaEncoding break; } - var videoStream = state.VideoStream; - var isColorDepth10 = IsColorDepth10(state); - - if (isColorDepth10 - && _mediaEncoder.SupportsHwaccel("opencl") - && encodingOptions.EnableTonemapping - && !string.IsNullOrEmpty(videoStream.VideoRange) - && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase)) { - // Enhance workload when tone mapping with AMF on some APUs - param += " -preanalysis true"; + param += " -header_insertion_mode gop"; } if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) { - param += " -header_insertion_mode gop -gops_per_idr 1"; + param += " -gops_per_idr 1"; } } - else if (string.Equals(videoEncoder, "libvpx", StringComparison.OrdinalIgnoreCase)) // webm + else if (string.Equals(videoEncoder, "libvpx", StringComparison.OrdinalIgnoreCase)) // vp8 { // Values 0-3, 0 being highest quality but slower var profileScore = 0; @@ -1169,11 +1762,60 @@ namespace MediaBrowser.Controller.MediaEncoding param += string.Format( CultureInfo.InvariantCulture, " -speed 16 -quality good -profile:v {0} -slices 8 -crf {1} -qmin {2} -qmax {3}", - profileScore.ToString(_usCulture), + profileScore.ToString(CultureInfo.InvariantCulture), crf, qmin, qmax); } + else if (string.Equals(videoEncoder, "libvpx-vp9", StringComparison.OrdinalIgnoreCase)) // vp9 + { + // When `-deadline` is set to `good` or `best`, `-cpu-used` ranges from 0-5. + // When `-deadline` is set to `realtime`, `-cpu-used` ranges from 0-15. + // Resources: + // * https://trac.ffmpeg.org/wiki/Encode/VP9 + // * https://superuser.com/questions/1586934 + // * https://developers.google.com/media/vp9 + param += encodingOptions.EncoderPreset switch + { + "veryslow" => " -deadline best -cpu-used 0", + "slower" => " -deadline best -cpu-used 2", + "slow" => " -deadline best -cpu-used 3", + "medium" => " -deadline good -cpu-used 0", + "fast" => " -deadline good -cpu-used 1", + "faster" => " -deadline good -cpu-used 2", + "veryfast" => " -deadline good -cpu-used 3", + "superfast" => " -deadline good -cpu-used 4", + "ultrafast" => " -deadline good -cpu-used 5", + _ => " -deadline good -cpu-used 1" + }; + + // TODO: until VP9 gets its own CRF setting, base CRF on H.265. + int h265Crf = encodingOptions.H265Crf; + int defaultVp9Crf = 31; + if (h265Crf >= 0 && h265Crf <= 51) + { + // This conversion factor is chosen to match the default CRF for H.265 to the + // recommended 1080p CRF from Google. The factor also maps the logarithmic CRF + // scale of x265 [0, 51] to that of VP9 [0, 63] relatively well. + + // Resources: + // * https://developers.google.com/media/vp9/settings/vod + const float H265ToVp9CrfConversionFactor = 1.12F; + + var vp9Crf = Convert.ToInt32(h265Crf * H265ToVp9CrfConversionFactor); + + // Encoder allows for CRF values in the range [0, 63]. + vp9Crf = Math.Clamp(vp9Crf, 0, 63); + + param += FormattableString.Invariant($" -crf {vp9Crf}"); + } + else + { + param += FormattableString.Invariant($" -crf {defaultVp9Crf}"); + } + + param += " -row-mt 1 -profile 1"; + } else if (string.Equals(videoEncoder, "mpeg4", StringComparison.OrdinalIgnoreCase)) { param += " -mbd rd -flags +mv4+aic -trellis 2 -cmp 2 -subcmp 2 -bf 2"; @@ -1192,7 +1834,7 @@ namespace MediaBrowser.Controller.MediaEncoding var framerate = GetFramerateParam(state); if (framerate.HasValue) { - param += string.Format(CultureInfo.InvariantCulture, " -r {0}", framerate.Value.ToString(_usCulture)); + param += string.Format(CultureInfo.InvariantCulture, " -r {0}", framerate.Value.ToString(CultureInfo.InvariantCulture)); } var targetVideoCodec = state.ActualOutputVideoCodec; @@ -1203,7 +1845,7 @@ namespace MediaBrowser.Controller.MediaEncoding } var profile = state.GetRequestedProfiles(targetVideoCodec).FirstOrDefault() ?? string.Empty; - profile = Regex.Replace(profile, @"\s+", string.Empty); + profile = WhiteSpaceRegex().Replace(profile, string.Empty); // We only transcode to HEVC 8-bit for now, force Main Profile. if (profile.Contains("main10", StringComparison.OrdinalIgnoreCase) @@ -1225,6 +1867,14 @@ namespace MediaBrowser.Controller.MediaEncoding profile = "high"; } + // We only need Main profile of AV1 encoders. + if (videoEncoder.Contains("av1", StringComparison.OrdinalIgnoreCase) + && (profile.Contains("high", StringComparison.OrdinalIgnoreCase) + || profile.Contains("professional", StringComparison.OrdinalIgnoreCase))) + { + profile = "main"; + } + // h264_vaapi does not support Baseline profile, force Constrained Baseline in this case, // which is compatible (and ugly). if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) @@ -1264,19 +1914,12 @@ namespace MediaBrowser.Controller.MediaEncoding profile = "constrained_high"; } - // Currently hevc_amf only support encoding HEVC Main Profile, otherwise force Main Profile. - if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) - && profile.Contains("main10", StringComparison.OrdinalIgnoreCase)) - { - profile = "main"; - } - if (!string.IsNullOrEmpty(profile)) { - if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase)) + // Currently there's no profile option in av1_nvenc encoder + if (!(string.Equals(videoEncoder, "av1_nvenc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase))) { - // not supported by h264_omx param += " -profile:v:0 " + profile; } } @@ -1287,7 +1930,7 @@ namespace MediaBrowser.Controller.MediaEncoding { level = NormalizeTranscodingLevel(state, level); - // libx264, QSV, AMF, VAAPI can adjust the given level to match the output. + // libx264, QSV, AMF can adjust the given level to match the output. if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) || string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase)) { @@ -1296,24 +1939,48 @@ namespace MediaBrowser.Controller.MediaEncoding else if (string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) { // hevc_qsv use -level 51 instead of -level 153. - if (double.TryParse(level, NumberStyles.Any, _usCulture, out double hevcLevel)) + if (double.TryParse(level, CultureInfo.InvariantCulture, out double hevcLevel)) { param += " -level " + (hevcLevel / 3); } } + else if (string.Equals(videoEncoder, "av1_qsv", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "libsvtav1", StringComparison.OrdinalIgnoreCase)) + { + // libsvtav1 and av1_qsv use -level 60 instead of -level 16 + // https://aomedia.org/av1/specification/annex-a/ + if (int.TryParse(level, NumberStyles.Any, CultureInfo.InvariantCulture, out int av1Level)) + { + var x = 2 + (av1Level >> 2); + var y = av1Level & 3; + var res = (x * 10) + y; + param += " -level " + res; + } + } else if (string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) + || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase)) { param += " -level " + level; } else if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase)) + || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "av1_nvenc", StringComparison.OrdinalIgnoreCase)) { // level option may cause NVENC to fail. // NVENC cannot adjust the given level, just throw an error. } - else if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase) - || !string.Equals(videoEncoder, "libx265", StringComparison.OrdinalIgnoreCase)) + else if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "av1_vaapi", StringComparison.OrdinalIgnoreCase)) + { + // level option may cause corrupted frames on AMD VAAPI. + if (_mediaEncoder.IsVaapiDeviceInteliHD || _mediaEncoder.IsVaapiDeviceInteli965) + { + param += " -level " + level; + } + } + else if (!string.Equals(videoEncoder, "libx265", StringComparison.OrdinalIgnoreCase)) { param += " -level " + level; } @@ -1333,6 +2000,12 @@ namespace MediaBrowser.Controller.MediaEncoding param += " -x265-params:0 no-info=1"; } + if (string.Equals(videoEncoder, "libsvtav1", StringComparison.OrdinalIgnoreCase) + && _mediaEncoder.EncoderVersion >= _minFFmpegSvtAv1Params) + { + param += " -svtav1-params:0 rc=1:tune=0:film-grain=0:enable-overlays=1:enable-tf=0"; + } + return param; } @@ -1361,6 +2034,7 @@ namespace MediaBrowser.Controller.MediaEncoding // Can't stream copy if we're burning in subtitles if (request.SubtitleStreamIndex.HasValue + && request.SubtitleStreamIndex.Value >= 0 && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode) { return false; @@ -1376,7 +2050,8 @@ namespace MediaBrowser.Controller.MediaEncoding // Source and target codecs must match if (string.IsNullOrEmpty(videoStream.Codec) - || !state.SupportedVideoCodecs.Contains(videoStream.Codec, StringComparer.OrdinalIgnoreCase)) + || (state.SupportedVideoCodecs.Length != 0 + && !state.SupportedVideoCodecs.Contains(videoStream.Codec, StringComparison.OrdinalIgnoreCase))) { return false; } @@ -1394,10 +2069,10 @@ namespace MediaBrowser.Controller.MediaEncoding var requestedProfile = requestedProfiles[0]; // strip spaces because they may be stripped out on the query string as well if (!string.IsNullOrEmpty(videoStream.Profile) - && !requestedProfiles.Contains(videoStream.Profile.Replace(" ", string.Empty, StringComparison.Ordinal), StringComparer.OrdinalIgnoreCase)) + && !requestedProfiles.Contains(videoStream.Profile.Replace(" ", string.Empty, StringComparison.Ordinal), StringComparison.OrdinalIgnoreCase)) { - var currentScore = GetVideoProfileScore(videoStream.Profile); - var requestedScore = GetVideoProfileScore(requestedProfile); + var currentScore = GetVideoProfileScore(videoStream.Codec, videoStream.Profile); + var requestedScore = GetVideoProfileScore(videoStream.Codec, requestedProfile); if (currentScore == -1 || currentScore > requestedScore) { @@ -1406,6 +2081,20 @@ namespace MediaBrowser.Controller.MediaEncoding } } + var requestedRangeTypes = state.GetRequestedRangeTypes(videoStream.Codec); + if (requestedRangeTypes.Length > 0) + { + if (videoStream.VideoRangeType == VideoRangeType.Unknown) + { + return false; + } + + if (!requestedRangeTypes.Contains(videoStream.VideoRangeType.ToString(), StringComparison.OrdinalIgnoreCase)) + { + return false; + } + } + // Video width must fall within requested value if (request.MaxWidth.HasValue && (!videoStream.Width.HasValue || videoStream.Width.Value > request.MaxWidth.Value)) @@ -1457,8 +2146,7 @@ namespace MediaBrowser.Controller.MediaEncoding // If a specific level was requested, the source must match or be less than var level = state.GetRequestedLevel(videoStream.Codec); - if (!string.IsNullOrEmpty(level) - && double.TryParse(level, NumberStyles.Any, _usCulture, out var requestLevel)) + if (double.TryParse(level, CultureInfo.InvariantCulture, out var requestLevel)) { if (!videoStream.Level.HasValue) { @@ -1479,7 +2167,7 @@ namespace MediaBrowser.Controller.MediaEncoding return false; } - return request.EnableAutoStreamCopy; + return true; } public bool CanStreamCopyAudio(EncodingJobInfo state, MediaStream audioStream, IEnumerable<string> supportedAudioCodecs) @@ -1501,7 +2189,7 @@ namespace MediaBrowser.Controller.MediaEncoding // Source and target codecs must match if (string.IsNullOrEmpty(audioStream.Codec) - || !supportedAudioCodecs.Contains(audioStream.Codec, StringComparer.OrdinalIgnoreCase)) + || !supportedAudioCodecs.Contains(audioStream.Codec, StringComparison.OrdinalIgnoreCase)) { return false; } @@ -1535,28 +2223,22 @@ namespace MediaBrowser.Controller.MediaEncoding } } - // Video bitrate must fall within requested value - if (request.AudioBitRate.HasValue) + // Audio bitrate must fall within requested value + if (request.AudioBitRate.HasValue + && audioStream.BitRate.HasValue + && audioStream.BitRate.Value > request.AudioBitRate.Value) { - if (!audioStream.BitRate.HasValue || audioStream.BitRate.Value <= 0) - { - return false; - } - - if (audioStream.BitRate.Value > request.AudioBitRate.Value) - { - return false; - } + return false; } return request.EnableAutoStreamCopy; } - public int? GetVideoBitrateParamValue(BaseEncodingJobOptions request, MediaStream videoStream, string outputVideoCodec) + public int GetVideoBitrateParamValue(BaseEncodingJobOptions request, MediaStream videoStream, string outputVideoCodec) { var bitrate = request.VideoBitRate; - if (videoStream != null) + if (videoStream is not null) { var isUpscaling = request.Height.HasValue && videoStream.Height.HasValue @@ -1584,7 +2266,8 @@ namespace MediaBrowser.Controller.MediaEncoding } } - return bitrate; + // Cap the max target bitrate to intMax/2 to satisfy the bufsize=bitrate*2. + return Math.Min(bitrate ?? 0, int.MaxValue / 2); } private int GetMinBitrate(int sourceBitrate, int requestedBitrate) @@ -1606,6 +2289,7 @@ namespace MediaBrowser.Controller.MediaEncoding private static double GetVideoBitrateScaleFactor(string codec) { + // hevc & vp9 - 40% more efficient than h.264 if (string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase)) @@ -1613,6 +2297,12 @@ namespace MediaBrowser.Controller.MediaEncoding return .6; } + // av1 - 50% more efficient than h.264 + if (string.Equals(codec, "av1", StringComparison.OrdinalIgnoreCase)) + { + return .5; + } + return 1; } @@ -1620,7 +2310,9 @@ namespace MediaBrowser.Controller.MediaEncoding { var inputScaleFactor = GetVideoBitrateScaleFactor(inputVideoCodec); var outputScaleFactor = GetVideoBitrateScaleFactor(outputVideoCodec); - var scaleFactor = outputScaleFactor / inputScaleFactor; + + // Don't scale the real bitrate lower than the requested bitrate + var scaleFactor = Math.Max(outputScaleFactor / inputScaleFactor, 1); if (bitrate <= 500000) { @@ -1642,75 +2334,132 @@ namespace MediaBrowser.Controller.MediaEncoding return Convert.ToInt32(scaleFactor * bitrate); } - public int? GetAudioBitrateParam(BaseEncodingJobOptions request, MediaStream audioStream) + public int? GetAudioBitrateParam(BaseEncodingJobOptions request, MediaStream audioStream, int? outputAudioChannels) { - return GetAudioBitrateParam(request.AudioBitRate, request.AudioCodec, audioStream); + return GetAudioBitrateParam(request.AudioBitRate, request.AudioCodec, audioStream, outputAudioChannels); } - public int? GetAudioBitrateParam(int? audioBitRate, string audioCodec, MediaStream audioStream) + public int? GetAudioBitrateParam(int? audioBitRate, string audioCodec, MediaStream audioStream, int? outputAudioChannels) { - if (audioStream == null) + if (audioStream is null) { return null; } - if (audioBitRate.HasValue && string.IsNullOrEmpty(audioCodec)) + var inputChannels = audioStream.Channels ?? 0; + var outputChannels = outputAudioChannels ?? 0; + var bitrate = audioBitRate ?? int.MaxValue; + + if (string.IsNullOrEmpty(audioCodec) + || string.Equals(audioCodec, "aac", StringComparison.OrdinalIgnoreCase) + || string.Equals(audioCodec, "mp3", StringComparison.OrdinalIgnoreCase) + || string.Equals(audioCodec, "opus", StringComparison.OrdinalIgnoreCase) + || string.Equals(audioCodec, "vorbis", StringComparison.OrdinalIgnoreCase) + || string.Equals(audioCodec, "ac3", StringComparison.OrdinalIgnoreCase) + || string.Equals(audioCodec, "eac3", StringComparison.OrdinalIgnoreCase)) { - return Math.Min(384000, audioBitRate.Value); + return (inputChannels, outputChannels) switch + { + (>= 6, >= 6 or 0) => Math.Min(640000, bitrate), + (> 0, > 0) => Math.Min(outputChannels * 128000, bitrate), + (> 0, _) => Math.Min(inputChannels * 128000, bitrate), + (_, _) => Math.Min(384000, bitrate) + }; } - if (audioBitRate.HasValue && !string.IsNullOrEmpty(audioCodec)) + if (string.Equals(audioCodec, "dts", StringComparison.OrdinalIgnoreCase) + || string.Equals(audioCodec, "dca", StringComparison.OrdinalIgnoreCase)) { - if (string.Equals(audioCodec, "aac", StringComparison.OrdinalIgnoreCase) - || string.Equals(audioCodec, "mp3", StringComparison.OrdinalIgnoreCase) - || string.Equals(audioCodec, "ac3", StringComparison.OrdinalIgnoreCase) - || string.Equals(audioCodec, "eac3", StringComparison.OrdinalIgnoreCase)) + return (inputChannels, outputChannels) switch { - if ((audioStream.Channels ?? 0) >= 6) - { - return Math.Min(640000, audioBitRate.Value); - } + (>= 6, >= 6 or 0) => Math.Min(768000, bitrate), + (> 0, > 0) => Math.Min(outputChannels * 136000, bitrate), + (> 0, _) => Math.Min(inputChannels * 136000, bitrate), + (_, _) => Math.Min(672000, bitrate) + }; + } - return Math.Min(384000, audioBitRate.Value); - } + // Empty bitrate area is not allow on iOS + // Default audio bitrate to 128K per channel if we don't have codec specific defaults + // https://ffmpeg.org/ffmpeg-codecs.html#toc-Codec-Options + return 128000 * (outputAudioChannels ?? audioStream.Channels ?? 2); + } - if (string.Equals(audioCodec, "flac", StringComparison.OrdinalIgnoreCase) - || string.Equals(audioCodec, "alac", StringComparison.OrdinalIgnoreCase)) + public string GetAudioVbrModeParam(string encoder, int bitratePerChannel) + { + if (string.Equals(encoder, "libfdk_aac", StringComparison.OrdinalIgnoreCase)) + { + return " -vbr:a " + bitratePerChannel switch { - if ((audioStream.Channels ?? 0) >= 6) - { - return Math.Min(3584000, audioBitRate.Value); - } + < 32000 => "1", + < 48000 => "2", + < 64000 => "3", + < 96000 => "4", + _ => "5" + }; + } - return Math.Min(1536000, audioBitRate.Value); - } + if (string.Equals(encoder, "libmp3lame", StringComparison.OrdinalIgnoreCase)) + { + return " -qscale:a " + bitratePerChannel switch + { + < 48000 => "8", + < 64000 => "6", + < 88000 => "4", + < 112000 => "2", + _ => "0" + }; } - // Empty bitrate area is not allow on iOS - // Default audio bitrate to 128K if it is not being requested - // https://ffmpeg.org/ffmpeg-codecs.html#toc-Codec-Options - return 128000; + if (string.Equals(encoder, "libvorbis", StringComparison.OrdinalIgnoreCase)) + { + return " -qscale:a " + bitratePerChannel switch + { + < 40000 => "0", + < 56000 => "2", + < 80000 => "4", + < 112000 => "6", + _ => "8" + }; + } + + return null; } - public string GetAudioFilterParam(EncodingJobInfo state, EncodingOptions encodingOptions, bool isHls) + public string GetAudioFilterParam(EncodingJobInfo state, EncodingOptions encodingOptions) { var channels = state.OutputAudioChannels; var filters = new List<string>(); - // Boost volume to 200% when downsampling from 6ch to 2ch if (channels.HasValue - && channels.Value <= 2 - && state.AudioStream != null + && channels.Value == 2 + && state.AudioStream is not null && state.AudioStream.Channels.HasValue - && state.AudioStream.Channels.Value > 5 - && !encodingOptions.DownMixAudioBoost.Equals(1)) + && state.AudioStream.Channels.Value > 5) { - filters.Add("volume=" + encodingOptions.DownMixAudioBoost.ToString(_usCulture)); + switch (encodingOptions.DownMixStereoAlgorithm) + { + case DownMixStereoAlgorithms.Dave750: + filters.Add("volume=4.25"); + filters.Add("pan=stereo|c0=0.5*c2+0.707*c0+0.707*c4+0.5*c3|c1=0.5*c2+0.707*c1+0.707*c5+0.5*c3"); + break; + case DownMixStereoAlgorithms.NightmodeDialogue: + filters.Add("pan=stereo|c0=c2+0.30*c0+0.30*c4|c1=c2+0.30*c1+0.30*c5"); + break; + case DownMixStereoAlgorithms.None: + default: + if (!encodingOptions.DownMixAudioBoost.Equals(1)) + { + filters.Add("volume=" + encodingOptions.DownMixAudioBoost.ToString(CultureInfo.InvariantCulture)); + } + + break; + } } var isCopyingTimestamps = state.CopyTimestamps || state.TranscodingType != TranscodingJobType.Progressive; - if (state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode && !isCopyingTimestamps) + if (state.SubtitleStream is not null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode && !isCopyingTimestamps) { var seconds = TimeSpan.FromTicks(state.StartTimeTicks ?? 0).TotalSeconds; @@ -1738,94 +2487,55 @@ namespace MediaBrowser.Controller.MediaEncoding /// <returns>System.Nullable{System.Int32}.</returns> public int? GetNumAudioChannelsParam(EncodingJobInfo state, MediaStream audioStream, string outputAudioCodec) { - if (audioStream == null) + if (audioStream is null) { return null; } var request = state.BaseRequest; - var inputChannels = audioStream?.Channels; + var codec = outputAudioCodec ?? string.Empty; - if (inputChannels <= 0) - { - inputChannels = null; - } + int? resultChannels = state.GetRequestedAudioChannels(codec); - var codec = outputAudioCodec ?? string.Empty; + var inputChannels = audioStream.Channels; - int? transcoderChannelLimit; - if (codec.IndexOf("wma", StringComparison.OrdinalIgnoreCase) != -1) + if (inputChannels > 0) { - // wmav2 currently only supports two channel output - transcoderChannelLimit = 2; - } - else if (codec.IndexOf("mp3", StringComparison.OrdinalIgnoreCase) != -1) - { - // libmp3lame currently only supports two channel output - transcoderChannelLimit = 2; - } - else if (codec.IndexOf("aac", StringComparison.OrdinalIgnoreCase) != -1) - { - // aac is able to handle 8ch(7.1 layout) - transcoderChannelLimit = 8; - } - else - { - // If we don't have any media info then limit it to 6 to prevent encoding errors due to asking for too many channels - transcoderChannelLimit = 6; + resultChannels = inputChannels < resultChannels ? inputChannels : resultChannels ?? inputChannels; } var isTranscodingAudio = !IsCopyCodec(codec); - int? resultChannels = state.GetRequestedAudioChannels(codec); if (isTranscodingAudio) { - resultChannels = GetMinValue(request.TranscodingMaxAudioChannels, resultChannels); - } + var audioEncoder = GetAudioEncoder(state); + if (!_audioTranscodeChannelLookup.TryGetValue(audioEncoder, out var transcoderChannelLimit)) + { + // Set default max transcoding channels to 8 to prevent encoding errors due to asking for too many channels. + transcoderChannelLimit = 8; + } - if (inputChannels.HasValue) - { - resultChannels = resultChannels.HasValue - ? Math.Min(resultChannels.Value, inputChannels.Value) - : inputChannels.Value; - } + // Set resultChannels to minimum between resultChannels, TranscodingMaxAudioChannels, transcoderChannelLimit + resultChannels = transcoderChannelLimit < resultChannels ? transcoderChannelLimit : resultChannels ?? transcoderChannelLimit; - if (isTranscodingAudio && transcoderChannelLimit.HasValue) - { - resultChannels = resultChannels.HasValue - ? Math.Min(resultChannels.Value, transcoderChannelLimit.Value) - : transcoderChannelLimit.Value; - } + if (request.TranscodingMaxAudioChannels < resultChannels) + { + resultChannels = request.TranscodingMaxAudioChannels; + } - // Avoid transcoding to audio channels other than 1ch, 2ch, 6ch (5.1 layout) and 8ch (7.1 layout). - // https://developer.apple.com/documentation/http_live_streaming/hls_authoring_specification_for_apple_devices - if (isTranscodingAudio - && state.TranscodingType != TranscodingJobType.Progressive - && resultChannels.HasValue - && ((resultChannels.Value > 2 && resultChannels.Value < 6) || resultChannels.Value == 7)) - { - resultChannels = 2; + // Avoid transcoding to audio channels other than 1ch, 2ch, 6ch (5.1 layout) and 8ch (7.1 layout). + // https://developer.apple.com/documentation/http_live_streaming/hls_authoring_specification_for_apple_devices + if (state.TranscodingType != TranscodingJobType.Progressive + && ((resultChannels > 2 && resultChannels < 6) || resultChannels == 7)) + { + resultChannels = 2; + } } return resultChannels; } - private int? GetMinValue(int? val1, int? val2) - { - if (!val1.HasValue) - { - return val2; - } - - if (!val2.HasValue) - { - return val1; - } - - return Math.Min(val1.Value, val2.Value); - } - /// <summary> /// Enforces the resolution limit. /// </summary> @@ -1845,19 +2555,40 @@ namespace MediaBrowser.Controller.MediaEncoding /// <summary> /// Gets the fast seek command line parameter. /// </summary> - /// <param name="request">The request.</param> + /// <param name="state">The state.</param> + /// <param name="options">The options.</param> + /// <param name="segmentContainer">Segment Container.</param> /// <returns>System.String.</returns> /// <value>The fast seek command line parameter.</value> - public string GetFastSeekCommandLineParameter(BaseEncodingJobOptions request) + public string GetFastSeekCommandLineParameter(EncodingJobInfo state, EncodingOptions options, string segmentContainer) { - var time = request.StartTimeTicks ?? 0; + var time = state.BaseRequest.StartTimeTicks ?? 0; + var seekParam = string.Empty; if (time > 0) { - return string.Format(CultureInfo.InvariantCulture, "-ss {0}", _mediaEncoder.GetTimeParameter(time)); + seekParam += string.Format(CultureInfo.InvariantCulture, "-ss {0}", _mediaEncoder.GetTimeParameter(time)); + + if (state.IsVideoRequest) + { + var outputVideoCodec = GetVideoEncoder(state, options); + var segmentFormat = GetSegmentFileExtension(segmentContainer).TrimStart('.'); + + // Important: If this is ever re-enabled, make sure not to use it with wtv because it breaks seeking + // Disable -noaccurate_seek on mpegts container due to the timestamps issue on some clients, + // but it's still required for fMP4 container otherwise the audio can't be synced to the video. + if (!string.Equals(state.InputContainer, "wtv", StringComparison.OrdinalIgnoreCase) + && !string.Equals(segmentFormat, "ts", StringComparison.OrdinalIgnoreCase) + && state.TranscodingType != TranscodingJobType.Progressive + && !state.EnableBreakOnNonKeyFrames(outputVideoCodec) + && (state.BaseRequest.StartTimeTicks ?? 0) > 0) + { + seekParam += " -noaccurate_seek"; + } + } } - return string.Empty; + return seekParam; } /// <summary> @@ -1870,31 +2601,33 @@ namespace MediaBrowser.Controller.MediaEncoding // If we don't have known media info // If input is video, use -sn to drop subtitles // Otherwise just return empty - if (state.VideoStream == null && state.AudioStream == null) + if (state.VideoStream is null && state.AudioStream is null) { return state.IsInputVideo ? "-sn" : string.Empty; } - // We have media info, but we don't know the stream indexes - if (state.VideoStream != null && state.VideoStream.Index == -1) + // We have media info, but we don't know the stream index + if (state.VideoStream is not null && state.VideoStream.Index == -1) { return "-sn"; } - // We have media info, but we don't know the stream indexes - if (state.AudioStream != null && state.AudioStream.Index == -1) + // We have media info, but we don't know the stream index + if (state.AudioStream is not null && state.AudioStream.Index == -1) { return state.IsInputVideo ? "-sn" : string.Empty; } var args = string.Empty; - if (state.VideoStream != null) + if (state.VideoStream is not null) { + int videoStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.VideoStream); + args += string.Format( CultureInfo.InvariantCulture, "-map 0:{0}", - state.VideoStream.Index); + videoStreamIndex); } else { @@ -1902,12 +2635,30 @@ namespace MediaBrowser.Controller.MediaEncoding args += "-vn"; } - if (state.AudioStream != null) + if (state.AudioStream is not null) { - args += string.Format( - CultureInfo.InvariantCulture, - " -map 0:{0}", - state.AudioStream.Index); + int audioStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.AudioStream); + if (state.AudioStream.IsExternal) + { + bool hasExternalGraphicsSubs = state.SubtitleStream is not null + && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode + && state.SubtitleStream.IsExternal + && !state.SubtitleStream.IsTextSubtitleStream; + int externalAudioMapIndex = hasExternalGraphicsSubs ? 2 : 1; + + args += string.Format( + CultureInfo.InvariantCulture, + " -map {0}:{1}", + externalAudioMapIndex, + audioStreamIndex); + } + else + { + args += string.Format( + CultureInfo.InvariantCulture, + " -map 0:{0}", + audioStreamIndex); + } } else { @@ -1915,20 +2666,51 @@ namespace MediaBrowser.Controller.MediaEncoding } var subtitleMethod = state.SubtitleDeliveryMethod; - if (state.SubtitleStream == null || subtitleMethod == SubtitleDeliveryMethod.Hls) + if (state.SubtitleStream is null || subtitleMethod == SubtitleDeliveryMethod.Hls) { args += " -map -0:s"; } else if (subtitleMethod == SubtitleDeliveryMethod.Embed) { + int subtitleStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.SubtitleStream); + args += string.Format( CultureInfo.InvariantCulture, " -map 0:{0}", - state.SubtitleStream.Index); + subtitleStreamIndex); } else if (state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream) { - args += " -map 1:0 -sn"; + int externalSubtitleStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.SubtitleStream); + + args += string.Format( + CultureInfo.InvariantCulture, + " -map 1:{0} -sn", + externalSubtitleStreamIndex); + } + + return args; + } + + /// <summary> + /// Gets the negative map args by filters. + /// </summary> + /// <param name="state">The state.</param> + /// <param name="videoProcessFilters">The videoProcessFilters.</param> + /// <returns>System.String.</returns> + public string GetNegativeMapArgsByFilters(EncodingJobInfo state, string videoProcessFilters) + { + string args = string.Empty; + + // http://ffmpeg.org/ffmpeg-all.html#toc-Complex-filtergraphs-1 + if (state.VideoStream is not null && videoProcessFilters.Contains("-filter_complex", StringComparison.Ordinal)) + { + int videoStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.VideoStream); + + args += string.Format( + CultureInfo.InvariantCulture, + "-map -0:{0} ", + videoStreamIndex); } return args; @@ -1950,7 +2732,7 @@ namespace MediaBrowser.Controller.MediaEncoding { var stream = streams.FirstOrDefault(s => s.Index == desiredIndex.Value); - if (stream != null) + if (stream is not null) { return stream; } @@ -1966,206 +2748,422 @@ namespace MediaBrowser.Controller.MediaEncoding return returnFirstIfNoIndex ? streams.FirstOrDefault() : null; } - /// <summary> - /// Gets the graphical subtitle param. - /// </summary> - public string GetGraphicalSubtitleParam( - EncodingJobInfo state, - EncodingOptions options, - string outputVideoCodec) + public static (int? Width, int? Height) GetFixedOutputSize( + int? videoWidth, + int? videoHeight, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight) { - outputVideoCodec ??= string.Empty; + if (!videoWidth.HasValue && !requestedWidth.HasValue) + { + return (null, null); + } - var outputSizeParam = ReadOnlySpan<char>.Empty; - var request = state.BaseRequest; + if (!videoHeight.HasValue && !requestedHeight.HasValue) + { + return (null, null); + } + + int inputWidth = Convert.ToInt32(videoWidth ?? requestedWidth, CultureInfo.InvariantCulture); + int inputHeight = Convert.ToInt32(videoHeight ?? requestedHeight, CultureInfo.InvariantCulture); + int outputWidth = requestedWidth ?? inputWidth; + int outputHeight = requestedHeight ?? inputHeight; + + // Don't transcode video to bigger than 4k when using HW. + int maximumWidth = Math.Min(requestedMaxWidth ?? outputWidth, 4096); + int maximumHeight = Math.Min(requestedMaxHeight ?? outputHeight, 4096); + + if (outputWidth > maximumWidth || outputHeight > maximumHeight) + { + var scaleW = (double)maximumWidth / outputWidth; + var scaleH = (double)maximumHeight / outputHeight; + var scale = Math.Min(scaleW, scaleH); + outputWidth = Math.Min(maximumWidth, (int)(outputWidth * scale)); + outputHeight = Math.Min(maximumHeight, (int)(outputHeight * scale)); + } + + outputWidth = 2 * (outputWidth / 2); + outputHeight = 2 * (outputHeight / 2); - outputSizeParam = GetOutputSizeParamInternal(state, options, outputVideoCodec); + return (outputWidth, outputHeight); + } - var videoSizeParam = string.Empty; - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options) ?? string.Empty; - var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + public static string GetHwScaleFilter( + string hwScaleSuffix, + string videoFormat, + int? videoWidth, + int? videoHeight, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight) + { + var (outWidth, outHeight) = GetFixedOutputSize( + videoWidth, + videoHeight, + requestedWidth, + requestedHeight, + requestedMaxWidth, + requestedMaxHeight); - var isVaapiDecoder = videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiHevcEncoder = outputVideoCodec.IndexOf("hevc_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvH264Encoder = outputVideoCodec.Contains("h264_qsv", StringComparison.OrdinalIgnoreCase); - var isQsvHevcEncoder = outputVideoCodec.Contains("hevc_qsv", StringComparison.OrdinalIgnoreCase); - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - var isNvencEncoder = outputVideoCodec.Contains("nvenc", StringComparison.OrdinalIgnoreCase); - var isTonemappingSupported = IsTonemappingSupported(state, options); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, options); - var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isVaapiH264Encoder || isVaapiHevcEncoder); - var isTonemappingSupportedOnQsv = string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isQsvH264Encoder || isQsvHevcEncoder); + var isFormatFixed = !string.IsNullOrEmpty(videoFormat); + var isSizeFixed = !videoWidth.HasValue + || outWidth.Value != videoWidth.Value + || !videoHeight.HasValue + || outHeight.Value != videoHeight.Value; - // Tonemapping and burn-in graphical subtitles requires overlay_vaapi. - // But it's still in ffmpeg mailing list. Disable it for now. - if (isTonemappingSupportedOnVaapi && isTonemappingSupported && !isVppTonemappingSupported) + var arg1 = isSizeFixed ? ("=w=" + outWidth.Value + ":h=" + outHeight.Value) : string.Empty; + var arg2 = isFormatFixed ? ("format=" + videoFormat) : string.Empty; + if (isFormatFixed) { - return GetOutputSizeParam(state, options, outputVideoCodec); + arg2 = (isSizeFixed ? ':' : '=') + arg2; } - // Setup subtitle scaling - if (state.VideoStream != null && state.VideoStream.Width.HasValue && state.VideoStream.Height.HasValue) + if (!string.IsNullOrEmpty(hwScaleSuffix) && (isSizeFixed || isFormatFixed)) { - // Adjust the size of graphical subtitles to fit the video stream. - var videoStream = state.VideoStream; - var inputWidth = videoStream?.Width; - var inputHeight = videoStream?.Height; - var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight); + return string.Format( + CultureInfo.InvariantCulture, + "scale_{0}{1}{2}", + hwScaleSuffix, + arg1, + arg2); + } - if (width.HasValue && height.HasValue) + return string.Empty; + } + + public static string GetCustomSwScaleFilter( + int? videoWidth, + int? videoHeight, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight) + { + var (outWidth, outHeight) = GetFixedOutputSize( + videoWidth, + videoHeight, + requestedWidth, + requestedHeight, + requestedMaxWidth, + requestedMaxHeight); + + if (outWidth.HasValue && outHeight.HasValue) + { + return string.Format( + CultureInfo.InvariantCulture, + "scale=s={0}x{1}:flags=fast_bilinear", + outWidth.Value, + outHeight.Value); + } + + return string.Empty; + } + + public static string GetAlphaSrcFilter( + EncodingJobInfo state, + int? videoWidth, + int? videoHeight, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight, + int? framerate) + { + var reqTicks = state.BaseRequest.StartTimeTicks ?? 0; + var startTime = TimeSpan.FromTicks(reqTicks).ToString(@"hh\\\:mm\\\:ss\\\.fff", CultureInfo.InvariantCulture); + var (outWidth, outHeight) = GetFixedOutputSize( + videoWidth, + videoHeight, + requestedWidth, + requestedHeight, + requestedMaxWidth, + requestedMaxHeight); + + if (outWidth.HasValue && outHeight.HasValue) + { + return string.Format( + CultureInfo.InvariantCulture, + "alphasrc=s={0}x{1}:r={2}:start='{3}'", + outWidth.Value, + outHeight.Value, + framerate ?? 10, + reqTicks > 0 ? startTime : 0); + } + + return string.Empty; + } + + public static string GetSwScaleFilter( + EncodingJobInfo state, + EncodingOptions options, + string videoEncoder, + int? videoWidth, + int? videoHeight, + Video3DFormat? threedFormat, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight) + { + var isV4l2 = string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase); + var scaleVal = isV4l2 ? 64 : 2; + + // If fixed dimensions were supplied + if (requestedWidth.HasValue && requestedHeight.HasValue) + { + if (isV4l2) { - videoSizeParam = string.Format( - CultureInfo.InvariantCulture, - "scale={0}x{1}", - width.Value, - height.Value); + var widthParam = requestedWidth.Value.ToString(CultureInfo.InvariantCulture); + var heightParam = requestedHeight.Value.ToString(CultureInfo.InvariantCulture); + + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc({0}/64)*64:trunc({1}/2)*2", + widthParam, + heightParam); } - if (!string.IsNullOrEmpty(videoSizeParam) - && !(isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + return GetFixedSwScaleFilter(threedFormat, requestedWidth.Value, requestedHeight.Value); + } + + // If Max dimensions were supplied, for width selects lowest even number between input width and width req size and selects lowest even number from in width*display aspect and requested size + + if (requestedMaxWidth.HasValue && requestedMaxHeight.HasValue) + { + var maxWidthParam = requestedMaxWidth.Value.ToString(CultureInfo.InvariantCulture); + var maxHeightParam = requestedMaxHeight.Value.ToString(CultureInfo.InvariantCulture); + + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(min(max(iw\\,ih*a)\\,min({0}\\,{1}*a))/{2})*{2}:trunc(min(max(iw/a\\,ih)\\,min({0}/a\\,{1}))/2)*2", + maxWidthParam, + maxHeightParam, + scaleVal); + } + + // If a fixed width was requested + if (requestedWidth.HasValue) + { + if (threedFormat.HasValue) { - // For QSV, feed it into hardware encoder now - if (isLinux && (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "hevc_qsv", StringComparison.OrdinalIgnoreCase))) - { - videoSizeParam += ",hwupload=extra_hw_frames=64"; - } + // This method can handle 0 being passed in for the requested height + return GetFixedSwScaleFilter(threedFormat, requestedWidth.Value, 0); } + + var widthParam = requestedWidth.Value.ToString(CultureInfo.InvariantCulture); + + return string.Format( + CultureInfo.InvariantCulture, + "scale={0}:trunc(ow/a/2)*2", + widthParam); } - var mapPrefix = state.SubtitleStream.IsExternal ? - 1 : - 0; + // If a fixed height was requested + if (requestedHeight.HasValue) + { + var heightParam = requestedHeight.Value.ToString(CultureInfo.InvariantCulture); - var subtitleStreamIndex = state.SubtitleStream.IsExternal - ? 0 - : state.SubtitleStream.Index; + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(oh*a/{1})*{1}:{0}", + heightParam, + scaleVal); + } + + // If a max width was requested + if (requestedMaxWidth.HasValue) + { + var maxWidthParam = requestedMaxWidth.Value.ToString(CultureInfo.InvariantCulture); - // Setup default filtergraph utilizing FFMpeg overlay() and FFMpeg scale() (see the return of this function for index reference) - // Always put the scaler before the overlay for better performance - var retStr = !outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay\""; + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(min(max(iw\\,ih*a)\\,{0})/{1})*{1}:trunc(ow/a/2)*2", + maxWidthParam, + scaleVal); + } - // When the input may or may not be hardware VAAPI decodable - if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) + // If a max height was requested + if (requestedMaxHeight.HasValue) { - /* - [base]: HW scaling video to OutputSize - [sub]: SW scaling subtitle to FixedOutputSize - [base][sub]: SW overlay - */ - retStr = !outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3},hwdownload[base];[base][sub]overlay,format=nv12,hwupload\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]hwdownload[base];[base][sub]overlay,format=nv12,hwupload\""; + var maxHeightParam = requestedMaxHeight.Value.ToString(CultureInfo.InvariantCulture); + + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(oh*a/{1})*{1}:min(max(iw/a\\,ih)\\,{0})", + maxHeightParam, + scaleVal); } - // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first - else if (_mediaEncoder.SupportsHwaccel("vaapi") && videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1 - && (string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "libx265", StringComparison.OrdinalIgnoreCase))) + return string.Empty; + } + + private static string GetFixedSwScaleFilter(Video3DFormat? threedFormat, int requestedWidth, int requestedHeight) + { + var widthParam = requestedWidth.ToString(CultureInfo.InvariantCulture); + var heightParam = requestedHeight.ToString(CultureInfo.InvariantCulture); + + string filter = null; + + if (threedFormat.HasValue) { - /* - [base]: SW scaling video to OutputSize - [sub]: SW scaling subtitle to FixedOutputSize - [base][sub]: SW overlay - */ - retStr = !outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay\""; + switch (threedFormat.Value) + { + case Video3DFormat.HalfSideBySide: + filter = "crop=iw/2:ih:0:0,scale=(iw*2):ih,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; + // hsbs crop width in half,scale to correct size, set the display aspect,crop out any black bars we may have made the scale width to requestedWidth. Work out the correct height based on the display aspect it will maintain the aspect where -1 in this case (3d) may not. + break; + case Video3DFormat.FullSideBySide: + filter = "crop=iw/2:ih:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; + // fsbs crop width in half,set the display aspect,crop out any black bars we may have made the scale width to requestedWidth. + break; + case Video3DFormat.HalfTopAndBottom: + filter = "crop=iw:ih/2:0:0,scale=(iw*2):ih),setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; + // htab crop height in half,scale to correct size, set the display aspect,crop out any black bars we may have made the scale width to requestedWidth + break; + case Video3DFormat.FullTopAndBottom: + filter = "crop=iw:ih/2:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; + // ftab crop height in half, set the display aspect,crop out any black bars we may have made the scale width to requestedWidth + break; + default: + break; + } } - else if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) + + // default + if (filter is null) { - /* - QSV in FFMpeg can now setup hardware overlay for transcodes. - For software decoding and hardware encoding option, frames must be hwuploaded into hardware - with fixed frame size. - Currently only supports linux. - */ - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported) + if (requestedHeight > 0) { - retStr = " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3},hwdownload,format=nv12[base];[base][sub]overlay\""; + filter = "scale=trunc({0}/2)*2:trunc({1}/2)*2"; } - else if (isLinux) + else { - retStr = !outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay_qsv\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay_qsv\""; + filter = "scale={0}:trunc({0}/a/2)*2"; } } - else if (isNvdecDecoder && isNvencEncoder) - { - retStr = !outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay,format=nv12|yuv420p,hwupload_cuda\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay,format=nv12|yuv420p,hwupload_cuda\""; - } + return string.Format(CultureInfo.InvariantCulture, filter, widthParam, heightParam); + } + + public static string GetSwDeinterlaceFilter(EncodingJobInfo state, EncodingOptions options) + { + var doubleRateDeint = options.DeinterlaceDoubleRate && state.VideoStream?.AverageFrameRate <= 30; return string.Format( CultureInfo.InvariantCulture, - retStr, - mapPrefix, - subtitleStreamIndex, - state.VideoStream.Index, - outputSizeParam.ToString(), - videoSizeParam); + "{0}={1}:-1:0", + string.Equals(options.DeinterlaceMethod, "bwdif", StringComparison.OrdinalIgnoreCase) ? "bwdif" : "yadif", + doubleRateDeint ? "1" : "0"); } - public static (int? width, int? height) GetFixedOutputSize( - int? videoWidth, - int? videoHeight, - int? requestedWidth, - int? requestedHeight, - int? requestedMaxWidth, - int? requestedMaxHeight) + public static string GetHwDeinterlaceFilter(EncodingJobInfo state, EncodingOptions options, string hwDeintSuffix) { - if (!videoWidth.HasValue && !requestedWidth.HasValue) + var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.AverageFrameRate ?? 60) <= 30; + if (hwDeintSuffix.Contains("cuda", StringComparison.OrdinalIgnoreCase)) { - return (null, null); + return string.Format( + CultureInfo.InvariantCulture, + "yadif_cuda={0}:-1:0", + doubleRateDeint ? "1" : "0"); } - if (!videoHeight.HasValue && !requestedHeight.HasValue) + if (hwDeintSuffix.Contains("vaapi", StringComparison.OrdinalIgnoreCase)) { - return (null, null); + return string.Format( + CultureInfo.InvariantCulture, + "deinterlace_vaapi=rate={0}", + doubleRateDeint ? "field" : "frame"); } - decimal inputWidth = Convert.ToDecimal(videoWidth ?? requestedWidth, CultureInfo.InvariantCulture); - decimal inputHeight = Convert.ToDecimal(videoHeight ?? requestedHeight, CultureInfo.InvariantCulture); - decimal outputWidth = requestedWidth.HasValue ? Convert.ToDecimal(requestedWidth.Value) : inputWidth; - decimal outputHeight = requestedHeight.HasValue ? Convert.ToDecimal(requestedHeight.Value) : inputHeight; - decimal maximumWidth = requestedMaxWidth.HasValue ? Convert.ToDecimal(requestedMaxWidth.Value) : outputWidth; - decimal maximumHeight = requestedMaxHeight.HasValue ? Convert.ToDecimal(requestedMaxHeight.Value) : outputHeight; + if (hwDeintSuffix.Contains("qsv", StringComparison.OrdinalIgnoreCase)) + { + return "deinterlace_qsv=mode=2"; + } - if (outputWidth > maximumWidth || outputHeight > maximumHeight) + if (hwDeintSuffix.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase)) { - var scale = Math.Min(maximumWidth / outputWidth, maximumHeight / outputHeight); - outputWidth = Math.Min(maximumWidth, Math.Truncate(outputWidth * scale)); - outputHeight = Math.Min(maximumHeight, Math.Truncate(outputHeight * scale)); + return string.Format( + CultureInfo.InvariantCulture, + "yadif_videotoolbox={0}:-1:0", + doubleRateDeint ? "1" : "0"); } - outputWidth = 2 * Math.Truncate(outputWidth / 2); - outputHeight = 2 * Math.Truncate(outputHeight / 2); + return string.Empty; + } + + public string GetHwTonemapFilter(EncodingOptions options, string hwTonemapSuffix, string videoFormat) + { + if (string.IsNullOrEmpty(hwTonemapSuffix)) + { + return string.Empty; + } + + var args = string.Empty; + var algorithm = options.TonemappingAlgorithm; - return (Convert.ToInt32(outputWidth), Convert.ToInt32(outputHeight)); + if (string.Equals(hwTonemapSuffix, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + args = "procamp_vaapi=b={1}:c={2},tonemap_vaapi=format={0}:p=bt709:t=bt709:m=bt709:extra_hw_frames=32"; + + return string.Format( + CultureInfo.InvariantCulture, + args, + videoFormat ?? "nv12", + options.VppTonemappingBrightness, + options.VppTonemappingContrast); + } + else + { + args = "tonemap_{0}=format={1}:p=bt709:t=bt709:m=bt709:tonemap={2}:peak={3}:desat={4}"; + + if (string.Equals(options.TonemappingMode, "max", StringComparison.OrdinalIgnoreCase) + || string.Equals(options.TonemappingMode, "rgb", StringComparison.OrdinalIgnoreCase)) + { + if (_mediaEncoder.EncoderVersion >= _minFFmpegOclCuTonemapMode) + { + args += ":tonemap_mode={5}"; + } + } + + if (options.TonemappingParam != 0) + { + args += ":param={6}"; + } + + if (string.Equals(options.TonemappingRange, "tv", StringComparison.OrdinalIgnoreCase) + || string.Equals(options.TonemappingRange, "pc", StringComparison.OrdinalIgnoreCase)) + { + args += ":range={7}"; + } + } + + return string.Format( + CultureInfo.InvariantCulture, + args, + hwTonemapSuffix, + videoFormat ?? "nv12", + algorithm, + options.TonemappingPeak, + options.TonemappingDesat, + options.TonemappingMode, + options.TonemappingParam, + options.TonemappingRange); } - public List<string> GetScalingFilters( - EncodingJobInfo state, + public string GetLibplaceboFilter( EncodingOptions options, + string videoFormat, + bool doTonemap, int? videoWidth, int? videoHeight, - Video3DFormat? threedFormat, - string videoDecoder, - string videoEncoder, int? requestedWidth, int? requestedHeight, int? requestedMaxWidth, int? requestedMaxHeight) { - var filters = new List<string>(); - var (width, height) = GetFixedOutputSize( + var (outWidth, outHeight) = GetFixedOutputSize( videoWidth, videoHeight, requestedWidth, @@ -2173,761 +3171,2475 @@ namespace MediaBrowser.Controller.MediaEncoding requestedMaxWidth, requestedMaxHeight); - if ((string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) - && width.HasValue - && height.HasValue) - { - // Given the input dimensions (inputWidth, inputHeight), determine the output dimensions - // (outputWidth, outputHeight). The user may request precise output dimensions or maximum - // output dimensions. Output dimensions are guaranteed to be even. - var outputWidth = width.Value; - var outputHeight = height.Value; - var qsv_or_vaapi = string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase); - var isDeintEnabled = state.DeInterlace("h264", true) - || state.DeInterlace("avc", true) - || state.DeInterlace("h265", true) - || state.DeInterlace("hevc", true); - - var isVaapiDecoder = videoDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); - var isVaapiH264Encoder = videoEncoder.Contains("h264_vaapi", StringComparison.OrdinalIgnoreCase); - var isVaapiHevcEncoder = videoEncoder.Contains("hevc_vaapi", StringComparison.OrdinalIgnoreCase); - var isQsvH264Encoder = videoEncoder.Contains("h264_qsv", StringComparison.OrdinalIgnoreCase); - var isQsvHevcEncoder = videoEncoder.Contains("hevc_qsv", StringComparison.OrdinalIgnoreCase); - var isTonemappingSupported = IsTonemappingSupported(state, options); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, options); - var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isVaapiH264Encoder || isVaapiHevcEncoder); - var isTonemappingSupportedOnQsv = string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isQsvH264Encoder || isQsvHevcEncoder); - var isP010PixFmtRequired = (isTonemappingSupportedOnVaapi && (isTonemappingSupported || isVppTonemappingSupported)) - || (isTonemappingSupportedOnQsv && isVppTonemappingSupported); - - var outputPixFmt = "format=nv12"; - if (isP010PixFmtRequired) - { - outputPixFmt = "format=p010"; - } - - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported) - { - qsv_or_vaapi = false; - } - - if (!videoWidth.HasValue - || outputWidth != videoWidth.Value - || !videoHeight.HasValue - || outputHeight != videoHeight.Value) - { - // Force nv12 pixel format to enable 10-bit to 8-bit colour conversion. - // use vpp_qsv filter to avoid green bar when the fixed output size is requested. - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "{0}=w={1}:h={2}{3}{4}", - qsv_or_vaapi ? "vpp_qsv" : "scale_vaapi", - outputWidth, - outputHeight, - ":" + outputPixFmt, - (qsv_or_vaapi && isDeintEnabled) ? ":deinterlace=1" : string.Empty)); + var isFormatFixed = !string.IsNullOrEmpty(videoFormat); + var isSizeFixed = !videoWidth.HasValue + || outWidth.Value != videoWidth.Value + || !videoHeight.HasValue + || outHeight.Value != videoHeight.Value; + + var sizeArg = isSizeFixed ? (":w=" + outWidth.Value + ":h=" + outHeight.Value) : string.Empty; + var formatArg = isFormatFixed ? (":format=" + videoFormat) : string.Empty; + var tonemapArg = string.Empty; + + if (doTonemap) + { + var algorithm = options.TonemappingAlgorithm; + var mode = options.TonemappingMode; + var range = options.TonemappingRange; + + if (string.Equals(algorithm, "bt2390", StringComparison.OrdinalIgnoreCase)) + { + algorithm = "bt.2390"; + } + else if (string.Equals(algorithm, "none", StringComparison.OrdinalIgnoreCase)) + { + algorithm = "clip"; } - // Assert 10-bit is P010 so as we can avoid the extra scaler to get a bit more fps on high res HDR videos. - else if (!isP010PixFmtRequired) + tonemapArg = ":tonemapping=" + algorithm; + + if (string.Equals(mode, "max", StringComparison.OrdinalIgnoreCase) + || string.Equals(mode, "rgb", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "{0}={1}{2}", - qsv_or_vaapi ? "vpp_qsv" : "scale_vaapi", - outputPixFmt, - (qsv_or_vaapi && isDeintEnabled) ? ":deinterlace=1" : string.Empty)); + tonemapArg += ":tonemapping_mode=" + mode; } + + tonemapArg += ":peak_detect=0:color_primaries=bt709:color_trc=bt709:colorspace=bt709"; + + if (string.Equals(range, "tv", StringComparison.OrdinalIgnoreCase) + || string.Equals(range, "pc", StringComparison.OrdinalIgnoreCase)) + { + tonemapArg += ":range=" + range; + } + } + + return string.Format( + CultureInfo.InvariantCulture, + "libplacebo=upscaler=none:downscaler=none{0}{1}{2}", + sizeArg, + formatArg, + tonemapArg); + } + + /// <summary> + /// Gets the parameter of software filter chain. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="vidEncoder">Video encoder to use.</param> + /// <returns>The tuple contains three lists: main, sub and overlay filters.</returns> + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetSwVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isV4l2Encoder = vidEncoder.Contains("h264_v4l2m2m", StringComparison.OrdinalIgnoreCase); + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, false)); + + // INPUT sw surface(memory/copy-back from vram) + // sw deint + if (doDeintH2645) + { + var deintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(deintFilter); } - else if ((videoDecoder ?? string.Empty).Contains("cuda", StringComparison.OrdinalIgnoreCase) - && width.HasValue - && height.HasValue) + + var outFormat = isSwDecoder ? "yuv420p" : "nv12"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + if (isVaapiEncoder) + { + outFormat = "nv12"; + } + else if (isV4l2Encoder) + { + outFormat = "yuv420p"; + } + + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // sw tonemap <= TODO: finsh the fast tonemap filter + + // OUTPUT yuv420p/nv12 surface(memory) + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (hasTextSubs) + { + // subtitles=f='*.ass':alpha=0 + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + else if (hasGraphicalSubs) + { + // [0:s]scale=s=1280x720 + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// <summary> + /// Gets the parameter of Nvidia NVENC filter chain. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="vidEncoder">Video encoder to use.</param> + /// <returns>The tuple contains three lists: main, sub and overlay filters.</returns> + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetNvidiaVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) { - var outputWidth = width.Value; - var outputHeight = height.Value; + return (null, null, null); + } - var isTonemappingSupported = IsTonemappingSupported(state, options); - var isTonemappingSupportedOnNvenc = string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase); - var isCudaFormatConversionSupported = _mediaEncoder.SupportsFilter("scale_cuda", "Output format (default \"same\")"); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); - var outputPixFmt = string.Empty; - if (isCudaFormatConversionSupported) + // legacy cuvid pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || !IsCudaFullSupported() + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + return GetSwVidFilterChain(state, options, vidEncoder); + } + + // prefered nvdec/cuvid + cuda filters + nvenc pipeline + return GetNvidiaVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetNvidiaVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isNvDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); + var isNvencEncoder = vidEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isNvencEncoder; + var isCuInCuOut = isNvDecoder && isNvencEncoder; + + var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.AverageFrameRate ?? 60) <= 30; + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doCuTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doCuTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doCuTonemap ? "yuv420p10le" : "yuv420p"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // sw => hw + if (doCuTonemap) + { + mainFilters.Add("hwupload=derive_device=cuda"); + } + } + + if (isNvDecoder) + { + // INPUT cuda surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "cuda"); + mainFilters.Add(deintFilter); + } + + var outFormat = doCuTonemap ? string.Empty : "yuv420p"; + var hwScaleFilter = GetHwScaleFilter("cuda", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw scale + mainFilters.Add(hwScaleFilter); + } + + // hw tonemap + if (doCuTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "cuda", "yuv420p"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForCuTonemap = isSwDecoder && doCuTonemap; + if ((isNvDecoder && isSwEncoder) || (isUploadForCuTonemap && hasSubs)) + { + memoryOutput = true; + + // OUTPUT yuv420p surface(memory) + mainFilters.Add("hwdownload"); + mainFilters.Add("format=yuv420p"); + } + + // OUTPUT yuv420p surface(memory) + if (isSwDecoder && isNvencEncoder && !isUploadForCuTonemap) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) { - outputPixFmt = "format=nv12"; - if (isTonemappingSupported && isTonemappingSupportedOnNvenc) + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + // OUTPUT cuda(yuv420p) surface(vram) + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (isCuInCuOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) { - outputPixFmt = "format=p010"; + // scale=s=1280x720,format=yuva420p,hwupload + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + subFilters.Add("format=yuva420p"); } + else if (hasTextSubs) + { + // alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=yuva420p"); + subFilters.Add(subTextSubtitlesFilter); + } + + subFilters.Add("hwupload=derive_device=cuda"); + overlayFilters.Add("overlay_cuda=eof_action=endall:shortest=1:repeatlast=0"); + } + } + else + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); } + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// <summary> + /// Gets the parameter of AMD AMF filter chain. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="vidEncoder">Video encoder to use.</param> + /// <returns>The tuple contains three lists: main, sub and overlay filters.</returns> + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAmdVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var isWindows = OperatingSystem.IsWindows(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("amf", StringComparison.OrdinalIgnoreCase); + var isAmfDx11OclSupported = isWindows && _mediaEncoder.SupportsHwaccel("d3d11va") && IsOpenclFullSupported(); + + // legacy d3d11va pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || !isAmfDx11OclSupported + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + return GetSwVidFilterChain(state, options, vidEncoder); + } + + // prefered d3d11va + opencl filters + amf pipeline + return GetAmdDx11VidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAmdDx11VidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isAmfEncoder = vidEncoder.Contains("amf", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isAmfEncoder; + var isDxInDxOut = isD3d11vaDecoder && isAmfEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doOclTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); - if (!videoWidth.HasValue - || outputWidth != videoWidth.Value - || !videoHeight.HasValue - || outputHeight != videoHeight.Value) + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale_cuda=w={0}:h={1}{2}", - outputWidth, - outputHeight, - isCudaFormatConversionSupported ? (":" + outputPixFmt) : string.Empty)); + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); } - else if (isCudaFormatConversionSupported) + + var outFormat = doOclTonemap ? "yuv420p10le" : "yuv420p"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale_cuda={0}", - outputPixFmt)); + mainFilters.Add("hwupload=derive_device=d3d11va:extra_hw_frames=16"); + mainFilters.Add("format=d3d11"); + mainFilters.Add("hwmap=derive_device=opencl"); } } - else if ((videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1 - && width.HasValue - && height.HasValue) + + if (isD3d11vaDecoder) { - // Nothing to do, it's handled as an input resize filter + // INPUT d3d11 surface(vram) + // map from d3d11va to opencl via d3d11-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl"); + + // hw deint <= TODO: finsh the 'yadif_opencl' filter + + var outFormat = doOclTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("opencl", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw scale + mainFilters.Add(hwScaleFilter); } - else + + // hw tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + if (isD3d11vaDecoder && isSwEncoder) { - var isExynosV4L2 = string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase); + memoryOutput = true; - // If fixed dimensions were supplied - if (requestedWidth.HasValue && requestedHeight.HasValue) + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl. + var hwTransferFilter = hasGraphicalSubs ? "hwdownload" : "hwmap=mode=read"; + mainFilters.Add(hwTransferFilter); + mainFilters.Add("format=nv12"); + } + + // OUTPUT yuv420p surface + if (isSwDecoder && isAmfEncoder && !isUploadForOclTonemap) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if ((isDxInDxOut || isUploadForOclTonemap) && !hasSubs) + { + // OUTPUT d3d11(nv12) surface(vram) + // reverse-mapping via d3d11-opencl interop. + mainFilters.Add("hwmap=derive_device=d3d11va:reverse=1"); + mainFilters.Add("format=d3d11"); + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (isDxInDxOut || isUploadForOclTonemap) + { + if (hasSubs) { - if (isExynosV4L2) + if (hasGraphicalSubs) { - var widthParam = requestedWidth.Value.ToString(_usCulture); - var heightParam = requestedHeight.Value.ToString(_usCulture); - - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc({0}/64)*64:trunc({1}/2)*2", - widthParam, - heightParam)); + // scale=s=1280x720,format=yuva420p,hwupload + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + subFilters.Add("format=yuva420p"); } - else + else if (hasTextSubs) { - filters.Add(GetFixedSizeScalingFilter(threedFormat, requestedWidth.Value, requestedHeight.Value)); + // alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=yuva420p"); + subFilters.Add(subTextSubtitlesFilter); } + + subFilters.Add("hwupload=derive_device=opencl"); + overlayFilters.Add("overlay_opencl=eof_action=endall:shortest=1:repeatlast=0"); + overlayFilters.Add("hwmap=derive_device=d3d11va:reverse=1"); + overlayFilters.Add("format=d3d11"); + } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); } + } - // If Max dimensions were supplied, for width selects lowest even number between input width and width req size and selects lowest even number from in width*display aspect and requested size - else if (requestedMaxWidth.HasValue && requestedMaxHeight.HasValue) + return (mainFilters, subFilters, overlayFilters); + } + + /// <summary> + /// Gets the parameter of Intel QSV filter chain. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="vidEncoder">Video encoder to use.</param> + /// <returns>The tuple contains three lists: main, sub and overlay filters.</returns> + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetIntelVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var isWindows = OperatingSystem.IsWindows(); + var isLinux = OperatingSystem.IsLinux(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvOclSupported = _mediaEncoder.SupportsHwaccel("qsv") && IsOpenclFullSupported(); + var isIntelDx11OclSupported = isWindows + && _mediaEncoder.SupportsHwaccel("d3d11va") + && isQsvOclSupported; + var isIntelVaapiOclSupported = isLinux + && IsVaapiSupported(state) + && isQsvOclSupported; + + // legacy qsv pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || (!isIntelVaapiOclSupported && !isIntelDx11OclSupported) + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + return GetSwVidFilterChain(state, options, vidEncoder); + } + + // prefered qsv(vaapi) + opencl filters pipeline + if (isIntelVaapiOclSupported) + { + return GetIntelQsvVaapiVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + // prefered qsv(d3d11) + opencl filters pipeline + if (isIntelDx11OclSupported) + { + return GetIntelQsvDx11VidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + return (null, null, null); + } + + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetIntelQsvDx11VidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isQsvDecoder = vidDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvEncoder = vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isD3d11vaDecoder || isQsvDecoder; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isQsvEncoder; + var isQsvInQsvOut = isHwDecoder && isQsvEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doOclTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) { - var maxWidthParam = requestedMaxWidth.Value.ToString(_usCulture); - var maxHeightParam = requestedMaxHeight.Value.ToString(_usCulture); + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12"); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); - if (isExynosV4L2) + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload=derive_device=opencl"); + } + } + else if (isD3d11vaDecoder || isQsvDecoder) + { + var outFormat = doOclTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + if (isD3d11vaDecoder) + { + if (!string.IsNullOrEmpty(hwScaleFilter) || doDeintH2645) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,min({0}\\,{1}*dar))/64)*64:trunc(min(max(iw/dar\\,ih)\\,min({0}/dar\\,{1}))/2)*2", - maxWidthParam, - maxHeightParam)); + // INPUT d3d11 surface(vram) + // map from d3d11va to qsv. + mainFilters.Add("hwmap=derive_device=qsv"); } else { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,min({0}\\,{1}*dar))/2)*2:trunc(min(max(iw/dar\\,ih)\\,min({0}/dar\\,{1}))/2)*2", - maxWidthParam, - maxHeightParam)); + // Insert a qsv scaler to sync the decoder surface, + // msdk will passthrough this internally. + mainFilters.Add("hwmap=derive_device=qsv,scale_qsv"); } } - // If a fixed width was requested - else if (requestedWidth.HasValue) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "qsv"); + mainFilters.Add(deintFilter); + } + + // hw scale + mainFilters.Add(hwScaleFilter); + } + + if (doOclTonemap && isHwDecoder) + { + // map from qsv to opencl via qsv(d3d11)-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl"); + } + + // hw tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + var isHwmapUsable = isSwEncoder && doOclTonemap; + if ((isHwDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl. + // qsv hwmap is not fully implemented for the time being. + mainFilters.Add(isHwmapUsable ? "hwmap=mode=read" : "hwdownload"); + mainFilters.Add("format=nv12"); + } + + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isQsvEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) { - if (threedFormat.HasValue) + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (isQsvInQsvOut && doOclTonemap) + { + // OUTPUT qsv(nv12) surface(vram) + // reverse-mapping via qsv(d3d11)-opencl interop. + mainFilters.Add("hwmap=derive_device=qsv:reverse=1"); + mainFilters.Add("format=qsv"); + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (isQsvInQsvOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) { - // This method can handle 0 being passed in for the requested height - filters.Add(GetFixedSizeScalingFilter(threedFormat, requestedWidth.Value, 0)); + // scale,format=bgra,hwupload + // overlay_qsv can handle overlay scaling, + // add a dummy scale filter to pair with -canvas_size. + subFilters.Add("scale=flags=fast_bilinear"); + subFilters.Add("format=bgra"); } - else + else if (hasTextSubs) { - var widthParam = requestedWidth.Value.ToString(_usCulture); - - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale={0}:trunc(ow/a/2)*2", - widthParam)); + // alphasrc=s=1280x720:r=10:start=0,format=bgra,subtitles,hwupload + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); } + + // qsv requires a fixed pool size. + // default to 64 otherwise it will fail on certain iGPU. + subFilters.Add("hwupload=derive_device=qsv:extra_hw_frames=64"); + + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var overlaySize = (overlayW.HasValue && overlayH.HasValue) + ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + : string.Empty; + var overlayQsvFilter = string.Format( + CultureInfo.InvariantCulture, + "overlay_qsv=eof_action=endall:shortest=1:repeatlast=0{0}", + overlaySize); + overlayFilters.Add(overlayQsvFilter); + } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetIntelQsvVaapiVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isQsvDecoder = vidDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvEncoder = vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isVaapiDecoder || isQsvDecoder; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isQsvEncoder; + var isQsvInQsvOut = isHwDecoder && isQsvEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doVaVppTonemap = IsVaapiVppTonemapAvailable(state, options); + var doOclTonemap = !doVaVppTonemap && IsHwTonemapAvailable(state, options); + var doTonemap = doVaVppTonemap || doOclTonemap; + var doDeintH2645 = doDeintH264 || doDeintHevc; + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12"); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload=derive_device=opencl"); + } + } + else if (isVaapiDecoder || isQsvDecoder) + { + // INPUT vaapi/qsv surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, isVaapiDecoder ? "vaapi" : "qsv"); + mainFilters.Add(deintFilter); } - // If a fixed height was requested - else if (requestedHeight.HasValue) + var outFormat = doTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter(isVaapiDecoder ? "vaapi" : "qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + // allocate extra pool sizes for vaapi vpp + if (!string.IsNullOrEmpty(hwScaleFilter) && isVaapiDecoder) + { + hwScaleFilter += ":extra_hw_frames=24"; + } + + // hw scale + mainFilters.Add(hwScaleFilter); + } + + // vaapi vpp tonemap + if (doVaVppTonemap && isHwDecoder) + { + if (isQsvDecoder) { - var heightParam = requestedHeight.Value.ToString(_usCulture); + // map from qsv to vaapi. + mainFilters.Add("hwmap=derive_device=vaapi"); + } + + var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12"); + mainFilters.Add(tonemapFilter); + + if (isQsvDecoder) + { + // map from vaapi to qsv. + mainFilters.Add("hwmap=derive_device=qsv"); + } + } + + if (doOclTonemap && isHwDecoder) + { + // map from qsv to opencl via qsv(vaapi)-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl"); + } + + // ocl tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + var isHwmapUsable = isSwEncoder && (doOclTonemap || isVaapiDecoder); + if ((isHwDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl/vaapi. + // qsv hwmap is not fully implemented for the time being. + mainFilters.Add(isHwmapUsable ? "hwmap=mode=read" : "hwdownload"); + mainFilters.Add("format=nv12"); + } - if (isExynosV4L2) + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isQsvEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (isQsvInQsvOut) + { + if (doOclTonemap) + { + // OUTPUT qsv(nv12) surface(vram) + // reverse-mapping via qsv(vaapi)-opencl interop. + // add extra pool size to avoid the 'cannot allocate memory' error on hevc_qsv. + mainFilters.Add("hwmap=derive_device=qsv:reverse=1:extra_hw_frames=16"); + mainFilters.Add("format=qsv"); + } + else if (isVaapiDecoder) + { + mainFilters.Add("hwmap=derive_device=qsv"); + mainFilters.Add("format=qsv"); + } + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (isQsvInQsvOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/64)*64:{0}", - heightParam)); + subFilters.Add("scale=flags=fast_bilinear"); + subFilters.Add("format=bgra"); } - else + else if (hasTextSubs) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/2)*2:{0}", - heightParam)); + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); } + + // qsv requires a fixed pool size. + // default to 64 otherwise it will fail on certain iGPU. + subFilters.Add("hwupload=derive_device=qsv:extra_hw_frames=64"); + + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var overlaySize = (overlayW.HasValue && overlayH.HasValue) + ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + : string.Empty; + var overlayQsvFilter = string.Format( + CultureInfo.InvariantCulture, + "overlay_qsv=eof_action=endall:shortest=1:repeatlast=0{0}", + overlaySize); + overlayFilters.Add(overlayQsvFilter); + } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=pass:shortest=1:repeatlast=0"); + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// <summary> + /// Gets the parameter of Intel/AMD VAAPI filter chain. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="vidEncoder">Video encoder to use.</param> + /// <returns>The tuple contains three lists: main, sub and overlay filters.</returns> + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetVaapiVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var isLinux = OperatingSystem.IsLinux(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiFullSupported = isLinux && IsVaapiSupported(state) && IsVaapiFullSupported(); + var isVaapiOclSupported = isVaapiFullSupported && IsOpenclFullSupported(); + var isVaapiVkSupported = isVaapiFullSupported && IsVulkanFullSupported(); + + // legacy vaapi pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || !isVaapiOclSupported + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder); + + if (!isSwEncoder) + { + var newfilters = new List<string>(); + var noOverlay = swFilterChain.OverlayFilters.Count == 0; + newfilters.AddRange(noOverlay ? swFilterChain.MainFilters : swFilterChain.OverlayFilters); + newfilters.Add("hwupload=derive_device=vaapi"); + + var mainFilters = noOverlay ? newfilters : swFilterChain.MainFilters; + var overlayFilters = noOverlay ? swFilterChain.OverlayFilters : newfilters; + return (mainFilters, swFilterChain.SubFilters, overlayFilters); + } + + return swFilterChain; + } + + // prefered vaapi + opencl filters pipeline + if (_mediaEncoder.IsVaapiDeviceInteliHD) + { + // Intel iHD path, with extra vpp tonemap and overlay support. + return GetIntelVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + // prefered vaapi + vulkan filters pipeline + if (_mediaEncoder.IsVaapiDeviceAmd + && isVaapiVkSupported + && _mediaEncoder.IsVaapiDeviceSupportVulkanDrmInterop) + { + // AMD radeonsi path(targeting Polaris/gfx8+), with extra vulkan tonemap and overlay support. + return GetAmdVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + // Intel i965 and Amd legacy driver path, only featuring scale and deinterlace support. + return GetVaapiLimitedVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetIntelVaapiFullVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isVaapiEncoder; + var isVaInVaOut = isVaapiDecoder && isVaapiEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doVaVppTonemap = isVaapiDecoder && IsVaapiVppTonemapAvailable(state, options); + var doOclTonemap = !doVaVppTonemap && IsHwTonemapAvailable(state, options); + var doTonemap = doVaVppTonemap || doOclTonemap; + var doDeintH2645 = doDeintH264 || doDeintHevc; + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doOclTonemap ? "yuv420p10le" : "nv12"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload=derive_device=opencl"); + } + } + else if (isVaapiDecoder) + { + // INPUT vaapi surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi"); + mainFilters.Add(deintFilter); + } + + var outFormat = doTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + // allocate extra pool sizes for vaapi vpp + if (!string.IsNullOrEmpty(hwScaleFilter)) + { + hwScaleFilter += ":extra_hw_frames=24"; + } + + // hw scale + mainFilters.Add(hwScaleFilter); + } + + // vaapi vpp tonemap + if (doVaVppTonemap && isVaapiDecoder) + { + var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12"); + mainFilters.Add(tonemapFilter); + } + + if (doOclTonemap && isVaapiDecoder) + { + // map from vaapi to opencl via vaapi-opencl interop(Intel only). + mainFilters.Add("hwmap=derive_device=opencl"); + } + + // ocl tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + if (doOclTonemap && isVaInVaOut) + { + // OUTPUT vaapi(nv12) surface(vram) + // reverse-mapping via vaapi-opencl interop. + mainFilters.Add("hwmap=derive_device=vaapi:reverse=1"); + mainFilters.Add("format=vaapi"); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + var isHwmapNotUsable = isUploadForOclTonemap && isVaapiEncoder; + if ((isVaapiDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl/vaapi. + mainFilters.Add(isHwmapNotUsable ? "hwdownload" : "hwmap=mode=read"); + mainFilters.Add("format=nv12"); + } + + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isVaapiEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); } + } - // If a max width was requested - else if (requestedMaxWidth.HasValue) + if (memoryOutput && isVaapiEncoder) + { + if (!hasGraphicalSubs) { - var maxWidthParam = requestedMaxWidth.Value.ToString(_usCulture); + mainFilters.Add("hwupload_vaapi"); + } + } - if (isExynosV4L2) + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (isVaInVaOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,{0})/64)*64:trunc(ow/dar/2)*2", - maxWidthParam)); + subFilters.Add("scale=flags=fast_bilinear"); + subFilters.Add("format=bgra"); } - else + else if (hasTextSubs) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,{0})/2)*2:trunc(ow/dar/2)*2", - maxWidthParam)); + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); } + + subFilters.Add("hwupload=derive_device=vaapi"); + + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var overlaySize = (overlayW.HasValue && overlayH.HasValue) + ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + : string.Empty; + var overlayVaapiFilter = string.Format( + CultureInfo.InvariantCulture, + "overlay_vaapi=eof_action=endall:shortest=1:repeatlast=0{0}", + overlaySize); + overlayFilters.Add(overlayVaapiFilter); } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=pass:shortest=1:repeatlast=0"); + + if (isVaapiEncoder) + { + overlayFilters.Add("hwupload_vaapi"); + } + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAmdVaapiFullVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isVaapiEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doVkTonemap = IsVulkanHwTonemapAvailable(state, options); + var doDeintH2645 = doDeintH264 || doDeintHevc; - // If a max height was requested - else if (requestedMaxHeight.HasValue) + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doVkTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) { - var maxHeightParam = requestedMaxHeight.Value.ToString(_usCulture); + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } - if (isExynosV4L2) + if (doVkTonemap || hasSubs) + { + // sw => hw + mainFilters.Add("hwupload=derive_device=vulkan"); + mainFilters.Add("format=vulkan"); + } + else + { + // sw scale + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=nv12"); + } + } + else if (isVaapiDecoder) + { + // INPUT vaapi surface(vram) + if (doVkTonemap || hasSubs) + { + // map from vaapi to vulkan/drm via interop (Polaris/gfx8+). + mainFilters.Add("hwmap=derive_device=vulkan"); + mainFilters.Add("format=vulkan"); + } + else + { + // hw deint + if (doDeintH2645) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/64)*64:min(max(iw/dar\\,ih)\\,{0})", - maxHeightParam)); + var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi"); + mainFilters.Add(deintFilter); } - else + + // hw scale + var hwScaleFilter = GetHwScaleFilter("vaapi", "nv12", inW, inH, reqW, reqH, reqMaxW, reqMaxH); + mainFilters.Add(hwScaleFilter); + } + } + + // vk libplacebo + if (doVkTonemap || hasSubs) + { + var libplaceboFilter = GetLibplaceboFilter(options, "bgra", doVkTonemap, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + mainFilters.Add(libplaceboFilter); + } + + if (doVkTonemap && !hasSubs) + { + // OUTPUT vaapi(nv12) surface(vram) + // map from vulkan/drm to vaapi via interop (Polaris/gfx8+). + mainFilters.Add("hwmap=derive_device=vaapi"); + mainFilters.Add("format=vaapi"); + + // clear the surf->meta_offset and output nv12 + mainFilters.Add("scale_vaapi=format=nv12"); + + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi"); + mainFilters.Add(deintFilter); + } + } + + if (!hasSubs) + { + // OUTPUT nv12 surface(memory) + if (isSwEncoder && (doVkTonemap || isVaapiDecoder)) + { + mainFilters.Add("hwdownload"); + mainFilters.Add("format=nv12"); + } + + if (isSwDecoder && isVaapiEncoder && !doVkTonemap) + { + mainFilters.Add("hwupload_vaapi"); + } + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (hasSubs) + { + if (hasGraphicalSubs) + { + // scale=s=1280x720,format=bgra,hwupload + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + subFilters.Add("format=bgra"); + } + else if (hasTextSubs) + { + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); + } + + subFilters.Add("hwupload=derive_device=vulkan"); + subFilters.Add("format=vulkan"); + + overlayFilters.Add("overlay_vulkan=eof_action=endall:shortest=1:repeatlast=0"); + + if (isSwEncoder) + { + // OUTPUT nv12 surface(memory) + overlayFilters.Add("scale_vulkan=format=nv12"); + overlayFilters.Add("hwdownload"); + overlayFilters.Add("format=nv12"); + } + else if (isVaapiEncoder) + { + // OUTPUT vaapi(nv12) surface(vram) + // map from vulkan/drm to vaapi via interop (Polaris/gfx8+). + overlayFilters.Add("hwmap=derive_device=vaapi"); + overlayFilters.Add("format=vaapi"); + + // clear the surf->meta_offset and output nv12 + overlayFilters.Add("scale_vaapi=format=nv12"); + + // hw deint + if (doDeintH2645) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/2)*2:min(max(iw/dar\\,ih)\\,{0})", - maxHeightParam)); + var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi"); + overlayFilters.Add(deintFilter); } } } - return filters; + return (mainFilters, subFilters, overlayFilters); } - private string GetFixedSizeScalingFilter(Video3DFormat? threedFormat, int requestedWidth, int requestedHeight) + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetVaapiLimitedVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) { - var widthParam = requestedWidth.ToString(CultureInfo.InvariantCulture); - var heightParam = requestedHeight.ToString(CultureInfo.InvariantCulture); + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; - string filter = null; + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isVaapiEncoder; + var isVaInVaOut = isVaapiDecoder && isVaapiEncoder; + var isi965Driver = _mediaEncoder.IsVaapiDeviceInteli965; + var isAmdDriver = _mediaEncoder.IsVaapiDeviceAmd; - if (threedFormat.HasValue) + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doOclTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + + /* Make main filters for video stream */ + var mainFilters = new List<string>(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + + var outFormat = string.Empty; + if (isSwDecoder) { - switch (threedFormat.Value) + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) { - case Video3DFormat.HalfSideBySide: - filter = "crop=iw/2:ih:0:0,scale=(iw*2):ih,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; - // hsbs crop width in half,scale to correct size, set the display aspect,crop out any black bars we may have made the scale width to requestedWidth. Work out the correct height based on the display aspect it will maintain the aspect where -1 in this case (3d) may not. - break; - case Video3DFormat.FullSideBySide: - filter = "crop=iw/2:ih:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; - // fsbs crop width in half,set the display aspect,crop out any black bars we may have made the scale width to requestedWidth. - break; - case Video3DFormat.HalfTopAndBottom: - filter = "crop=iw:ih/2:0:0,scale=(iw*2):ih),setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; - // htab crop height in half,scale to correct size, set the display aspect,crop out any black bars we may have made the scale width to requestedWidth - break; - case Video3DFormat.FullTopAndBottom: - filter = "crop=iw:ih/2:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1,scale={0}:trunc({0}/dar/2)*2"; - // ftab crop height in half, set the display aspect,crop out any black bars we may have made the scale width to requestedWidth - break; - default: - break; + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + outFormat = doOclTonemap ? "yuv420p10le" : "nv12"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload=derive_device=opencl"); } } + else if (isVaapiDecoder) + { + // INPUT vaapi surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi"); + mainFilters.Add(deintFilter); + } - // default - if (filter == null) + outFormat = doOclTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + // allocate extra pool sizes for vaapi vpp + if (!string.IsNullOrEmpty(hwScaleFilter)) + { + hwScaleFilter += ":extra_hw_frames=24"; + } + + // hw scale + mainFilters.Add(hwScaleFilter); + } + + if (doOclTonemap && isVaapiDecoder) { - if (requestedHeight > 0) + if (isi965Driver) { - filter = "scale=trunc({0}/2)*2:trunc({1}/2)*2"; + // map from vaapi to opencl via vaapi-opencl interop(Intel only). + mainFilters.Add("hwmap=derive_device=opencl"); } else { - filter = "scale={0}:trunc({0}/dar/2)*2"; + mainFilters.Add("hwdownload"); + mainFilters.Add("format=p010le"); + mainFilters.Add("hwupload=derive_device=opencl"); } } - return string.Format(CultureInfo.InvariantCulture, filter, widthParam, heightParam); + // ocl tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + if (doOclTonemap && isVaInVaOut) + { + if (isi965Driver) + { + // OUTPUT vaapi(nv12) surface(vram) + // reverse-mapping via vaapi-opencl interop. + mainFilters.Add("hwmap=derive_device=vaapi:reverse=1"); + mainFilters.Add("format=vaapi"); + } + } + + var memoryOutput = false; + var isUploadForOclTonemap = doOclTonemap && (isSwDecoder || (isVaapiDecoder && !isi965Driver)); + var isHwmapNotUsable = hasGraphicalSubs || isUploadForOclTonemap; + var isHwmapForSubs = hasSubs && isVaapiDecoder; + var isHwUnmapForTextSubs = hasTextSubs && isVaInVaOut && !isUploadForOclTonemap; + if ((isVaapiDecoder && isSwEncoder) || isUploadForOclTonemap || isHwmapForSubs) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl/vaapi. + mainFilters.Add(isHwmapNotUsable ? "hwdownload" : "hwmap"); + mainFilters.Add("format=nv12"); + } + + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isVaapiEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (isHwUnmapForTextSubs) + { + mainFilters.Add("hwmap"); + mainFilters.Add("format=vaapi"); + } + else if (memoryOutput && isVaapiEncoder) + { + if (!hasGraphicalSubs) + { + mainFilters.Add("hwupload_vaapi"); + } + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List<string>(); + var overlayFilters = new List<string>(); + if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=pass:shortest=1:repeatlast=0"); + + if (isVaapiEncoder) + { + overlayFilters.Add("hwupload_vaapi"); + } + } + } + + return (mainFilters, subFilters, overlayFilters); } - public string GetOutputSizeParam( + /// <summary> + /// Gets the parameter of Apple VideoToolBox filter chain. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="vidEncoder">Video encoder to use.</param> + /// <returns>The tuple contains three lists: main, sub and overlay filters.</returns> + public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAppleVidFilterChain( EncodingJobInfo state, EncodingOptions options, - string outputVideoCodec) + string vidEncoder) { - string filters = GetOutputSizeParamInternal(state, options, outputVideoCodec); - return string.IsNullOrEmpty(filters) ? string.Empty : " -vf \"" + filters + "\""; + if (!string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder); + + if (!options.EnableHardwareEncoding) + { + return swFilterChain; + } + + if (_mediaEncoder.EncoderVersion.CompareTo(new Version("5.0.0")) < 0) + { + // All features used here requires ffmpeg 5.0 or later, fallback to software filters if using an old ffmpeg + return swFilterChain; + } + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + var newfilters = new List<string>(); + var noOverlay = swFilterChain.OverlayFilters.Count == 0; + var supportsHwDeint = _mediaEncoder.SupportsFilter("yadif_videotoolbox"); + // fallback to software filters if we are using filters not supported by hardware yet. + var useHardwareFilters = noOverlay && (!doDeintH2645 || supportsHwDeint); + + if (!useHardwareFilters) + { + return swFilterChain; + } + + // ffmpeg cannot use videotoolbox to scale + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + newfilters.Add(swScaleFilter); + + // hwupload on videotoolbox encoders can automatically convert AVFrame into its CVPixelBuffer equivalent + // videotoolbox will automatically convert the CVPixelBuffer to a pixel format the encoder supports, so we don't have to set a pixel format explicitly here + // This will reduce CPU usage significantly on UHD videos with 10 bit colors because we bypassed the ffmpeg pixel format conversion + newfilters.Add("hwupload"); + + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox"); + newfilters.Add(deintFilter); + } + + return (newfilters, swFilterChain.SubFilters, swFilterChain.OverlayFilters); } /// <summary> - /// If we're going to put a fixed size on the command line, this will calculate it. + /// Gets the parameter of video processing filters. /// </summary> - public string GetOutputSizeParamInternal( + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="outputVideoCodec">Video codec to use.</param> + /// <returns>The video processing filters parameter.</returns> + public string GetVideoProcessingFilterParam( EncodingJobInfo state, EncodingOptions options, string outputVideoCodec) { - // http://sonnati.wordpress.com/2012/10/19/ffmpeg-the-swiss-army-knife-of-internet-streaming-part-vi/ - - var request = state.BaseRequest; var videoStream = state.VideoStream; - var filters = new List<string>(); + if (videoStream is null) + { + return string.Empty; + } - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options) ?? string.Empty; - var inputWidth = videoStream?.Width; - var inputHeight = videoStream?.Height; - var threeDFormat = state.MediaSource.Video3DFormat; + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; - var isSwDecoder = string.IsNullOrEmpty(videoDecoder); - var isD3d11vaDecoder = videoDecoder.IndexOf("d3d11va", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiDecoder = videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiHevcEncoder = outputVideoCodec.IndexOf("hevc_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvH264Encoder = outputVideoCodec.IndexOf("h264_qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvHevcEncoder = outputVideoCodec.IndexOf("hevc_qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - var isNvencEncoder = outputVideoCodec.Contains("nvenc", StringComparison.OrdinalIgnoreCase); - var isCuvidH264Decoder = videoDecoder.Contains("h264_cuvid", StringComparison.OrdinalIgnoreCase); - var isCuvidHevcDecoder = videoDecoder.Contains("hevc_cuvid", StringComparison.OrdinalIgnoreCase); - var isLibX264Encoder = outputVideoCodec.IndexOf("libx264", StringComparison.OrdinalIgnoreCase) != -1; - var isLibX265Encoder = outputVideoCodec.IndexOf("libx265", StringComparison.OrdinalIgnoreCase) != -1; - var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); - var isColorDepth10 = IsColorDepth10(state); - var isTonemappingSupported = IsTonemappingSupported(state, options); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, options); - var isTonemappingSupportedOnNvenc = string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) && (isNvdecDecoder || isCuvidHevcDecoder || isSwDecoder); - var isTonemappingSupportedOnAmf = string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase) && (isD3d11vaDecoder || isSwDecoder); - var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isVaapiH264Encoder || isVaapiHevcEncoder); - var isTonemappingSupportedOnQsv = string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isQsvH264Encoder || isQsvHevcEncoder); - - var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - - // If double rate deinterlacing is enabled and the input framerate is 30fps or below, otherwise the output framerate will be too high for many devices - var doubleRateDeinterlace = options.DeinterlaceDoubleRate && (videoStream?.AverageFrameRate ?? 60) <= 30; - - var isScalingInAdvance = false; - var isCudaDeintInAdvance = false; - var isHwuploadCudaRequired = false; - var isDeinterlaceH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); - var isDeinterlaceHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); - - // Add OpenCL tonemapping filter for NVENC/AMF/VAAPI. - if (isTonemappingSupportedOnNvenc || isTonemappingSupportedOnAmf || (isTonemappingSupportedOnVaapi && !isVppTonemappingSupported)) - { - // Currently only with the use of NVENC decoder can we get a decent performance. - // Currently only the HEVC/H265 format is supported with NVDEC decoder. - // NVIDIA Pascal and Turing or higher are recommended. - // AMD Polaris and Vega or higher are recommended. - // Intel Kaby Lake or newer is required. - if (isTonemappingSupported) - { - var parameters = "tonemap_opencl=format=nv12:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:desat={1}:threshold={2}:peak={3}"; - - if (options.TonemappingParam != 0) - { - parameters += ":param={4}"; - } + List<string> mainFilters; + List<string> subFilters; + List<string> overlayFilters; - if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase)) - { - parameters += ":range={5}"; - } + if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetVaapiVidFilterChain(state, options, outputVideoCodec); + } + else if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetIntelVidFilterChain(state, options, outputVideoCodec); + } + else if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetNvidiaVidFilterChain(state, options, outputVideoCodec); + } + else if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetAmdVidFilterChain(state, options, outputVideoCodec); + } + else if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetAppleVidFilterChain(state, options, outputVideoCodec); + } + else + { + (mainFilters, subFilters, overlayFilters) = GetSwVidFilterChain(state, options, outputVideoCodec); + } - if (isSwDecoder || isD3d11vaDecoder) - { - isScalingInAdvance = true; - // Add zscale filter before tone mapping filter for performance. - var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight); - if (width.HasValue && height.HasValue) - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "zscale=s={0}x{1}", - width.Value, - height.Value)); - } + mainFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); + subFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); + overlayFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); - // Convert to hardware pixel format p010 when using SW decoder. - filters.Add("format=p010"); - } + var mainStr = string.Empty; + if (mainFilters?.Count > 0) + { + mainStr = string.Format( + CultureInfo.InvariantCulture, + "{0}", + string.Join(',', mainFilters)); + } - if ((isDeinterlaceH264 || isDeinterlaceHevc) && isNvdecDecoder) - { - isCudaDeintInAdvance = true; - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "yadif_cuda={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); - } + if (overlayFilters?.Count == 0) + { + // -vf "scale..." + return string.IsNullOrEmpty(mainStr) ? string.Empty : " -vf \"" + mainStr + "\""; + } - if (isVaapiDecoder || isNvdecDecoder) - { - isScalingInAdvance = true; - filters.AddRange( - GetScalingFilters( - state, - options, - inputWidth, - inputHeight, - threeDFormat, - videoDecoder, - outputVideoCodec, - request.Width, - request.Height, - request.MaxWidth, - request.MaxHeight)); - } + if (overlayFilters?.Count > 0 + && subFilters?.Count > 0 + && state.SubtitleStream is not null) + { + // overlay graphical/text subtitles + var subStr = string.Format( + CultureInfo.InvariantCulture, + "{0}", + string.Join(',', subFilters)); - // hwmap the HDR data to opencl device by cl-va p010 interop. - if (isVaapiDecoder) - { - filters.Add("hwmap"); - } + var overlayStr = string.Format( + CultureInfo.InvariantCulture, + "{0}", + string.Join(',', overlayFilters)); + + var mapPrefix = Convert.ToInt32(state.SubtitleStream.IsExternal); + var subtitleStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.SubtitleStream); + var videoStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.VideoStream); + + if (hasSubs) + { + // -filter_complex "[0:s]scale=s[sub]..." + var filterStr = string.IsNullOrEmpty(mainStr) + ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]{5}\"" + : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[main];[main][sub]{5}\""; - // convert cuda device data to p010 host data. - if (isNvdecDecoder) + if (hasTextSubs) { - filters.Add("hwdownload,format=p010"); + filterStr = string.IsNullOrEmpty(mainStr) + ? " -filter_complex \"{4}[sub];[0:{2}][sub]{5}\"" + : " -filter_complex \"{4}[sub];[0:{2}]{3}[main];[main][sub]{5}\""; } - if (isNvdecDecoder || isCuvidHevcDecoder || isSwDecoder || isD3d11vaDecoder) + return string.Format( + CultureInfo.InvariantCulture, + filterStr, + mapPrefix, + subtitleStreamIndex, + videoStreamIndex, + mainStr, + subStr, + overlayStr); + } + } + + return string.Empty; + } + + public string GetOverwriteColorPropertiesParam(EncodingJobInfo state, bool isTonemapAvailable) + { + if (isTonemapAvailable) + { + return GetInputHdrParam(state.VideoStream?.ColorTransfer); + } + + return GetOutputSdrParam(null); + } + + public string GetInputHdrParam(string colorTransfer) + { + if (string.Equals(colorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)) + { + // HLG + return "setparams=color_primaries=bt2020:color_trc=arib-std-b67:colorspace=bt2020nc"; + } + + // HDR10 + return "setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc"; + } + + public string GetOutputSdrParam(string tonemappingRange) + { + // SDR + if (string.Equals(tonemappingRange, "tv", StringComparison.OrdinalIgnoreCase)) + { + return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv"; + } + + if (string.Equals(tonemappingRange, "pc", StringComparison.OrdinalIgnoreCase)) + { + return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=pc"; + } + + return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709"; + } + + public static int GetVideoColorBitDepth(EncodingJobInfo state) + { + var videoStream = state.VideoStream; + if (videoStream is not null) + { + if (videoStream.BitDepth.HasValue) + { + return videoStream.BitDepth.Value; + } + + if (string.Equals(videoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuvj420p", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p", StringComparison.OrdinalIgnoreCase)) + { + return 8; + } + + if (string.Equals(videoStream.PixelFormat, "yuv420p10le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p10le", StringComparison.OrdinalIgnoreCase)) + { + return 10; + } + + if (string.Equals(videoStream.PixelFormat, "yuv420p12le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p12le", StringComparison.OrdinalIgnoreCase)) + { + return 12; + } + + return 8; + } + + return 0; + } + + /// <summary> + /// Gets the ffmpeg option string for the hardware accelerated video decoder. + /// </summary> + /// <param name="state">The encoding job info.</param> + /// <param name="options">The encoding options.</param> + /// <returns>The option string or null if none available.</returns> + protected string GetHardwareVideoDecoder(EncodingJobInfo state, EncodingOptions options) + { + var videoStream = state.VideoStream; + var mediaSource = state.MediaSource; + if (videoStream is null || mediaSource is null) + { + return null; + } + + // HWA decoders can handle both video files and video folders. + var videoType = state.VideoType; + if (videoType != VideoType.VideoFile + && videoType != VideoType.Iso + && videoType != VideoType.Dvd + && videoType != VideoType.BluRay) + { + return null; + } + + if (IsCopyCodec(state.OutputVideoCodec)) + { + return null; + } + + if (!string.IsNullOrEmpty(videoStream.Codec) && !string.IsNullOrEmpty(options.HardwareAccelerationType)) + { + var bitDepth = GetVideoColorBitDepth(state); + + // Only HEVC, VP9 and AV1 formats have 10-bit hardware decoder support now. + if (bitDepth == 10 + && !(string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "av1", StringComparison.OrdinalIgnoreCase))) + { + return null; + } + + if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + return GetQsvHwVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + return GetNvdecVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + return GetAmfVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + return GetVaapiVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + { + return GetVideotoolboxVidDecoder(state, options, videoStream, bitDepth); + } + } + + var whichCodec = videoStream.Codec; + if (string.Equals(whichCodec, "avc", StringComparison.OrdinalIgnoreCase)) + { + whichCodec = "h264"; + } + else if (string.Equals(whichCodec, "h265", StringComparison.OrdinalIgnoreCase)) + { + whichCodec = "hevc"; + } + + // Avoid a second attempt if no hardware acceleration is being used + options.HardwareDecodingCodecs = Array.FindAll(options.HardwareDecodingCodecs, val => !string.Equals(val, whichCodec, StringComparison.OrdinalIgnoreCase)); + + // leave blank so ffmpeg will decide + return null; + } + + /// <summary> + /// Gets a hw decoder name. + /// </summary> + /// <param name="options">Encoding options.</param> + /// <param name="decoderPrefix">Decoder prefix.</param> + /// <param name="decoderSuffix">Decoder suffix.</param> + /// <param name="videoCodec">Video codec to use.</param> + /// <param name="bitDepth">Video color bit depth.</param> + /// <returns>Hardware decoder name.</returns> + public string GetHwDecoderName(EncodingOptions options, string decoderPrefix, string decoderSuffix, string videoCodec, int bitDepth) + { + if (string.IsNullOrEmpty(decoderPrefix) || string.IsNullOrEmpty(decoderSuffix)) + { + return null; + } + + var decoderName = decoderPrefix + '_' + decoderSuffix; + + var isCodecAvailable = _mediaEncoder.SupportsDecoder(decoderName) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparison.OrdinalIgnoreCase); + if (bitDepth == 10 && isCodecAvailable) + { + if (string.Equals(videoCodec, "hevc", StringComparison.OrdinalIgnoreCase) + && options.HardwareDecodingCodecs.Contains("hevc", StringComparison.OrdinalIgnoreCase) + && !options.EnableDecodingColorDepth10Hevc) + { + return null; + } + + if (string.Equals(videoCodec, "vp9", StringComparison.OrdinalIgnoreCase) + && options.HardwareDecodingCodecs.Contains("vp9", StringComparison.OrdinalIgnoreCase) + && !options.EnableDecodingColorDepth10Vp9) + { + return null; + } + } + + if (string.Equals(decoderSuffix, "cuvid", StringComparison.OrdinalIgnoreCase) && options.EnableEnhancedNvdecDecoder) + { + return null; + } + + if (string.Equals(decoderSuffix, "qsv", StringComparison.OrdinalIgnoreCase) && options.PreferSystemNativeHwDecoder) + { + return null; + } + + return isCodecAvailable ? (" -c:v " + decoderName) : null; + } + + /// <summary> + /// Gets a hwaccel type to use as a hardware decoder depending on the system. + /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="options">Encoding options.</param> + /// <param name="videoCodec">Video codec to use.</param> + /// <param name="bitDepth">Video color bit depth.</param> + /// <param name="outputHwSurface">Specifies if output hw surface.</param> + /// <returns>Hardware accelerator type.</returns> + public string GetHwaccelType(EncodingJobInfo state, EncodingOptions options, string videoCodec, int bitDepth, bool outputHwSurface) + { + var isWindows = OperatingSystem.IsWindows(); + var isLinux = OperatingSystem.IsLinux(); + var isMacOS = OperatingSystem.IsMacOS(); + var isD3d11Supported = isWindows && _mediaEncoder.SupportsHwaccel("d3d11va"); + var isVaapiSupported = isLinux && IsVaapiSupported(state); + var isCudaSupported = (isLinux || isWindows) && IsCudaFullSupported(); + var isQsvSupported = (isLinux || isWindows) && _mediaEncoder.SupportsHwaccel("qsv"); + var isVideotoolboxSupported = isMacOS && _mediaEncoder.SupportsHwaccel("videotoolbox"); + var isCodecAvailable = options.HardwareDecodingCodecs.Contains(videoCodec, StringComparison.OrdinalIgnoreCase); + + var ffmpegVersion = _mediaEncoder.EncoderVersion; + + // Set the av1 codec explicitly to trigger hw accelerator, otherwise libdav1d will be used. + var isAv1 = ffmpegVersion < _minFFmpegImplictHwaccel + && string.Equals(videoCodec, "av1", StringComparison.OrdinalIgnoreCase); + + // Allow profile mismatch if decoding H.264 baseline with d3d11va and vaapi hwaccels. + var profileMismatch = string.Equals(videoCodec, "h264", StringComparison.OrdinalIgnoreCase) + && string.Equals(state.VideoStream?.Profile, "baseline", StringComparison.OrdinalIgnoreCase); + + // Disable the extra internal copy in nvdec. We already handle it in filter chain. + var nvdecNoInternalCopy = ffmpegVersion >= _minFFmpegHwaUnsafeOutput; + + if (bitDepth == 10 && isCodecAvailable) + { + if (string.Equals(videoCodec, "hevc", StringComparison.OrdinalIgnoreCase) + && options.HardwareDecodingCodecs.Contains("hevc", StringComparison.OrdinalIgnoreCase) + && !options.EnableDecodingColorDepth10Hevc) + { + return null; + } + + if (string.Equals(videoCodec, "vp9", StringComparison.OrdinalIgnoreCase) + && options.HardwareDecodingCodecs.Contains("vp9", StringComparison.OrdinalIgnoreCase) + && !options.EnableDecodingColorDepth10Vp9) + { + return null; + } + } + + // Intel qsv/d3d11va/vaapi + if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + if (options.PreferSystemNativeHwDecoder) + { + if (isVaapiSupported && isCodecAvailable) { - // Upload the HDR10 or HLG data to the OpenCL device, - // use tonemap_opencl filter for tone mapping, - // and then download the SDR data to memory. - filters.Add("hwupload"); + return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty) + + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - parameters, - options.TonemappingAlgorithm, - options.TonemappingDesat, - options.TonemappingThreshold, - options.TonemappingPeak, - options.TonemappingParam, - options.TonemappingRange)); - - if (isNvdecDecoder || isCuvidHevcDecoder || isSwDecoder || isD3d11vaDecoder) + if (isD3d11Supported && isCodecAvailable) { - filters.Add("hwdownload"); - filters.Add("format=nv12"); + // set -threads 3 to intel d3d11va decoder explicitly. Lower threads may result in dead lock. + // on newer devices such as Xe, the larger the init_pool_size, the longer the initialization time for opencl to derive from d3d11. + return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty) + + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + " -threads 3" + (isAv1 ? " -c:v av1" : string.Empty); } - - if (isNvdecDecoder && isNvencEncoder) + } + else + { + if (isQsvSupported && isCodecAvailable) { - isHwuploadCudaRequired = true; + return " -hwaccel qsv" + (outputHwSurface ? " -hwaccel_output_format qsv" : string.Empty); } + } + } - if (isVaapiDecoder) + // Nvidia cuda + if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + if (isCudaSupported && isCodecAvailable) + { + if (options.EnableEnhancedNvdecDecoder) { - // Reverse the data route from opencl to vaapi. - filters.Add("hwmap=derive_device=vaapi:reverse=1"); + // set -threads 1 to nvdec decoder explicitly since it doesn't implement threading support. + return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda" : string.Empty) + + (nvdecNoInternalCopy ? " -hwaccel_flags +unsafe_output" : string.Empty) + " -threads 1" + (isAv1 ? " -c:v av1" : string.Empty); } + + // cuvid decoder doesn't have threading issue. + return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda" : string.Empty); + } + } + + // Amd d3d11va + if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + if (isD3d11Supported && isCodecAvailable) + { + return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty) + + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } } - // When the input may or may not be hardware VAAPI decodable. - if ((isVaapiH264Encoder || isVaapiHevcEncoder) - && !(isTonemappingSupportedOnVaapi && (isTonemappingSupported || isVppTonemappingSupported))) + // Vaapi + if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) + && isVaapiSupported + && isCodecAvailable) { - filters.Add("format=nv12|vaapi"); - filters.Add("hwupload"); + return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty) + + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } - // When burning in graphical subtitles using overlay_qsv, upload videostream to the same qsv context. - else if (isLinux && hasGraphicalSubs && (isQsvH264Encoder || isQsvHevcEncoder) - && !(isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + // Apple videotoolbox + if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase) + && isVideotoolboxSupported + && isCodecAvailable) { - filters.Add("hwupload=extra_hw_frames=64"); + return " -hwaccel videotoolbox" + (outputHwSurface ? " -hwaccel_output_format videotoolbox_vld" : string.Empty); } - // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first. - else if ((IsVaapiSupported(state) && isVaapiDecoder) && (isLibX264Encoder || isLibX265Encoder) - && !(isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + return null; + } + + public string GetQsvHwVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + var isWindows = OperatingSystem.IsWindows(); + var isLinux = OperatingSystem.IsLinux(); + + if ((!isWindows && !isLinux) + || !string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) { - var codec = videoStream.Codec; + return null; + } - // Assert 10-bit hardware VAAPI decodable - if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase))) + var isQsvOclSupported = _mediaEncoder.SupportsHwaccel("qsv") && IsOpenclFullSupported(); + var isIntelDx11OclSupported = isWindows + && _mediaEncoder.SupportsHwaccel("d3d11va") + && isQsvOclSupported; + var isIntelVaapiOclSupported = isLinux + && IsVaapiSupported(state) + && isQsvOclSupported; + var hwSurface = (isIntelDx11OclSupported || isIntelVaapiOclSupported) + && _mediaEncoder.SupportsFilter("alphasrc"); + + var is8bitSwFormatsQsv = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsQsv = is8bitSwFormatsQsv || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + // TODO: add more 8/10bit and 4:4:4 formats for Qsv after finishing the ffcheck tool + + if (is8bitSwFormatsQsv) + { + if (string.Equals(videoStream.Codec, "avc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "h264", StringComparison.OrdinalIgnoreCase)) { - /* - Download data from GPU to CPU as p010le format. - Colorspace conversion is unnecessary here as libx264 will handle it. - If this step is missing, it will fail on AMD but not on intel. - */ - filters.Add("hwdownload"); - filters.Add("format=p010le"); + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface) + GetHwDecoderName(options, "h264", "qsv", "h264", bitDepth); } - // Assert 8-bit hardware VAAPI decodable - else if (!isColorDepth10) + if (string.Equals(videoStream.Codec, "vc1", StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwdownload"); - filters.Add("format=nv12"); + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface) + GetHwDecoderName(options, "vc1", "qsv", "vc1", bitDepth); } - } - // Add hardware deinterlace filter before scaling filter. - if (isDeinterlaceH264 || isDeinterlaceHevc) - { - if (isVaapiEncoder - || (isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + if (string.Equals(videoStream.Codec, "vp8", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "deinterlace_vaapi=rate={0}", - doubleRateDeinterlace ? "field" : "frame")); + return GetHwaccelType(state, options, "vp8", bitDepth, hwSurface) + GetHwDecoderName(options, "vp8", "qsv", "vp8", bitDepth); } - else if (isNvdecDecoder && !isCudaDeintInAdvance) + + if (string.Equals(videoStream.Codec, "mpeg2video", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "yadif_cuda={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface) + GetHwDecoderName(options, "mpeg2", "qsv", "mpeg2video", bitDepth); } } - // Add software deinterlace filter before scaling filter. - if ((isDeinterlaceH264 || isDeinterlaceHevc) - && !isVaapiH264Encoder - && !isVaapiHevcEncoder - && !isQsvH264Encoder - && !isQsvHevcEncoder - && !isNvdecDecoder - && !isCuvidH264Decoder) + if (is8_10bitSwFormatsQsv) { - if (string.Equals(options.DeinterlaceMethod, "bwdif", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "bwdif={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "qsv", "hevc", bitDepth); } - else + + if (string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "yadif={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface) + GetHwDecoderName(options, "vp9", "qsv", "vp9", bitDepth); + } + + if (string.Equals(videoStream.Codec, "av1", StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface) + GetHwDecoderName(options, "av1", "qsv", "av1", bitDepth); } } - // Add scaling filter: scale_*=format=nv12 or scale_*=w=*:h=*:format=nv12 or scale=expr - if (!isScalingInAdvance) + return null; + } + + public string GetNvdecVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || !string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) { - filters.AddRange( - GetScalingFilters( - state, - options, - inputWidth, - inputHeight, - threeDFormat, - videoDecoder, - outputVideoCodec, - request.Width, - request.Height, - request.MaxWidth, - request.MaxHeight)); + return null; } - // Add VPP tonemapping filter for VAAPI. - // Full hardware based video post processing, faster than OpenCL but lacks fine tuning options. - if ((isTonemappingSupportedOnVaapi || isTonemappingSupportedOnQsv) - && isVppTonemappingSupported) + var hwSurface = IsCudaFullSupported() && _mediaEncoder.SupportsFilter("alphasrc"); + var is8bitSwFormatsNvdec = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsNvdec = is8bitSwFormatsNvdec || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + // TODO: add more 8/10/12bit and 4:4:4 formats for Nvdec after finishing the ffcheck tool + + if (is8bitSwFormatsNvdec) { - filters.Add("tonemap_vaapi=format=nv12:transfer=bt709:matrix=bt709:primaries=bt709"); + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface) + GetHwDecoderName(options, "h264", "cuvid", "h264", bitDepth); + } + + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface) + GetHwDecoderName(options, "mpeg2", "cuvid", "mpeg2video", bitDepth); + } + + if (string.Equals("vc1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface) + GetHwDecoderName(options, "vc1", "cuvid", "vc1", bitDepth); + } + + if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg4", bitDepth, hwSurface) + GetHwDecoderName(options, "mpeg4", "cuvid", "mpeg4", bitDepth); + } + + if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp8", bitDepth, hwSurface) + GetHwDecoderName(options, "vp8", "cuvid", "vp8", bitDepth); + } } - // Another case is when using Nvenc decoder. - if (isNvdecDecoder && !isTonemappingSupported) + if (is8_10bitSwFormatsNvdec) { - var codec = videoStream.Codec; - var isCudaFormatConversionSupported = _mediaEncoder.SupportsFilter("scale_cuda", "Output format (default \"same\")"); + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "cuvid", "hevc", bitDepth); + } - // Assert 10-bit hardware decodable - if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase))) + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - if (isCudaFormatConversionSupported) - { - if (isLibX264Encoder || isLibX265Encoder || hasSubs) - { - if (isNvencEncoder) - { - isHwuploadCudaRequired = true; - } + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface) + GetHwDecoderName(options, "vp9", "cuvid", "vp9", bitDepth); + } - filters.Add("hwdownload"); - filters.Add("format=nv12"); - } - } - else - { - // Download data from GPU to CPU as p010 format. - filters.Add("hwdownload"); - filters.Add("format=p010"); + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface) + GetHwDecoderName(options, "av1", "cuvid", "av1", bitDepth); + } + } - // Cuda lacks of a pixel format converter. - if (isNvencEncoder) - { - isHwuploadCudaRequired = true; - filters.Add("format=yuv420p"); - } - } + return null; + } + + public string GetAmfVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if (!OperatingSystem.IsWindows() + || !string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var hwSurface = _mediaEncoder.SupportsHwaccel("d3d11va") + && IsOpenclFullSupported() + && _mediaEncoder.SupportsFilter("alphasrc"); + var is8bitSwFormatsAmf = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsAmf = is8bitSwFormatsAmf || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + + if (is8bitSwFormatsAmf) + { + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface); } - // Assert 8-bit hardware decodable - else if (!isColorDepth10 && (isLibX264Encoder || isLibX265Encoder || hasSubs)) + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - if (isNvencEncoder) - { - isHwuploadCudaRequired = true; - } + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface); + } - filters.Add("hwdownload"); - filters.Add("format=nv12"); + if (string.Equals("vc1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface); } } - // Add parameters to use VAAPI with burn-in text subtitles (GH issue #642) - if (isVaapiH264Encoder - || isVaapiHevcEncoder - || (isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + if (is8_10bitSwFormatsAmf) { - if (hasTextSubs) + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - // Convert hw context from ocl to va. - // For tonemapping and text subs burn-in. - if (isTonemappingSupportedOnVaapi && isTonemappingSupported && !isVppTonemappingSupported) - { - filters.Add("scale_vaapi"); - } + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface); + } + + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface); + } - // Test passed on Intel and AMD gfx - filters.Add("hwmap=mode=read+write"); - filters.Add("format=nv12"); + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface); } } - if (hasTextSubs) + return null; + } + + public string GetVaapiVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if (!OperatingSystem.IsLinux() + || !string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var hwSurface = IsVaapiSupported(state) + && IsVaapiFullSupported() + && IsOpenclFullSupported() + && _mediaEncoder.SupportsFilter("alphasrc"); + var is8bitSwFormatsVaapi = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsVaapi = is8bitSwFormatsVaapi || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + + if (is8bitSwFormatsVaapi) { - var subParam = GetTextSubtitleParam(state); + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface); + } - filters.Add(subParam); + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface); + } - // Ensure proper filters are passed to ffmpeg in case of hardware acceleration via VA-API - // Reference: https://trac.ffmpeg.org/wiki/Hardware/VAAPI - if (isVaapiH264Encoder || isVaapiHevcEncoder) + if (string.Equals("vc1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwmap"); + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface); } - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported) + if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwmap,format=vaapi"); + return GetHwaccelType(state, options, "vp8", bitDepth, hwSurface); } + } - if (isNvdecDecoder && isNvencEncoder) + if (is8_10bitSwFormatsVaapi) + { + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - isHwuploadCudaRequired = true; + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface); + } + + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface); + } + + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface); } } - // Interop the VAAPI data to QSV for hybrid tonemapping - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported && !hasGraphicalSubs) + return null; + } + + public string GetVideotoolboxVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if (!OperatingSystem.IsMacOS() + || !string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwmap=derive_device=qsv,scale_qsv"); + return null; } - if (isHwuploadCudaRequired && !hasGraphicalSubs) + var is8bitSwFormatsVt = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + + if (is8bitSwFormatsVt) { - filters.Add("hwupload_cuda"); + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, false); + } + + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg2video", bitDepth, false); + } + + if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg4", bitDepth, false); + } } - var output = string.Empty; - if (filters.Count > 0) + if (is8_10bitSwFormatsVt) { - output += string.Format( - CultureInfo.InvariantCulture, - "{0}", - string.Join(',', filters)); + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, false); + } + + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp9", bitDepth, false); + } } - return output; + return null; } /// <summary> /// Gets the number of threads. /// </summary> + /// <param name="state">Encoding state.</param> + /// <param name="encodingOptions">Encoding options.</param> + /// <param name="outputVideoCodec">Video codec to use.</param> + /// <returns>Number of threads.</returns> #nullable enable public static int GetNumberOfThreads(EncodingJobInfo? state, EncodingOptions encodingOptions, string? outputVideoCodec) { - if (string.Equals(outputVideoCodec, "libvpx", StringComparison.OrdinalIgnoreCase)) - { - // per docs: - // -threads number of threads to use for encoding, can't be 0 [auto] with VP8 - // (recommended value : number of real cores - 1) - return Math.Max(Environment.ProcessorCount - 1, 1); - } + // VP8 and VP9 encoders must have their thread counts set. + bool mustSetThreadCount = string.Equals(outputVideoCodec, "libvpx", StringComparison.OrdinalIgnoreCase) + || string.Equals(outputVideoCodec, "libvpx-vp9", StringComparison.OrdinalIgnoreCase); var threads = state?.BaseRequest.CpuCoreLimit ?? encodingOptions.EncodingThreadCount; - // Automatic if (threads <= 0) { - return 0; + // Automatically set thread count + return mustSetThreadCount ? Math.Max(Environment.ProcessorCount - 1, 1) : 0; } - else if (threads >= Environment.ProcessorCount) + + if (threads >= Environment.ProcessorCount) { return Environment.ProcessorCount; } @@ -2938,7 +5650,7 @@ namespace MediaBrowser.Controller.MediaEncoding #nullable disable public void TryStreamCopy(EncodingJobInfo state) { - if (state.VideoStream != null && CanStreamCopyVideo(state, state.VideoStream)) + if (state.VideoStream is not null && CanStreamCopyVideo(state, state.VideoStream)) { state.OutputVideoCodec = "copy"; } @@ -2947,13 +5659,13 @@ namespace MediaBrowser.Controller.MediaEncoding var user = state.User; // If the user doesn't have access to transcoding, then force stream copy, regardless of whether it will be compatible or not - if (user != null && !user.HasPermission(PermissionKind.EnableVideoPlaybackTranscoding)) + if (user is not null && !user.HasPermission(PermissionKind.EnableVideoPlaybackTranscoding)) { state.OutputVideoCodec = "copy"; } } - if (state.AudioStream != null + if (state.AudioStream is not null && CanStreamCopyAudio(state, state.AudioStream, state.SupportedAudioCodecs)) { state.OutputAudioCodec = "copy"; @@ -2963,31 +5675,30 @@ namespace MediaBrowser.Controller.MediaEncoding var user = state.User; // If the user doesn't have access to transcoding, then force stream copy, regardless of whether it will be compatible or not - if (user != null && !user.HasPermission(PermissionKind.EnableAudioPlaybackTranscoding)) + if (user is not null && !user.HasPermission(PermissionKind.EnableAudioPlaybackTranscoding)) { state.OutputAudioCodec = "copy"; } } } - public string GetInputModifier(EncodingJobInfo state, EncodingOptions encodingOptions) + public string GetInputModifier(EncodingJobInfo state, EncodingOptions encodingOptions, string segmentContainer) { var inputModifier = string.Empty; - var probeSizeArgument = string.Empty; + var analyzeDurationArgument = string.Empty; - string analyzeDurationArgument; - if (state.MediaSource.AnalyzeDurationMs.HasValue) + // Apply -analyzeduration as per the environment variable, + // otherwise ffmpeg will break on certain files due to default value is 0. + // The default value of -probesize is more than enough, so leave it as is. + var ffmpegAnalyzeDuration = _config.GetFFmpegAnalyzeDuration() ?? string.Empty; + + if (state.MediaSource.AnalyzeDurationMs > 0) { analyzeDurationArgument = "-analyzeduration " + (state.MediaSource.AnalyzeDurationMs.Value * 1000).ToString(CultureInfo.InvariantCulture); } - else + else if (!string.IsNullOrEmpty(ffmpegAnalyzeDuration)) { - analyzeDurationArgument = string.Empty; - } - - if (!string.IsNullOrEmpty(probeSizeArgument)) - { - inputModifier += " " + probeSizeArgument; + analyzeDurationArgument = "-analyzeduration " + ffmpegAnalyzeDuration; } if (!string.IsNullOrEmpty(analyzeDurationArgument)) @@ -3006,12 +5717,21 @@ namespace MediaBrowser.Controller.MediaEncoding inputModifier = inputModifier.Trim(); - inputModifier += " " + GetFastSeekCommandLineParameter(state.BaseRequest); + var refererParam = GetRefererParam(state); + + if (!string.IsNullOrEmpty(refererParam)) + { + inputModifier += " " + refererParam; + } + + inputModifier = inputModifier.Trim(); + + inputModifier += " " + GetFastSeekCommandLineParameter(state, encodingOptions, segmentContainer); inputModifier = inputModifier.Trim(); if (state.InputProtocol == MediaProtocol.Rtsp) { - inputModifier += " -rtsp_transport tcp -rtsp_transport udp -rtsp_flags prefer_tcp"; + inputModifier += " -rtsp_transport tcp+udp -rtsp_flags prefer_tcp"; } if (!string.IsNullOrEmpty(state.InputAudioSync)) @@ -3060,61 +5780,8 @@ namespace MediaBrowser.Controller.MediaEncoding inputModifier += " -fflags " + string.Join(string.Empty, flags); } - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions); - - if (!string.IsNullOrEmpty(videoDecoder)) - { - inputModifier += " " + videoDecoder; - - if (!IsCopyCodec(state.OutputVideoCodec) - && (videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1) - { - var videoStream = state.VideoStream; - var inputWidth = videoStream?.Width; - var inputHeight = videoStream?.Height; - var request = state.BaseRequest; - - var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight); - - if ((videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1 - && width.HasValue - && height.HasValue) - { - if (width.HasValue && height.HasValue) - { - inputModifier += string.Format( - CultureInfo.InvariantCulture, - " -resize {0}x{1}", - width.Value, - height.Value); - } - - if (state.DeInterlace("h264", true)) - { - inputModifier += " -deint 1"; - - if (!encodingOptions.DeinterlaceDoubleRate || (videoStream?.AverageFrameRate ?? 60) > 30) - { - inputModifier += " -drop_second_field 1"; - } - } - } - } - } - if (state.IsVideoRequest) { - var outputVideoCodec = GetVideoEncoder(state, encodingOptions); - - // Important: If this is ever re-enabled, make sure not to use it with wtv because it breaks seeking - if (!string.Equals(state.InputContainer, "wtv", StringComparison.OrdinalIgnoreCase) - && state.TranscodingType != TranscodingJobType.Progressive - && !state.EnableBreakOnNonKeyFrames(outputVideoCodec) - && (state.BaseRequest.StartTimeTicks ?? 0) > 0) - { - inputModifier += " -noaccurate_seek"; - } - if (!string.IsNullOrEmpty(state.InputContainer) && state.VideoType == VideoType.VideoFile && string.IsNullOrEmpty(encodingOptions.HardwareAccelerationType)) { var inputFormat = GetInputFormat(state.InputContainer); @@ -3139,15 +5806,9 @@ namespace MediaBrowser.Controller.MediaEncoding MediaSourceInfo mediaSource, string requestedUrl) { - if (state == null) - { - throw new ArgumentNullException(nameof(state)); - } + ArgumentNullException.ThrowIfNull(state); - if (mediaSource == null) - { - throw new ArgumentNullException(nameof(mediaSource)); - } + ArgumentNullException.ThrowIfNull(mediaSource); var path = mediaSource.Path; var protocol = mediaSource.Protocol; @@ -3211,7 +5872,7 @@ namespace MediaBrowser.Controller.MediaEncoding state.SubtitleDeliveryMethod = videoRequest.SubtitleMethod; state.AudioStream = GetMediaStream(mediaStreams, videoRequest.AudioStreamIndex, MediaStreamType.Audio); - if (state.SubtitleStream != null && !state.SubtitleStream.IsExternal) + if (state.SubtitleStream is not null && !state.SubtitleStream.IsExternal) { state.InternalSubtitleStreamOffset = mediaStreams.Where(i => i.Type == MediaStreamType.Subtitle && !i.IsExternal).ToList().IndexOf(state.SubtitleStream); } @@ -3229,7 +5890,7 @@ namespace MediaBrowser.Controller.MediaEncoding var request = state.BaseRequest; var supportedAudioCodecs = state.SupportedAudioCodecs; - if (request != null && supportedAudioCodecs != null && supportedAudioCodecs.Length > 0) + if (request is not null && supportedAudioCodecs is not null && supportedAudioCodecs.Length > 0) { var supportedAudioCodecsList = supportedAudioCodecs.ToList(); @@ -3242,7 +5903,7 @@ namespace MediaBrowser.Controller.MediaEncoding } var supportedVideoCodecs = state.SupportedVideoCodecs; - if (request != null && supportedVideoCodecs != null && supportedVideoCodecs.Length > 0) + if (request is not null && supportedVideoCodecs is not null && supportedVideoCodecs.Length > 0) { var supportedVideoCodecsList = supportedVideoCodecs.ToList(); @@ -3262,21 +5923,29 @@ namespace MediaBrowser.Controller.MediaEncoding return; } - var inputChannels = audioStream == null ? 6 : audioStream.Channels ?? 6; + var inputChannels = audioStream is null ? 6 : audioStream.Channels ?? 6; + var shiftAudioCodecs = new List<string>(); if (inputChannels >= 6) { - return; + // DTS and TrueHD are not supported by HLS + // Keep them in the supported codecs list, but shift them to the end of the list so that if transcoding happens, another codec is used + shiftAudioCodecs.Add("dca"); + shiftAudioCodecs.Add("truehd"); + } + else + { + // Transcoding to 2ch ac3 or eac3 almost always causes a playback failure + // Keep them in the supported codecs list, but shift them to the end of the list so that if transcoding happens, another codec is used + shiftAudioCodecs.Add("ac3"); + shiftAudioCodecs.Add("eac3"); } - // Transcoding to 2ch ac3 almost always causes a playback failure - // Keep it in the supported codecs list, but shift it to the end of the list so that if transcoding happens, another codec is used - var shiftAudioCodecs = new[] { "ac3", "eac3" }; - if (audioCodecs.All(i => shiftAudioCodecs.Contains(i, StringComparer.OrdinalIgnoreCase))) + if (audioCodecs.All(i => shiftAudioCodecs.Contains(i, StringComparison.OrdinalIgnoreCase))) { return; } - while (shiftAudioCodecs.Contains(audioCodecs[0], StringComparer.OrdinalIgnoreCase)) + while (shiftAudioCodecs.Contains(audioCodecs[0], StringComparison.OrdinalIgnoreCase)) { var removed = shiftAudioCodecs[0]; audioCodecs.RemoveAt(0); @@ -3286,25 +5955,31 @@ namespace MediaBrowser.Controller.MediaEncoding private void ShiftVideoCodecsIfNeeded(List<string> videoCodecs, EncodingOptions encodingOptions) { - // Shift hevc/h265 to the end of list if hevc encoding is not allowed. - if (encodingOptions.AllowHevcEncoding) + // No need to shift if there is only one supported video codec. + if (videoCodecs.Count < 2) { return; } - // No need to shift if there is only one supported video codec. - if (videoCodecs.Count < 2) + // Shift codecs to the end of list if it's not allowed. + var shiftVideoCodecs = new List<string>(); + if (!encodingOptions.AllowHevcEncoding) { - return; + shiftVideoCodecs.Add("hevc"); + shiftVideoCodecs.Add("h265"); } - var shiftVideoCodecs = new[] { "hevc", "h265" }; - if (videoCodecs.All(i => shiftVideoCodecs.Contains(i, StringComparer.OrdinalIgnoreCase))) + if (!encodingOptions.AllowAv1Encoding) + { + shiftVideoCodecs.Add("av1"); + } + + if (videoCodecs.All(i => shiftVideoCodecs.Contains(i, StringComparison.OrdinalIgnoreCase))) { return; } - while (shiftVideoCodecs.Contains(videoCodecs[0], StringComparer.OrdinalIgnoreCase)) + while (shiftVideoCodecs.Contains(videoCodecs[0], StringComparison.OrdinalIgnoreCase)) { var removed = shiftVideoCodecs[0]; videoCodecs.RemoveAt(0); @@ -3314,7 +5989,7 @@ namespace MediaBrowser.Controller.MediaEncoding private void NormalizeSubtitleEmbed(EncodingJobInfo state) { - if (state.SubtitleStream == null || state.SubtitleDeliveryMethod != SubtitleDeliveryMethod.Embed) + if (state.SubtitleStream is null || state.SubtitleDeliveryMethod != SubtitleDeliveryMethod.Embed) { return; } @@ -3327,310 +6002,9 @@ namespace MediaBrowser.Controller.MediaEncoding } } - /// <summary> - /// Gets the ffmpeg option string for the hardware accelerated video decoder. - /// </summary> - /// <param name="state">The encoding job info.</param> - /// <param name="encodingOptions">The encoding options.</param> - /// <returns>The option string or null if none available.</returns> - protected string GetHardwareAcceleratedVideoDecoder(EncodingJobInfo state, EncodingOptions encodingOptions) - { - var videoStream = state.VideoStream; - - if (videoStream == null) - { - return null; - } - - var videoType = state.MediaSource.VideoType ?? VideoType.VideoFile; - // Only use alternative encoders for video files. - // When using concat with folder rips, if the mfx session fails to initialize, ffmpeg will be stuck retrying and will not exit gracefully - // Since transcoding of folder rips is experimental anyway, it's not worth adding additional variables such as this. - if (videoType != VideoType.VideoFile) - { - return null; - } - - if (IsCopyCodec(state.OutputVideoCodec)) - { - return null; - } - - if (!string.IsNullOrEmpty(videoStream.Codec) && !string.IsNullOrEmpty(encodingOptions.HardwareAccelerationType)) - { - var isColorDepth10 = IsColorDepth10(state); - - // Only hevc and vp9 formats have 10-bit hardware decoder support now. - if (isColorDepth10 && !(string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase))) - { - return null; - } - - // Hybrid VPP tonemapping with VAAPI - if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) - && IsVppTonemappingSupported(state, encodingOptions)) - { - // Since tonemap_vaapi only support HEVC for now, no need to check the codec again. - return GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10); - } - - if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_qsv", "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwDecoderName(encodingOptions, "hevc_qsv", "hevc", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_qsv", "mpeg2video", isColorDepth10); - case "vc1": - return GetHwDecoderName(encodingOptions, "vc1_qsv", "vc1", isColorDepth10); - case "vp8": - return GetHwDecoderName(encodingOptions, "vp8_qsv", "vp8", isColorDepth10); - case "vp9": - return GetHwDecoderName(encodingOptions, "vp9_qsv", "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "h264", isColorDepth10) - : GetHwDecoderName(encodingOptions, "h264_cuvid", "h264", isColorDepth10); - case "hevc": - case "h265": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10) - : GetHwDecoderName(encodingOptions, "hevc_cuvid", "hevc", isColorDepth10); - case "mpeg2video": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "mpeg2video", isColorDepth10) - : GetHwDecoderName(encodingOptions, "mpeg2_cuvid", "mpeg2video", isColorDepth10); - case "vc1": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "vc1", isColorDepth10) - : GetHwDecoderName(encodingOptions, "vc1_cuvid", "vc1", isColorDepth10); - case "mpeg4": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "mpeg4", isColorDepth10) - : GetHwDecoderName(encodingOptions, "mpeg4_cuvid", "mpeg4", isColorDepth10); - case "vp8": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "vp8", isColorDepth10) - : GetHwDecoderName(encodingOptions, "vp8_cuvid", "vp8", isColorDepth10); - case "vp9": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "vp9", isColorDepth10) - : GetHwDecoderName(encodingOptions, "vp9_cuvid", "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "mediacodec", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_mediacodec", "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwDecoderName(encodingOptions, "hevc_mediacodec", "hevc", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_mediacodec", "mpeg2video", isColorDepth10); - case "mpeg4": - return GetHwDecoderName(encodingOptions, "mpeg4_mediacodec", "mpeg4", isColorDepth10); - case "vp8": - return GetHwDecoderName(encodingOptions, "vp8_mediacodec", "vp8", isColorDepth10); - case "vp9": - return GetHwDecoderName(encodingOptions, "vp9_mediacodec", "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "omx", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_mmal", "h264", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_mmal", "mpeg2video", isColorDepth10); - case "mpeg4": - return GetHwDecoderName(encodingOptions, "mpeg4_mmal", "mpeg4", isColorDepth10); - case "vc1": - return GetHwDecoderName(encodingOptions, "vc1_mmal", "vc1", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwaccelType(state, encodingOptions, "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10); - case "mpeg2video": - return GetHwaccelType(state, encodingOptions, "mpeg2video", isColorDepth10); - case "vc1": - return GetHwaccelType(state, encodingOptions, "vc1", isColorDepth10); - case "mpeg4": - return GetHwaccelType(state, encodingOptions, "mpeg4", isColorDepth10); - case "vp9": - return GetHwaccelType(state, encodingOptions, "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwaccelType(state, encodingOptions, "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10); - case "mpeg2video": - return GetHwaccelType(state, encodingOptions, "mpeg2video", isColorDepth10); - case "vc1": - return GetHwaccelType(state, encodingOptions, "vc1", isColorDepth10); - case "vp8": - return GetHwaccelType(state, encodingOptions, "vp8", isColorDepth10); - case "vp9": - return GetHwaccelType(state, encodingOptions, "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_opencl", "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwDecoderName(encodingOptions, "hevc_opencl", "hevc", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_opencl", "mpeg2video", isColorDepth10); - case "mpeg4": - return GetHwDecoderName(encodingOptions, "mpeg4_opencl", "mpeg4", isColorDepth10); - case "vc1": - return GetHwDecoderName(encodingOptions, "vc1_opencl", "vc1", isColorDepth10); - case "vp8": - return GetHwDecoderName(encodingOptions, "vp8_opencl", "vp8", isColorDepth10); - case "vp9": - return GetHwDecoderName(encodingOptions, "vp9_opencl", "vp9", isColorDepth10); - } - } - } - - var whichCodec = videoStream.Codec?.ToLowerInvariant(); - switch (whichCodec) - { - case "avc": - whichCodec = "h264"; - break; - case "h265": - whichCodec = "hevc"; - break; - } - - // Avoid a second attempt if no hardware acceleration is being used - encodingOptions.HardwareDecodingCodecs = encodingOptions.HardwareDecodingCodecs.Where(val => val != whichCodec).ToArray(); - - // leave blank so ffmpeg will decide - return null; - } - - /// <summary> - /// Gets a hw decoder name. - /// </summary> - public string GetHwDecoderName(EncodingOptions options, string decoder, string videoCodec, bool isColorDepth10) - { - var isCodecAvailable = _mediaEncoder.SupportsDecoder(decoder) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase); - if (isColorDepth10 && isCodecAvailable) - { - if ((options.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Hevc) - || (options.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Vp9)) - { - return null; - } - } - - return isCodecAvailable ? ("-c:v " + decoder) : null; - } - - /// <summary> - /// Gets a hwaccel type to use as a hardware decoder(dxva/vaapi) depending on the system. - /// </summary> - public string GetHwaccelType(EncodingJobInfo state, EncodingOptions options, string videoCodec, bool isColorDepth10) - { - var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); - var isWindows8orLater = Environment.OSVersion.Version.Major > 6 || (Environment.OSVersion.Version.Major == 6 && Environment.OSVersion.Version.Minor > 1); - var isDxvaSupported = _mediaEncoder.SupportsHwaccel("dxva2") || _mediaEncoder.SupportsHwaccel("d3d11va"); - var isCodecAvailable = options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase); - - if (isColorDepth10 && isCodecAvailable) - { - if ((options.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Hevc) - || (options.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Vp9)) - { - return null; - } - } - - if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) - { - // Currently there is no AMF decoder on Linux, only have h264 encoder. - if (isDxvaSupported && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase)) - { - if (isWindows && isWindows8orLater) - { - return "-hwaccel d3d11va"; - } - - if (isWindows && !isWindows8orLater) - { - return "-hwaccel dxva2"; - } - } - } - - if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) - || (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) - && IsVppTonemappingSupported(state, options))) - { - if (IsVaapiSupported(state) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase)) - { - if (isLinux) - { - return "-hwaccel vaapi"; - } - } - } - - if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) - { - if (options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase)) - { - return "-hwaccel cuda"; - } - } - - return null; - } - public string GetSubtitleEmbedArguments(EncodingJobInfo state) { - if (state.SubtitleStream == null || state.SubtitleDeliveryMethod != SubtitleDeliveryMethod.Embed) + if (state.SubtitleStream is null || state.SubtitleDeliveryMethod != SubtitleDeliveryMethod.Embed) { return string.Empty; } @@ -3662,18 +6036,18 @@ namespace MediaBrowser.Controller.MediaEncoding && state.BaseRequest.Context == EncodingContext.Streaming) { // Comparison: https://github.com/jansmolders86/mediacenterjs/blob/master/lib/transcoding/desktop.js - format = " -f mp4 -movflags frag_keyframe+empty_moov"; + format = " -f mp4 -movflags frag_keyframe+empty_moov+delay_moov"; } var threads = GetNumberOfThreads(state, encodingOptions, videoCodec); - var inputModifier = GetInputModifier(state, encodingOptions); + var inputModifier = GetInputModifier(state, encodingOptions, null); return string.Format( CultureInfo.InvariantCulture, "{0} {1}{2} {3} {4} -map_metadata -1 -map_chapters -1 -threads {5} {6}{7}{8} -y \"{9}\"", inputModifier, - GetInputArgument(state, encodingOptions), + GetInputArgument(state, encodingOptions, null), keyFrame, GetMapArgs(state), GetProgressiveVideoArguments(state, encodingOptions, videoCodec, defaultPreset), @@ -3711,7 +6085,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (IsCopyCodec(videoCodec)) { - if (state.VideoStream != null + if (state.VideoStream is not null && string.Equals(state.OutputContainer, "ts", StringComparison.OrdinalIgnoreCase) && !string.Equals(state.VideoStream.NalLengthSize, "0", StringComparison.OrdinalIgnoreCase)) { @@ -3741,29 +6115,18 @@ namespace MediaBrowser.Controller.MediaEncoding args += keyFrameArg; - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasGraphicalSubs = state.SubtitleStream is not null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; var hasCopyTs = false; - // Add resolution params, if specified - if (!hasGraphicalSubs) - { - var outputSizeParam = GetOutputSizeParam(state, encodingOptions, videoCodec); - - args += outputSizeParam; - - hasCopyTs = outputSizeParam.IndexOf("copyts", StringComparison.OrdinalIgnoreCase) != -1; - } + // video processing filters. + var videoProcessParam = GetVideoProcessingFilterParam(state, encodingOptions, videoCodec); - // This is for graphical subs - if (hasGraphicalSubs) - { - var graphicalSubtitleParam = GetGraphicalSubtitleParam(state, encodingOptions, videoCodec); + var negativeMapArgs = GetNegativeMapArgsByFilters(state, videoProcessParam); - args += graphicalSubtitleParam; + args = negativeMapArgs + args + videoProcessParam; - hasCopyTs = graphicalSubtitleParam.IndexOf("copyts", StringComparison.OrdinalIgnoreCase) != -1; - } + hasCopyTs = videoProcessParam.Contains("copyts", StringComparison.OrdinalIgnoreCase); if (state.RunTimeTicks.HasValue && state.BaseRequest.CopyTimestamps) { @@ -3774,7 +6137,7 @@ namespace MediaBrowser.Controller.MediaEncoding args += " -avoid_negative_ts disabled"; - if (!(state.SubtitleStream != null && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream)) + if (!(state.SubtitleStream is not null && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream)) { args += " -start_at_zero"; } @@ -3801,7 +6164,7 @@ namespace MediaBrowser.Controller.MediaEncoding public string GetProgressiveVideoAudioArguments(EncodingJobInfo state, EncodingOptions encodingOptions) { // If the video doesn't have an audio stream, return a default. - if (state.AudioStream == null && state.VideoStream != null) + if (state.AudioStream is null && state.VideoStream is not null) { return string.Empty; } @@ -3816,27 +6179,33 @@ namespace MediaBrowser.Controller.MediaEncoding return args; } - // Add the number of audio channels var channels = state.OutputAudioChannels; - if (channels.HasValue) + if (channels.HasValue && ((channels.Value != 2 && state.AudioStream.Channels <= 5) || encodingOptions.DownMixStereoAlgorithm == DownMixStereoAlgorithms.None)) { args += " -ac " + channels.Value; } var bitrate = state.OutputAudioBitrate; - - if (bitrate.HasValue) + if (bitrate.HasValue && !LosslessAudioCodecs.Contains(codec, StringComparison.OrdinalIgnoreCase)) { - args += " -ab " + bitrate.Value.ToString(_usCulture); + var vbrParam = GetAudioVbrModeParam(codec, bitrate.Value / (channels ?? 2)); + if (encodingOptions.EnableAudioVbr && vbrParam is not null) + { + args += vbrParam; + } + else + { + args += " -ab " + bitrate.Value.ToString(CultureInfo.InvariantCulture); + } } if (state.OutputAudioSampleRate.HasValue) { - args += " -ar " + state.OutputAudioSampleRate.Value.ToString(_usCulture); + args += " -ar " + state.OutputAudioSampleRate.Value.ToString(CultureInfo.InvariantCulture); } - args += GetAudioFilterParam(state, encodingOptions, false); + args += GetAudioFilterParam(state, encodingOptions); return args; } @@ -3846,35 +6215,67 @@ namespace MediaBrowser.Controller.MediaEncoding var audioTranscodeParams = new List<string>(); var bitrate = state.OutputAudioBitrate; + var channels = state.OutputAudioChannels; + var outputCodec = state.OutputAudioCodec; + + if (bitrate.HasValue && !LosslessAudioCodecs.Contains(outputCodec, StringComparison.OrdinalIgnoreCase)) + { + var vbrParam = GetAudioVbrModeParam(GetAudioEncoder(state), bitrate.Value / (channels ?? 2)); + if (encodingOptions.EnableAudioVbr && vbrParam is not null) + { + audioTranscodeParams.Add(vbrParam); + } + else + { + audioTranscodeParams.Add("-ab " + bitrate.Value.ToString(CultureInfo.InvariantCulture)); + } + } - if (bitrate.HasValue) + if (channels.HasValue) { - audioTranscodeParams.Add("-ab " + bitrate.Value.ToString(_usCulture)); + audioTranscodeParams.Add("-ac " + state.OutputAudioChannels.Value.ToString(CultureInfo.InvariantCulture)); } - if (state.OutputAudioChannels.HasValue) + if (!string.IsNullOrEmpty(outputCodec)) { - audioTranscodeParams.Add("-ac " + state.OutputAudioChannels.Value.ToString(_usCulture)); + audioTranscodeParams.Add("-acodec " + GetAudioEncoder(state)); } - // opus will fail on 44100 - if (!string.Equals(state.OutputAudioCodec, "opus", StringComparison.OrdinalIgnoreCase)) + if (!string.Equals(outputCodec, "opus", StringComparison.OrdinalIgnoreCase)) { - if (state.OutputAudioSampleRate.HasValue) + // opus only supports specific sampling rates + var sampleRate = state.OutputAudioSampleRate; + if (sampleRate.HasValue) { - audioTranscodeParams.Add("-ar " + state.OutputAudioSampleRate.Value.ToString(_usCulture)); + var sampleRateValue = sampleRate.Value switch + { + <= 8000 => 8000, + <= 12000 => 12000, + <= 16000 => 16000, + <= 24000 => 24000, + _ => 48000 + }; + + audioTranscodeParams.Add("-ar " + sampleRateValue.ToString(CultureInfo.InvariantCulture)); } } + // Copy the movflags from GetProgressiveVideoFullCommandLine + // See #9248 and the associated PR for why this is needed + if (_mp4ContainerNames.Contains(state.OutputContainer)) + { + audioTranscodeParams.Add("-movflags empty_moov+delay_moov"); + } + var threads = GetNumberOfThreads(state, encodingOptions, null); - var inputModifier = GetInputModifier(state, encodingOptions); + var inputModifier = GetInputModifier(state, encodingOptions, null); return string.Format( CultureInfo.InvariantCulture, "{0} {1}{7}{8} -threads {2}{3} {4} -id3v2_version 3 -write_id3v1 1{6} -y \"{5}\"", inputModifier, - GetInputArgument(state, encodingOptions), + GetInputArgument(state, encodingOptions, null), threads, " -vn", string.Join(' ', audioTranscodeParams), @@ -3884,46 +6285,31 @@ namespace MediaBrowser.Controller.MediaEncoding string.Empty).Trim(); } - public static bool IsCopyCodec(string codec) - { - return string.Equals(codec, "copy", StringComparison.OrdinalIgnoreCase); - } - - public static bool IsColorDepth10(EncodingJobInfo state) + public static int FindIndex(IReadOnlyList<MediaStream> mediaStreams, MediaStream streamToFind) { - var result = false; - var videoStream = state.VideoStream; + var index = 0; + var length = mediaStreams.Count; - if (videoStream != null) + for (var i = 0; i < length; i++) { - if (!string.IsNullOrEmpty(videoStream.PixelFormat)) + var currentMediaStream = mediaStreams[i]; + if (currentMediaStream == streamToFind) { - result = videoStream.PixelFormat.Contains("p10", StringComparison.OrdinalIgnoreCase); - if (result) - { - return true; - } + return index; } - if (!string.IsNullOrEmpty(videoStream.Profile)) + if (string.Equals(currentMediaStream.Path, streamToFind.Path, StringComparison.Ordinal)) { - result = videoStream.Profile.Contains("Main 10", StringComparison.OrdinalIgnoreCase) - || videoStream.Profile.Contains("High 10", StringComparison.OrdinalIgnoreCase) - || videoStream.Profile.Contains("Profile 2", StringComparison.OrdinalIgnoreCase); - if (result) - { - return true; - } - } - - result = (videoStream.BitDepth ?? 8) == 10; - if (result) - { - return true; + index++; } } - return result; + return -1; + } + + public static bool IsCopyCodec(string codec) + { + return string.Equals(codec, "copy", StringComparison.OrdinalIgnoreCase); } } } |