diff options
Diffstat (limited to 'MediaBrowser.MediaEncoding/Encoder')
3 files changed, 220 insertions, 54 deletions
diff --git a/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs new file mode 100644 index 000000000..a8ff58b09 --- /dev/null +++ b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs @@ -0,0 +1,87 @@ +#pragma warning disable CA1031 + +using System; +using System.Linq; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using Microsoft.Extensions.Logging; + +namespace MediaBrowser.MediaEncoding.Encoder; + +/// <summary> +/// Helper class for Apple platform specific operations. +/// </summary> +[SupportedOSPlatform("macos")] +public static class ApplePlatformHelper +{ + private static readonly string[] _av1DecodeBlacklistedCpuClass = ["M1", "M2"]; + + private static string GetSysctlValue(ReadOnlySpan<byte> name) + { + IntPtr length = IntPtr.Zero; + // Get length of the value + int osStatus = SysctlByName(name, IntPtr.Zero, ref length, IntPtr.Zero, 0); + + if (osStatus != 0) + { + throw new NotSupportedException($"Failed to get sysctl value for {System.Text.Encoding.UTF8.GetString(name)} with error {osStatus}"); + } + + IntPtr buffer = Marshal.AllocHGlobal(length.ToInt32()); + try + { + osStatus = SysctlByName(name, buffer, ref length, IntPtr.Zero, 0); + if (osStatus != 0) + { + throw new NotSupportedException($"Failed to get sysctl value for {System.Text.Encoding.UTF8.GetString(name)} with error {osStatus}"); + } + + return Marshal.PtrToStringAnsi(buffer) ?? string.Empty; + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + + private static int SysctlByName(ReadOnlySpan<byte> name, IntPtr oldp, ref IntPtr oldlenp, IntPtr newp, uint newlen) + { + return NativeMethods.SysctlByName(name.ToArray(), oldp, ref oldlenp, newp, newlen); + } + + /// <summary> + /// Check if the current system has hardware acceleration for AV1 decoding. + /// </summary> + /// <param name="logger">The logger used for error logging.</param> + /// <returns>Boolean indicates the hwaccel support.</returns> + public static bool HasAv1HardwareAccel(ILogger logger) + { + if (!RuntimeInformation.OSArchitecture.Equals(Architecture.Arm64)) + { + return false; + } + + try + { + string cpuBrandString = GetSysctlValue("machdep.cpu.brand_string"u8); + return !_av1DecodeBlacklistedCpuClass.Any(blacklistedCpuClass => cpuBrandString.Contains(blacklistedCpuClass, StringComparison.OrdinalIgnoreCase)); + } + catch (NotSupportedException e) + { + logger.LogError("Error getting CPU brand string: {Message}", e.Message); + } + catch (Exception e) + { + logger.LogError("Unknown error occured: {Exception}", e); + } + + return false; + } + + private static class NativeMethods + { + [DllImport("libc", EntryPoint = "sysctlbyname", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] + internal static extern int SysctlByName(byte[] name, IntPtr oldp, ref IntPtr oldlenp, IntPtr newp, uint newlen); + } +} diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs index 23d9ca7ef..f4e8c39c1 100644 --- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs +++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs @@ -5,15 +5,17 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; +using System.Runtime.Versioning; using System.Text.RegularExpressions; +using MediaBrowser.Controller.MediaEncoding; using Microsoft.Extensions.Logging; namespace MediaBrowser.MediaEncoding.Encoder { public partial class EncoderValidator { - private static readonly string[] _requiredDecoders = new[] - { + private static readonly string[] _requiredDecoders = + [ "h264", "hevc", "vp8", @@ -55,10 +57,10 @@ namespace MediaBrowser.MediaEncoding.Encoder "vp8_rkmpp", "vp9_rkmpp", "av1_rkmpp" - }; + ]; - private static readonly string[] _requiredEncoders = new[] - { + private static readonly string[] _requiredEncoders = + [ "libx264", "libx265", "libsvtav1", @@ -95,10 +97,10 @@ namespace MediaBrowser.MediaEncoding.Encoder "h264_rkmpp", "hevc_rkmpp", "mjpeg_rkmpp" - }; + ]; - private static readonly string[] _requiredFilters = new[] - { + private static readonly string[] _requiredFilters = + [ // sw "alphasrc", "zscale", @@ -121,6 +123,8 @@ namespace MediaBrowser.MediaEncoding.Encoder "tonemap_opencl", "overlay_opencl", "transpose_opencl", + "yadif_opencl", + "bwdif_opencl", // vaapi "scale_vaapi", "deinterlace_vaapi", @@ -146,17 +150,28 @@ namespace MediaBrowser.MediaEncoding.Encoder "scale_rkrga", "vpp_rkrga", "overlay_rkrga" + ]; + + private static readonly Dictionary<FilterOptionType, (string, string)> _filterOptionsDict = new Dictionary<FilterOptionType, (string, string)> + { + { FilterOptionType.ScaleCudaFormat, ("scale_cuda", "format") }, + { FilterOptionType.TonemapCudaName, ("tonemap_cuda", "GPU accelerated HDR to SDR tonemapping") }, + { FilterOptionType.TonemapOpenclBt2390, ("tonemap_opencl", "bt2390") }, + { FilterOptionType.OverlayOpenclFrameSync, ("overlay_opencl", "Action to take when encountering EOF from secondary input") }, + { FilterOptionType.OverlayVaapiFrameSync, ("overlay_vaapi", "Action to take when encountering EOF from secondary input") }, + { FilterOptionType.OverlayVulkanFrameSync, ("overlay_vulkan", "Action to take when encountering EOF from secondary input") }, + { FilterOptionType.TransposeOpenclReversal, ("transpose_opencl", "rotate by half-turn") }, + { FilterOptionType.OverlayOpenclAlphaFormat, ("overlay_opencl", "alpha_format") }, + { FilterOptionType.OverlayCudaAlphaFormat, ("overlay_cuda", "alpha_format") } }; - private static readonly Dictionary<int, string[]> _filterOptionsDict = new Dictionary<int, string[]> + private static readonly Dictionary<BitStreamFilterOptionType, (string, string)> _bsfOptionsDict = new Dictionary<BitStreamFilterOptionType, (string, string)> { - { 0, new string[] { "scale_cuda", "format" } }, - { 1, new string[] { "tonemap_cuda", "GPU accelerated HDR to SDR tonemapping" } }, - { 2, new string[] { "tonemap_opencl", "bt2390" } }, - { 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } }, - { 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } }, - { 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } }, - { 6, new string[] { "transpose_opencl", "rotate by half-turn" } } + { BitStreamFilterOptionType.HevcMetadataRemoveDovi, ("hevc_metadata", "remove_dovi") }, + { BitStreamFilterOptionType.HevcMetadataRemoveHdr10Plus, ("hevc_metadata", "remove_hdr10plus") }, + { BitStreamFilterOptionType.Av1MetadataRemoveDovi, ("av1_metadata", "remove_dovi") }, + { BitStreamFilterOptionType.Av1MetadataRemoveHdr10Plus, ("av1_metadata", "remove_hdr10plus") }, + { BitStreamFilterOptionType.DoviRpuStrip, ("dovi_rpu", "strip") } }; // These are the library versions that corresponds to our minimum ffmpeg version 4.4 according to the version table below @@ -283,7 +298,11 @@ namespace MediaBrowser.MediaEncoding.Encoder public IEnumerable<string> GetFilters() => GetFFmpegFilters(); - public IDictionary<int, bool> GetFiltersWithOption() => GetFFmpegFiltersWithOption(); + public IDictionary<FilterOptionType, bool> GetFiltersWithOption() => _filterOptionsDict + .ToDictionary(item => item.Key, item => CheckFilterWithOption(item.Value.Item1, item.Value.Item2)); + + public IDictionary<BitStreamFilterOptionType, bool> GetBitStreamFiltersWithOption() => _bsfOptionsDict + .ToDictionary(item => item.Key, item => CheckBitStreamFilterWithOption(item.Value.Item1, item.Value.Item2)); public Version? GetFFmpegVersion() { @@ -437,6 +456,12 @@ namespace MediaBrowser.MediaEncoding.Encoder } } + [SupportedOSPlatform("macos")] + public bool CheckIsVideoToolboxAv1DecodeAvailable() + { + return ApplePlatformHelper.HasAv1HardwareAccel(_logger); + } + private IEnumerable<string> GetHwaccelTypes() { string? output = null; @@ -451,10 +476,10 @@ namespace MediaBrowser.MediaEncoding.Encoder if (string.IsNullOrWhiteSpace(output)) { - return Enumerable.Empty<string>(); + return []; } - var found = output.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).Skip(1).Distinct().ToList(); + var found = output.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries).Skip(1).Distinct().ToList(); _logger.LogInformation("Available hwaccel types: {Types}", found); return found; @@ -488,6 +513,34 @@ namespace MediaBrowser.MediaEncoding.Encoder return false; } + public bool CheckBitStreamFilterWithOption(string filter, string option) + { + if (string.IsNullOrEmpty(filter) || string.IsNullOrEmpty(option)) + { + return false; + } + + string output; + try + { + output = GetProcessOutput(_encoderPath, "-h bsf=" + filter, false, null); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error detecting the given bit stream filter"); + return false; + } + + if (output.Contains("Bit stream filter " + filter, StringComparison.Ordinal)) + { + return output.Contains(option, StringComparison.Ordinal); + } + + _logger.LogWarning("Bit stream filter: {Name} with option {Option} is not available", filter, option); + + return false; + } + public bool CheckSupportedRuntimeKey(string keyDesc, Version? ffmpegVersion) { if (string.IsNullOrEmpty(keyDesc)) @@ -516,6 +569,11 @@ namespace MediaBrowser.MediaEncoding.Encoder return !string.IsNullOrEmpty(flag) && GetProcessExitCode(_encoderPath, $"-loglevel quiet -hwaccel_flags +{flag} -hide_banner -f lavfi -i nullsrc=s=1x1:d=100 -f null -"); } + public bool CheckSupportedProberOption(string option, string proberPath) + { + return !string.IsNullOrEmpty(option) && GetProcessExitCode(proberPath, $"-loglevel quiet -f lavfi -i nullsrc=s=1x1:d=1 -{option}"); + } + private IEnumerable<string> GetCodecs(Codec codec) { string codecstr = codec == Codec.Encoder ? "encoders" : "decoders"; @@ -527,12 +585,12 @@ namespace MediaBrowser.MediaEncoding.Encoder catch (Exception ex) { _logger.LogError(ex, "Error detecting available {Codec}", codecstr); - return Enumerable.Empty<string>(); + return []; } if (string.IsNullOrWhiteSpace(output)) { - return Enumerable.Empty<string>(); + return []; } var required = codec == Codec.Encoder ? _requiredEncoders : _requiredDecoders; @@ -557,12 +615,12 @@ namespace MediaBrowser.MediaEncoding.Encoder catch (Exception ex) { _logger.LogError(ex, "Error detecting available filters"); - return Enumerable.Empty<string>(); + return []; } if (string.IsNullOrWhiteSpace(output)) { - return Enumerable.Empty<string>(); + return []; } var found = FilterRegex() @@ -575,20 +633,6 @@ namespace MediaBrowser.MediaEncoding.Encoder return found; } - private Dictionary<int, bool> GetFFmpegFiltersWithOption() - { - Dictionary<int, bool> dict = new Dictionary<int, bool>(); - for (int i = 0; i < _filterOptionsDict.Count; i++) - { - if (_filterOptionsDict.TryGetValue(i, out var val) && val.Length == 2) - { - dict.Add(i, CheckFilterWithOption(val[0], val[1])); - } - } - - return dict; - } - private string GetProcessOutput(string path, string arguments, bool readStdErr, string? testKey) { var redirectStandardIn = !string.IsNullOrEmpty(testKey); diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index 14cf869f9..2eb647e26 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -72,10 +72,12 @@ namespace MediaBrowser.MediaEncoding.Encoder private List<string> _decoders = new List<string>(); private List<string> _hwaccels = new List<string>(); private List<string> _filters = new List<string>(); - private IDictionary<int, bool> _filtersWithOption = new Dictionary<int, bool>(); + private IDictionary<FilterOptionType, bool> _filtersWithOption = new Dictionary<FilterOptionType, bool>(); + private IDictionary<BitStreamFilterOptionType, bool> _bitStreamFiltersWithOption = new Dictionary<BitStreamFilterOptionType, bool>(); private bool _isPkeyPauseSupported = false; private bool _isLowPriorityHwDecodeSupported = false; + private bool _proberSupportsFirstVideoFrame = false; private bool _isVaapiDeviceAmd = false; private bool _isVaapiDeviceInteliHD = false; @@ -83,6 +85,8 @@ namespace MediaBrowser.MediaEncoding.Encoder private bool _isVaapiDeviceSupportVulkanDrmModifier = false; private bool _isVaapiDeviceSupportVulkanDrmInterop = false; + private bool _isVideoToolboxAv1DecodeAvailable = false; + private static string[] _vulkanImageDrmFmtModifierExts = { "VK_EXT_image_drm_format_modifier", @@ -159,6 +163,8 @@ namespace MediaBrowser.MediaEncoding.Encoder /// <inheritdoc /> public bool IsVaapiDeviceSupportVulkanDrmInterop => _isVaapiDeviceSupportVulkanDrmInterop; + public bool IsVideoToolboxAv1DecodeAvailable => _isVideoToolboxAv1DecodeAvailable; + [GeneratedRegex(@"[^\/\\]+?(\.[^\/\\\n.]+)?$")] private static partial Regex FfprobePathRegex(); @@ -218,6 +224,7 @@ namespace MediaBrowser.MediaEncoding.Encoder SetAvailableEncoders(validator.GetEncoders()); SetAvailableFilters(validator.GetFilters()); SetAvailableFiltersWithOption(validator.GetFiltersWithOption()); + SetAvailableBitStreamFiltersWithOption(validator.GetBitStreamFiltersWithOption()); SetAvailableHwaccels(validator.GetHwaccels()); SetMediaEncoderVersion(validator); @@ -225,6 +232,7 @@ namespace MediaBrowser.MediaEncoding.Encoder _isPkeyPauseSupported = validator.CheckSupportedRuntimeKey("p pause transcoding", _ffmpegVersion); _isLowPriorityHwDecodeSupported = validator.CheckSupportedHwaccelFlag("low_priority"); + _proberSupportsFirstVideoFrame = validator.CheckSupportedProberOption("only_first_vframe", _ffprobePath); // Check the Vaapi device vendor if (OperatingSystem.IsLinux() @@ -261,6 +269,12 @@ namespace MediaBrowser.MediaEncoding.Encoder _logger.LogInformation("VAAPI device {RenderNodePath} supports Vulkan DRM interop", options.VaapiDevice); } } + + // Check if VideoToolbox supports AV1 decode + if (OperatingSystem.IsMacOS() && SupportsHwaccel("videotoolbox")) + { + _isVideoToolboxAv1DecodeAvailable = validator.CheckIsVideoToolboxAv1DecodeAvailable(); + } } _logger.LogInformation("FFmpeg: {FfmpegPath}", _ffmpegPath ?? string.Empty); @@ -327,11 +341,16 @@ namespace MediaBrowser.MediaEncoding.Encoder _filters = list.ToList(); } - public void SetAvailableFiltersWithOption(IDictionary<int, bool> dict) + public void SetAvailableFiltersWithOption(IDictionary<FilterOptionType, bool> dict) { _filtersWithOption = dict; } + public void SetAvailableBitStreamFiltersWithOption(IDictionary<BitStreamFilterOptionType, bool> dict) + { + _bitStreamFiltersWithOption = dict; + } + public void SetMediaEncoderVersion(EncoderValidator validator) { _ffmpegVersion = validator.GetFFmpegVersion(); @@ -364,12 +383,12 @@ namespace MediaBrowser.MediaEncoding.Encoder /// <inheritdoc /> public bool SupportsFilterWithOption(FilterOptionType option) { - if (_filtersWithOption.TryGetValue((int)option, out var val)) - { - return val; - } + return _filtersWithOption.TryGetValue(option, out var val) && val; + } - return false; + public bool SupportsBitStreamFilterWithOption(BitStreamFilterOptionType option) + { + return _bitStreamFiltersWithOption.TryGetValue(option, out var val) && val; } public bool CanEncodeToAudioCodec(string codec) @@ -491,6 +510,12 @@ namespace MediaBrowser.MediaEncoding.Encoder var args = extractChapters ? "{0} -i {1} -threads {2} -v warning -print_format json -show_streams -show_chapters -show_format" : "{0} -i {1} -threads {2} -v warning -print_format json -show_streams -show_format"; + + if (_proberSupportsFirstVideoFrame) + { + args += " -show_frames -only_first_vframe"; + } + args = string.Format(CultureInfo.InvariantCulture, args, probeSizeArgument, inputPath, _threads).Trim(); var process = new Process @@ -512,7 +537,7 @@ namespace MediaBrowser.MediaEncoding.Encoder EnableRaisingEvents = true }; - _logger.LogInformation("Starting {ProcessFileName} with args {ProcessArgs}", _ffprobePath, args); + _logger.LogDebug("Starting {ProcessFileName} with args {ProcessArgs}", _ffprobePath, args); var memoryStream = new MemoryStream(); await using (memoryStream.ConfigureAwait(false)) @@ -612,7 +637,7 @@ namespace MediaBrowser.MediaEncoding.Encoder } catch (Exception ex) { - _logger.LogError(ex, "I-frame image extraction failed, will attempt standard way. Input: {Arguments}", inputArgument); + _logger.LogWarning(ex, "I-frame image extraction failed, will attempt standard way. Input: {Arguments}", inputArgument); } } @@ -690,13 +715,11 @@ namespace MediaBrowser.MediaEncoding.Encoder filters.Add(scaler); - // Use ffmpeg to sample 100 (we can drop this if required using thumbnail=50 for 50 frames) frames and pick the best thumbnail. Have a fall back just in case. - // mpegts need larger batch size otherwise the corrupted thumbnail will be created. Larger batch size will lower the processing speed. + // Use ffmpeg to sample N frames and pick the best thumbnail. Have a fall back just in case. var enableThumbnail = !useTradeoff && useIFrame && !string.Equals("wtv", container, StringComparison.OrdinalIgnoreCase); if (enableThumbnail) { - var useLargerBatchSize = string.Equals("mpegts", container, StringComparison.OrdinalIgnoreCase); - filters.Add("thumbnail=n=" + (useLargerBatchSize ? "50" : "24")); + filters.Add("thumbnail=n=24"); } // Use SW tonemap on HDR video stream only when the zscale or tonemapx filter is available. @@ -709,25 +732,37 @@ namespace MediaBrowser.MediaEncoding.Encoder { var peak = videoStream.VideoRangeType == VideoRangeType.DOVI ? "400" : "100"; enableHdrExtraction = true; - filters.Add($"tonemapx=tonemap=bt2390:desat=0:peak={peak}:t=bt709:m=bt709:p=bt709:format=yuv420p"); + filters.Add($"tonemapx=tonemap=bt2390:desat=0:peak={peak}:t=bt709:m=bt709:p=bt709:format=yuv420p:range=full"); } else if (SupportsFilter("zscale") && videoStream.VideoRangeType != VideoRangeType.DOVI) { enableHdrExtraction = true; - filters.Add("zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0:peak=100,zscale=t=bt709:m=bt709,format=yuv420p"); + filters.Add("zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0:peak=100,zscale=t=bt709:m=bt709:out_range=full,format=yuv420p"); } } var vf = string.Join(',', filters); var mapArg = imageStreamIndex.HasValue ? (" -map 0:" + imageStreamIndex.Value.ToString(CultureInfo.InvariantCulture)) : string.Empty; - var args = string.Format(CultureInfo.InvariantCulture, "-i {0}{3} -threads {4} -v quiet -vframes 1 -vf {2}{5} -f image2 \"{1}\"", inputPath, tempExtractPath, vf, mapArg, _threads, isAudio ? string.Empty : GetImageResolutionParameter()); + var args = string.Format( + CultureInfo.InvariantCulture, + "-i {0}{1} -threads {2} -v quiet -vframes 1 -vf {3}{4}{5} -f image2 \"{6}\"", + inputPath, + mapArg, + _threads, + vf, + isAudio ? string.Empty : GetImageResolutionParameter(), + EncodingHelper.GetVideoSyncOption("-1", EncoderVersion), // auto decide fps mode + tempExtractPath); if (offset.HasValue) { args = string.Format(CultureInfo.InvariantCulture, "-ss {0} ", GetTimeParameter(offset.Value)) + args; } - if (useIFrame && useTradeoff) + // The mpegts demuxer cannot seek to keyframes, so we have to let the + // decoder discard non-keyframes, which may contain corrupted images. + var seekMpegTs = offset.HasValue && string.Equals("mpegts", container, StringComparison.OrdinalIgnoreCase); + if (useIFrame && (useTradeoff || seekMpegTs)) { args = "-skip_frame nokey " + args; } |