diff options
Diffstat (limited to 'src')
15 files changed, 989 insertions, 0 deletions
diff --git a/src/Jellyfin.MediaEncoding.Hls/Extensions/MediaEncodingHlsServiceCollectionExtensions.cs b/src/Jellyfin.MediaEncoding.Hls/Extensions/MediaEncodingHlsServiceCollectionExtensions.cs new file mode 100644 index 000000000..327898366 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Extensions/MediaEncodingHlsServiceCollectionExtensions.cs @@ -0,0 +1,21 @@ +using Jellyfin.MediaEncoding.Hls.Playlist; +using Microsoft.Extensions.DependencyInjection; + +namespace Jellyfin.MediaEncoding.Hls.Extensions +{ + /// <summary> + /// Extensions for the <see cref="IServiceCollection"/> interface. + /// </summary> + public static class MediaEncodingHlsServiceCollectionExtensions + { + /// <summary> + /// Adds the hls playlist generators to the <see cref="IServiceCollection"/>. + /// </summary> + /// <param name="serviceCollection">An instance of the <see cref="IServiceCollection"/> interface.</param> + /// <returns>The updated service collection.</returns> + public static IServiceCollection AddHlsPlaylistGenerator(this IServiceCollection serviceCollection) + { + return serviceCollection.AddSingleton<IDynamicHlsPlaylistGenerator, DynamicHlsPlaylistGenerator>(); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Jellyfin.MediaEncoding.Hls.csproj b/src/Jellyfin.MediaEncoding.Hls/Jellyfin.MediaEncoding.Hls.csproj new file mode 100644 index 000000000..f5b7fb378 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Jellyfin.MediaEncoding.Hls.csproj @@ -0,0 +1,31 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net6.0</TargetFramework> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + </PropertyGroup> + + <!-- Code Analyzers--> + <ItemGroup Condition=" '$(Configuration)' == 'Debug' "> + <PackageReference Include="SerilogAnalyzer" Version="0.15.0" PrivateAssets="All" /> + <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="All" /> + <PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer" Version="1.1.31" PrivateAssets="All" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="../../MediaBrowser.Common/MediaBrowser.Common.csproj" /> + <ProjectReference Include="../../MediaBrowser.Controller/MediaBrowser.Controller.csproj" /> + <ProjectReference Include="../Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="5.0.0" /> + </ItemGroup> + + <ItemGroup> + <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo"> + <_Parameter1>Jellyfin.MediaEncoding.Hls.Tests</_Parameter1> + </AssemblyAttribute> + </ItemGroup> + +</Project> diff --git a/src/Jellyfin.MediaEncoding.Hls/Playlist/CreateMainPlaylistRequest.cs b/src/Jellyfin.MediaEncoding.Hls/Playlist/CreateMainPlaylistRequest.cs new file mode 100644 index 000000000..d6db1ca6e --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Playlist/CreateMainPlaylistRequest.cs @@ -0,0 +1,57 @@ +namespace Jellyfin.MediaEncoding.Hls.Playlist +{ + /// <summary> + /// Request class for the <see cref="IDynamicHlsPlaylistGenerator.CreateMainPlaylist(CreateMainPlaylistRequest)"/> method. + /// </summary> + public class CreateMainPlaylistRequest + { + /// <summary> + /// Initializes a new instance of the <see cref="CreateMainPlaylistRequest"/> class. + /// </summary> + /// <param name="filePath">The absolute file path to the file.</param> + /// <param name="desiredSegmentLengthMs">The desired segment length in milliseconds.</param> + /// <param name="totalRuntimeTicks">The total duration of the file in ticks.</param> + /// <param name="segmentContainer">The desired segment container eg. "ts".</param> + /// <param name="endpointPrefix">The URI prefix for the relative URL in the playlist.</param> + /// <param name="queryString">The desired query string to append (must start with ?).</param> + public CreateMainPlaylistRequest(string filePath, int desiredSegmentLengthMs, long totalRuntimeTicks, string segmentContainer, string endpointPrefix, string queryString) + { + FilePath = filePath; + DesiredSegmentLengthMs = desiredSegmentLengthMs; + TotalRuntimeTicks = totalRuntimeTicks; + SegmentContainer = segmentContainer; + EndpointPrefix = endpointPrefix; + QueryString = queryString; + } + + /// <summary> + /// Gets the file path. + /// </summary> + public string FilePath { get; } + + /// <summary> + /// Gets the desired segment length in milliseconds. + /// </summary> + public int DesiredSegmentLengthMs { get; } + + /// <summary> + /// Gets the total runtime in ticks. + /// </summary> + public long TotalRuntimeTicks { get; } + + /// <summary> + /// Gets the segment container. + /// </summary> + public string SegmentContainer { get; } + + /// <summary> + /// Gets the endpoint prefix for the URL. + /// </summary> + public string EndpointPrefix { get; } + + /// <summary> + /// Gets the query string. + /// </summary> + public string QueryString { get; } + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs b/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs new file mode 100644 index 000000000..cbc62eb59 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs @@ -0,0 +1,273 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using Jellyfin.Extensions.Json; +using Jellyfin.MediaEncoding.Keyframes; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.MediaEncoding; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.MediaEncoding.Hls.Playlist +{ + /// <inheritdoc /> + public class DynamicHlsPlaylistGenerator : IDynamicHlsPlaylistGenerator + { + private const string DefaultContainerExtension = ".ts"; + + private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + private readonly IServerConfigurationManager _serverConfigurationManager; + private readonly IMediaEncoder _mediaEncoder; + private readonly IApplicationPaths _applicationPaths; + private readonly KeyframeExtractor _keyframeExtractor; + private readonly ILogger<DynamicHlsPlaylistGenerator> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="DynamicHlsPlaylistGenerator"/> class. + /// </summary> + /// <param name="serverConfigurationManager">An instance of the see <see cref="IServerConfigurationManager"/> interface.</param> + /// <param name="mediaEncoder">An instance of the see <see cref="IMediaEncoder"/> interface.</param> + /// <param name="applicationPaths">An instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="loggerFactory">An instance of the see <see cref="ILoggerFactory"/> interface.</param> + public DynamicHlsPlaylistGenerator(IServerConfigurationManager serverConfigurationManager, IMediaEncoder mediaEncoder, IApplicationPaths applicationPaths, ILoggerFactory loggerFactory) + { + _serverConfigurationManager = serverConfigurationManager; + _mediaEncoder = mediaEncoder; + _applicationPaths = applicationPaths; + _keyframeExtractor = new KeyframeExtractor(loggerFactory.CreateLogger<KeyframeExtractor>()); + _logger = loggerFactory.CreateLogger<DynamicHlsPlaylistGenerator>(); + } + + private string KeyframeCachePath => Path.Combine(_applicationPaths.DataPath, "keyframes"); + + /// <inheritdoc /> + public string CreateMainPlaylist(CreateMainPlaylistRequest request) + { + IReadOnlyList<double> segments; + if (TryExtractKeyframes(request.FilePath, out var keyframeData)) + { + segments = ComputeSegments(keyframeData, request.DesiredSegmentLengthMs); + } + else + { + segments = ComputeEqualLengthSegments(request.DesiredSegmentLengthMs, request.TotalRuntimeTicks); + } + + var segmentExtension = GetSegmentFileExtension(request.SegmentContainer); + + // http://ffmpeg.org/ffmpeg-all.html#toc-hls-2 + var isHlsInFmp4 = string.Equals(segmentExtension, "mp4", StringComparison.OrdinalIgnoreCase); + var hlsVersion = isHlsInFmp4 ? "7" : "3"; + + var builder = new StringBuilder(128); + + builder.AppendLine("#EXTM3U") + .AppendLine("#EXT-X-PLAYLIST-TYPE:VOD") + .Append("#EXT-X-VERSION:") + .Append(hlsVersion) + .AppendLine() + .Append("#EXT-X-TARGETDURATION:") + .Append(Math.Ceiling(segments.Count > 0 ? segments.Max() : request.DesiredSegmentLengthMs)) + .AppendLine() + .AppendLine("#EXT-X-MEDIA-SEQUENCE:0"); + + var index = 0; + + if (isHlsInFmp4) + { + builder.Append("#EXT-X-MAP:URI=\"") + .Append(request.EndpointPrefix) + .Append("-1") + .Append(segmentExtension) + .Append(request.QueryString) + .Append('"') + .AppendLine(); + } + + long currentRuntimeInSeconds = 0; + foreach (var length in segments) + { + // Manually convert to ticks to avoid precision loss when converting double + var lengthTicks = Convert.ToInt64(length * TimeSpan.TicksPerSecond); + builder.Append("#EXTINF:") + .Append(length.ToString("0.000000", CultureInfo.InvariantCulture)) + .AppendLine(", nodesc") + .Append(request.EndpointPrefix) + .Append(index++) + .Append(segmentExtension) + .Append(request.QueryString) + .Append("&runtimeTicks=") + .Append(currentRuntimeInSeconds) + .Append("&actualSegmentLengthTicks=") + .Append(lengthTicks) + .AppendLine(); + + currentRuntimeInSeconds += lengthTicks; + } + + builder.AppendLine("#EXT-X-ENDLIST"); + + return builder.ToString(); + } + + private bool TryExtractKeyframes(string filePath, [NotNullWhen(true)] out KeyframeData? keyframeData) + { + keyframeData = null; + if (!IsExtractionAllowedForFile(filePath, _serverConfigurationManager.GetEncodingOptions().AllowAutomaticKeyframeExtractionForExtensions)) + { + return false; + } + + var succeeded = false; + var cachePath = GetCachePath(filePath); + if (TryReadFromCache(cachePath, out var cachedResult)) + { + keyframeData = cachedResult; + } + else + { + try + { + keyframeData = _keyframeExtractor.GetKeyframeData(filePath, _mediaEncoder.ProbePath, string.Empty); + } + catch (Exception ex) + { + _logger.LogError(ex, "Keyframe extraction failed for path {FilePath}", filePath); + return false; + } + + succeeded = keyframeData.KeyframeTicks.Count > 0; + if (succeeded) + { + CacheResult(cachePath, keyframeData); + } + } + + return succeeded; + } + + private void CacheResult(string cachePath, KeyframeData keyframeData) + { + var json = JsonSerializer.Serialize(keyframeData, _jsonOptions); + Directory.CreateDirectory(Path.GetDirectoryName(cachePath) ?? throw new ArgumentException($"Provided path ({cachePath}) is not valid.", nameof(cachePath))); + File.WriteAllText(cachePath, json); + } + + private string GetCachePath(string filePath) + { + var lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + ReadOnlySpan<char> filename = (filePath + "_" + lastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename.Slice(0, 1); + + return Path.Join(KeyframeCachePath, prefix, filename); + } + + private bool TryReadFromCache(string cachePath, [NotNullWhen(true)] out KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize<KeyframeData>(bytes, _jsonOptions); + return cachedResult != null; + } + + cachedResult = null; + return false; + } + + internal static bool IsExtractionAllowedForFile(ReadOnlySpan<char> filePath, string[] allowedExtensions) + { + var extension = Path.GetExtension(filePath); + if (extension.IsEmpty) + { + return false; + } + + // Remove the leading dot + var extensionWithoutDot = extension[1..]; + for (var i = 0; i < allowedExtensions.Length; i++) + { + var allowedExtension = allowedExtensions[i]; + if (extensionWithoutDot.Equals(allowedExtension, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + internal static IReadOnlyList<double> ComputeSegments(KeyframeData keyframeData, int desiredSegmentLengthMs) + { + if (keyframeData.KeyframeTicks.Count > 0 && keyframeData.TotalDuration < keyframeData.KeyframeTicks[^1]) + { + throw new ArgumentException("Invalid duration in keyframe data", nameof(keyframeData)); + } + + long lastKeyframe = 0; + var result = new List<double>(); + // Scale the segment length to ticks to match the keyframes + var desiredSegmentLengthTicks = TimeSpan.FromMilliseconds(desiredSegmentLengthMs).Ticks; + var desiredCutTime = desiredSegmentLengthTicks; + for (var j = 0; j < keyframeData.KeyframeTicks.Count; j++) + { + var keyframe = keyframeData.KeyframeTicks[j]; + if (keyframe >= desiredCutTime) + { + var currentSegmentLength = keyframe - lastKeyframe; + result.Add(TimeSpan.FromTicks(currentSegmentLength).TotalSeconds); + lastKeyframe = keyframe; + desiredCutTime += desiredSegmentLengthTicks; + } + } + + result.Add(TimeSpan.FromTicks(keyframeData.TotalDuration - lastKeyframe).TotalSeconds); + return result; + } + + internal static double[] ComputeEqualLengthSegments(int desiredSegmentLengthMs, long totalRuntimeTicks) + { + if (desiredSegmentLengthMs == 0 || totalRuntimeTicks == 0) + { + throw new InvalidOperationException($"Invalid segment length ({desiredSegmentLengthMs}) or runtime ticks ({totalRuntimeTicks})"); + } + + var desiredSegmentLength = TimeSpan.FromMilliseconds(desiredSegmentLengthMs); + + var segmentLengthTicks = desiredSegmentLength.Ticks; + var wholeSegments = totalRuntimeTicks / segmentLengthTicks; + var remainingTicks = totalRuntimeTicks % segmentLengthTicks; + + var segmentsLen = wholeSegments + (remainingTicks == 0 ? 0 : 1); + var segments = new double[segmentsLen]; + for (int i = 0; i < wholeSegments; i++) + { + segments[i] = desiredSegmentLength.TotalSeconds; + } + + if (remainingTicks != 0) + { + segments[^1] = TimeSpan.FromTicks(remainingTicks).TotalSeconds; + } + + return segments; + } + + // TODO copied from DynamicHlsController + private static string GetSegmentFileExtension(string segmentContainer) + { + if (!string.IsNullOrWhiteSpace(segmentContainer)) + { + return "." + segmentContainer; + } + + return DefaultContainerExtension; + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Playlist/IDynamicHlsPlaylistGenerator.cs b/src/Jellyfin.MediaEncoding.Hls/Playlist/IDynamicHlsPlaylistGenerator.cs new file mode 100644 index 000000000..7e766b9a2 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Playlist/IDynamicHlsPlaylistGenerator.cs @@ -0,0 +1,15 @@ +namespace Jellyfin.MediaEncoding.Hls.Playlist +{ + /// <summary> + /// Generator for dynamic HLS playlists where the segment lengths aren't known in advance. + /// </summary> + public interface IDynamicHlsPlaylistGenerator + { + /// <summary> + /// Creates the main playlist containing the main video or audio stream. + /// </summary> + /// <param name="request">An instance of the <see cref="CreateMainPlaylistRequest"/> class.</param> + /// <returns>The playlist as a formatted string.</returns> + string CreateMainPlaylist(CreateMainPlaylistRequest request); + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs new file mode 100644 index 000000000..351d880fe --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; + +namespace Jellyfin.MediaEncoding.Keyframes.FfProbe +{ + /// <summary> + /// FfProbe based keyframe extractor. + /// </summary> + public static class FfProbeKeyframeExtractor + { + private const string DefaultArguments = "-v error -skip_frame nokey -show_entries format=duration -show_entries stream=duration -show_entries packet=pts_time,flags -select_streams v -of csv \"{0}\""; + + /// <summary> + /// Extracts the keyframes using the ffprobe executable at the specified path. + /// </summary> + /// <param name="ffProbePath">The path to the ffprobe executable.</param> + /// <param name="filePath">The file path.</param> + /// <returns>An instance of <see cref="KeyframeData"/>.</returns> + public static KeyframeData GetKeyframeData(string ffProbePath, string filePath) + { + using var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = ffProbePath, + Arguments = string.Format(CultureInfo.InvariantCulture, DefaultArguments, filePath), + + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + + WindowStyle = ProcessWindowStyle.Hidden, + ErrorDialog = false, + }, + EnableRaisingEvents = true + }; + + process.Start(); + + return ParseStream(process.StandardOutput); + } + + internal static KeyframeData ParseStream(StreamReader reader) + { + var keyframes = new List<long>(); + double streamDuration = 0; + double formatDuration = 0; + + while (!reader.EndOfStream) + { + var line = reader.ReadLine().AsSpan(); + if (line.IsEmpty) + { + continue; + } + + var firstComma = line.IndexOf(','); + var lineType = line[..firstComma]; + var rest = line[(firstComma + 1)..]; + if (lineType.Equals("packet", StringComparison.OrdinalIgnoreCase)) + { + if (rest.EndsWith(",K_")) + { + // Trim the flags from the packet line. Example line: packet,7169.079000,K_ + var keyframe = double.Parse(rest[..^3], NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture); + // Have to manually convert to ticks to avoid rounding errors as TimeSpan is only precise down to 1 ms when converting double. + keyframes.Add(Convert.ToInt64(keyframe * TimeSpan.TicksPerSecond)); + } + } + else if (lineType.Equals("stream", StringComparison.OrdinalIgnoreCase)) + { + if (double.TryParse(rest, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var streamDurationResult)) + { + streamDuration = streamDurationResult; + } + } + else if (lineType.Equals("format", StringComparison.OrdinalIgnoreCase)) + { + if (double.TryParse(rest, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var formatDurationResult)) + { + formatDuration = formatDurationResult; + } + } + } + + // Prefer the stream duration as it should be more accurate + var duration = streamDuration > 0 ? streamDuration : formatDuration; + + return new KeyframeData(TimeSpan.FromSeconds(duration).Ticks, keyframes); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs new file mode 100644 index 000000000..fdd5dc577 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs @@ -0,0 +1,18 @@ +using System; + +namespace Jellyfin.MediaEncoding.Keyframes.FfTool +{ + /// <summary> + /// FfTool based keyframe extractor. + /// </summary> + public static class FfToolKeyframeExtractor + { + /// <summary> + /// Extracts the keyframes using the fftool executable at the specified path. + /// </summary> + /// <param name="ffToolPath">The path to the fftool executable.</param> + /// <param name="filePath">The file path.</param> + /// <returns>An instance of <see cref="KeyframeData"/>.</returns> + public static KeyframeData GetKeyframeData(string ffToolPath, string filePath) => throw new NotImplementedException(); + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj b/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj new file mode 100644 index 000000000..22db10d72 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj @@ -0,0 +1,29 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net6.0</TargetFramework> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="NEbml" Version="0.11.0" /> + </ItemGroup> + + <!-- Code Analyzers--> + <ItemGroup Condition=" '$(Configuration)' == 'Debug' "> + <PackageReference Include="SerilogAnalyzer" Version="0.15.0" PrivateAssets="All" /> + <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="All" /> + <PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer" Version="1.1.31" PrivateAssets="All" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" /> + </ItemGroup> + + <ItemGroup> + <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo"> + <_Parameter1>Jellyfin.MediaEncoding.Keyframes.Tests</_Parameter1> + </AssemblyAttribute> + </ItemGroup> + +</Project> diff --git a/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs new file mode 100644 index 000000000..1683cd22a --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; + +namespace Jellyfin.MediaEncoding.Keyframes +{ + /// <summary> + /// Keyframe information for a specific file. + /// </summary> + public class KeyframeData + { + /// <summary> + /// Initializes a new instance of the <see cref="KeyframeData"/> class. + /// </summary> + /// <param name="totalDuration">The total duration of the video stream in ticks.</param> + /// <param name="keyframeTicks">The video keyframes in ticks.</param> + public KeyframeData(long totalDuration, IReadOnlyList<long> keyframeTicks) + { + TotalDuration = totalDuration; + KeyframeTicks = keyframeTicks; + } + + /// <summary> + /// Gets the total duration of the stream in ticks. + /// </summary> + public long TotalDuration { get; } + + /// <summary> + /// Gets the keyframes in ticks. + /// </summary> + public IReadOnlyList<long> KeyframeTicks { get; } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/KeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeExtractor.cs new file mode 100644 index 000000000..5304a55f8 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeExtractor.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; +using Jellyfin.MediaEncoding.Keyframes.FfProbe; +using Jellyfin.MediaEncoding.Keyframes.FfTool; +using Jellyfin.MediaEncoding.Keyframes.Matroska; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.MediaEncoding.Keyframes +{ + /// <summary> + /// Manager class for the set of keyframe extractors. + /// </summary> + public class KeyframeExtractor + { + private readonly ILogger<KeyframeExtractor> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="KeyframeExtractor"/> class. + /// </summary> + /// <param name="logger">An instance of the <see cref="ILogger{KeyframeExtractor}"/> interface.</param> + public KeyframeExtractor(ILogger<KeyframeExtractor> logger) + { + _logger = logger; + } + + /// <summary> + /// Extracts the keyframe positions from a video file. + /// </summary> + /// <param name="filePath">Absolute file path to the media file.</param> + /// <param name="ffProbePath">Absolute file path to the ffprobe executable.</param> + /// <param name="ffToolPath">Absolute file path to the fftool executable.</param> + /// <returns>An instance of <see cref="KeyframeData"/>.</returns> + public KeyframeData GetKeyframeData(string filePath, string ffProbePath, string ffToolPath) + { + var extension = Path.GetExtension(filePath.AsSpan()); + if (extension.Equals(".mkv", StringComparison.OrdinalIgnoreCase)) + { + try + { + return MatroskaKeyframeExtractor.GetKeyframeData(filePath); + } + catch (Exception ex) + { + _logger.LogError(ex, "{ExtractorType} failed to extract keyframes", nameof(MatroskaKeyframeExtractor)); + } + } + + try + { + return FfToolKeyframeExtractor.GetKeyframeData(ffToolPath, filePath); + } + catch (Exception ex) + { + _logger.LogError(ex, "{ExtractorType} failed to extract keyframes", nameof(FfToolKeyframeExtractor)); + } + + try + { + return FfProbeKeyframeExtractor.GetKeyframeData(ffProbePath, filePath); + } + catch (Exception ex) + { + _logger.LogError(ex, "{ExtractorType} failed to extract keyframes", nameof(FfProbeKeyframeExtractor)); + } + + return new KeyframeData(0, Array.Empty<long>()); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs new file mode 100644 index 000000000..75d5aafe0 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs @@ -0,0 +1,178 @@ +using System; +using System.Buffers.Binary; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Models; +using NEbml.Core; + +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Extensions +{ + /// <summary> + /// Extension methods for the <see cref="EbmlReader"/> class. + /// </summary> + internal static class EbmlReaderExtensions + { + /// <summary> + /// Traverses the current container to find the element with <paramref name="identifier"/> identifier. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <param name="identifier">The element identifier.</param> + /// <returns>A value indicating whether the element was found.</returns> + internal static bool FindElement(this EbmlReader reader, ulong identifier) + { + while (reader.ReadNext()) + { + if (reader.ElementId.EncodedValue == identifier) + { + return true; + } + } + + return false; + } + + /// <summary> + /// Reads the current position in the file as an unsigned integer converted from binary. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <returns>The unsigned integer.</returns> + internal static uint ReadUIntFromBinary(this EbmlReader reader) + { + var buffer = new byte[4]; + reader.ReadBinary(buffer, 0, 4); + return BinaryPrimitives.ReadUInt32BigEndian(buffer); + } + + /// <summary> + /// Reads from the start of the file to retrieve the SeekHead segment. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <returns>Instance of <see cref="SeekHead"/>.</returns> + internal static SeekHead ReadSeekHead(this EbmlReader reader) + { + reader = reader ?? throw new ArgumentNullException(nameof(reader)); + + if (reader.ElementPosition != 0) + { + throw new InvalidOperationException("File position must be at 0"); + } + + // Skip the header + if (!reader.FindElement(MatroskaConstants.SegmentContainer)) + { + throw new InvalidOperationException("Expected a segment container"); + } + + reader.EnterContainer(); + + long? tracksPosition = null; + long? cuesPosition = null; + long? infoPosition = null; + // The first element should be a SeekHead otherwise we'll have to search manually + if (!reader.FindElement(MatroskaConstants.SeekHead)) + { + throw new InvalidOperationException("Expected a SeekHead"); + } + + reader.EnterContainer(); + while (reader.FindElement(MatroskaConstants.Seek)) + { + reader.EnterContainer(); + reader.ReadNext(); + var type = (ulong)reader.ReadUIntFromBinary(); + switch (type) + { + case MatroskaConstants.Tracks: + reader.ReadNext(); + tracksPosition = (long)reader.ReadUInt(); + break; + case MatroskaConstants.Cues: + reader.ReadNext(); + cuesPosition = (long)reader.ReadUInt(); + break; + case MatroskaConstants.Info: + reader.ReadNext(); + infoPosition = (long)reader.ReadUInt(); + break; + } + + reader.LeaveContainer(); + + if (tracksPosition.HasValue && cuesPosition.HasValue && infoPosition.HasValue) + { + break; + } + } + + reader.LeaveContainer(); + + if (!tracksPosition.HasValue || !cuesPosition.HasValue || !infoPosition.HasValue) + { + throw new InvalidOperationException("SeekHead is missing or does not contain Info, Tracks and Cues positions"); + } + + return new SeekHead(infoPosition.Value, tracksPosition.Value, cuesPosition.Value); + } + + /// <summary> + /// Reads from SegmentContainer to retrieve the Info segment. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <param name="position">The position of the info segment relative to the Segment container.</param> + /// <returns>Instance of <see cref="Info"/>.</returns> + internal static Info ReadInfo(this EbmlReader reader, long position) + { + reader.ReadAt(position); + + double? duration = null; + reader.EnterContainer(); + // Mandatory element + reader.FindElement(MatroskaConstants.TimestampScale); + var timestampScale = reader.ReadUInt(); + + if (reader.FindElement(MatroskaConstants.Duration)) + { + duration = reader.ReadFloat(); + } + + reader.LeaveContainer(); + + return new Info((long)timestampScale, duration); + } + + /// <summary> + /// Enters the Tracks segment and reads all tracks to find the specified type. + /// </summary> + /// <param name="reader">Instance of <see cref="EbmlReader"/>.</param> + /// <param name="tracksPosition">The relative position of the tracks segment.</param> + /// <param name="type">The track type identifier.</param> + /// <returns>The first track number with the specified type.</returns> + /// <exception cref="InvalidOperationException">Stream type is not found.</exception> + internal static ulong FindFirstTrackNumberByType(this EbmlReader reader, long tracksPosition, ulong type) + { + reader.ReadAt(tracksPosition); + + reader.EnterContainer(); + while (reader.FindElement(MatroskaConstants.TrackEntry)) + { + reader.EnterContainer(); + // Mandatory element + reader.FindElement(MatroskaConstants.TrackNumber); + var trackNumber = reader.ReadUInt(); + + // Mandatory element + reader.FindElement(MatroskaConstants.TrackType); + var trackType = reader.ReadUInt(); + + reader.LeaveContainer(); + if (trackType == MatroskaConstants.TrackTypeVideo) + { + reader.LeaveContainer(); + return trackNumber; + } + } + + reader.LeaveContainer(); + + throw new InvalidOperationException($"No stream with type {type} found"); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs new file mode 100644 index 000000000..d18418d45 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs @@ -0,0 +1,31 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska +{ + /// <summary> + /// Constants for the Matroska identifiers. + /// </summary> + public static class MatroskaConstants + { + internal const ulong SegmentContainer = 0x18538067; + + internal const ulong SeekHead = 0x114D9B74; + internal const ulong Seek = 0x4DBB; + + internal const ulong Info = 0x1549A966; + internal const ulong TimestampScale = 0x2AD7B1; + internal const ulong Duration = 0x4489; + + internal const ulong Tracks = 0x1654AE6B; + internal const ulong TrackEntry = 0xAE; + internal const ulong TrackNumber = 0xD7; + internal const ulong TrackType = 0x83; + + internal const ulong TrackTypeVideo = 0x1; + internal const ulong TrackTypeSubtitle = 0x11; + + internal const ulong Cues = 0x1C53BB6B; + internal const ulong CueTime = 0xB3; + internal const ulong CuePoint = 0xBB; + internal const ulong CueTrackPositions = 0xB7; + internal const ulong CuePointTrackNumber = 0xF7; + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs new file mode 100644 index 000000000..6a8a55643 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Extensions; +using NEbml.Core; + +namespace Jellyfin.MediaEncoding.Keyframes.Matroska +{ + /// <summary> + /// The keyframe extractor for the matroska container. + /// </summary> + public static class MatroskaKeyframeExtractor + { + /// <summary> + /// Extracts the keyframes in ticks (scaled using the container timestamp scale) from the matroska container. + /// </summary> + /// <param name="filePath">The file path.</param> + /// <returns>An instance of <see cref="KeyframeData"/>.</returns> + public static KeyframeData GetKeyframeData(string filePath) + { + using var stream = File.OpenRead(filePath); + using var reader = new EbmlReader(stream); + + var seekHead = reader.ReadSeekHead(); + var info = reader.ReadInfo(seekHead.InfoPosition); + var videoTrackNumber = reader.FindFirstTrackNumberByType(seekHead.TracksPosition, MatroskaConstants.TrackTypeVideo); + + var keyframes = new List<long>(); + reader.ReadAt(seekHead.CuesPosition); + reader.EnterContainer(); + + while (reader.FindElement(MatroskaConstants.CuePoint)) + { + reader.EnterContainer(); + ulong? trackNumber = null; + // Mandatory element + reader.FindElement(MatroskaConstants.CueTime); + var cueTime = reader.ReadUInt(); + + // Mandatory element + reader.FindElement(MatroskaConstants.CueTrackPositions); + reader.EnterContainer(); + if (reader.FindElement(MatroskaConstants.CuePointTrackNumber)) + { + trackNumber = reader.ReadUInt(); + } + + reader.LeaveContainer(); + + if (trackNumber == videoTrackNumber) + { + keyframes.Add(ScaleToTicks(cueTime, info.TimestampScale)); + } + + reader.LeaveContainer(); + } + + reader.LeaveContainer(); + + var result = new KeyframeData(ScaleToTicks(info.Duration ?? 0, info.TimestampScale), keyframes); + return result; + } + + private static long ScaleToTicks(ulong unscaledValue, long timestampScale) + { + // TimestampScale is in nanoseconds, scale it to get the value in ticks, 1 tick == 100 ns + return (long)unscaledValue * timestampScale / 100; + } + + private static long ScaleToTicks(double unscaledValue, long timestampScale) + { + // TimestampScale is in nanoseconds, scale it to get the value in ticks, 1 tick == 100 ns + return Convert.ToInt64(unscaledValue * timestampScale / 100); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs new file mode 100644 index 000000000..02c6741ec --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs @@ -0,0 +1,29 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Models +{ + /// <summary> + /// The matroska Info segment. + /// </summary> + internal class Info + { + /// <summary> + /// Initializes a new instance of the <see cref="Info"/> class. + /// </summary> + /// <param name="timestampScale">The timestamp scale in nanoseconds.</param> + /// <param name="duration">The duration of the entire file.</param> + public Info(long timestampScale, double? duration) + { + TimestampScale = timestampScale; + Duration = duration; + } + + /// <summary> + /// Gets the timestamp scale in nanoseconds. + /// </summary> + public long TimestampScale { get; } + + /// <summary> + /// Gets the total duration of the file. + /// </summary> + public double? Duration { get; } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs new file mode 100644 index 000000000..d9e346c03 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs @@ -0,0 +1,36 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Models +{ + /// <summary> + /// The matroska SeekHead segment. All positions are relative to the Segment container. + /// </summary> + internal class SeekHead + { + /// <summary> + /// Initializes a new instance of the <see cref="SeekHead"/> class. + /// </summary> + /// <param name="infoPosition">The relative file position of the info segment.</param> + /// <param name="tracksPosition">The relative file position of the tracks segment.</param> + /// <param name="cuesPosition">The relative file position of the cues segment.</param> + public SeekHead(long infoPosition, long tracksPosition, long cuesPosition) + { + InfoPosition = infoPosition; + TracksPosition = tracksPosition; + CuesPosition = cuesPosition; + } + + /// <summary> + /// Gets relative file position of the info segment. + /// </summary> + public long InfoPosition { get; } + + /// <summary> + /// Gets the relative file position of the tracks segment. + /// </summary> + public long TracksPosition { get; } + + /// <summary> + /// Gets the relative file position of the cues segment. + /// </summary> + public long CuesPosition { get; } + } +} |