From 6ffa9539bbfbfb1090b02cebc8a28283a8c69041 Mon Sep 17 00:00:00 2001 From: cvium Date: Tue, 11 Jan 2022 23:30:30 +0100 Subject: Refactor and add scheduled task --- .../Cache/CacheDecorator.cs | 87 ++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs (limited to 'src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs') diff --git a/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs new file mode 100644 index 000000000..f5f79ddc5 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs @@ -0,0 +1,87 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Text.Json; +using Jellyfin.Extensions.Json; +using Jellyfin.MediaEncoding.Hls.Extractors; +using Jellyfin.MediaEncoding.Keyframes; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; + +namespace Jellyfin.MediaEncoding.Hls.Cache; + +/// +public class CacheDecorator : IKeyframeExtractor +{ + private readonly IKeyframeExtractor _keyframeExtractor; + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + private readonly string _keyframeCachePath; + + /// + /// Initializes a new instance of the class. + /// + /// An instance of the interface. + /// An instance of the interface. + public CacheDecorator(IApplicationPaths applicationPaths, IKeyframeExtractor keyframeExtractor) + { + _keyframeExtractor = keyframeExtractor; + ArgumentNullException.ThrowIfNull(applicationPaths); + + // TODO make the dir configurable + _keyframeCachePath = Path.Combine(applicationPaths.DataPath, "keyframes"); + } + + /// + public bool IsMetadataBased => _keyframeExtractor.IsMetadataBased; + + /// + public bool TryExtractKeyframes(string filePath, [NotNullWhen(true)] out KeyframeData? keyframeData) + { + keyframeData = null; + var cachePath = GetCachePath(_keyframeCachePath, filePath); + if (TryReadFromCache(cachePath, out var cachedResult)) + { + keyframeData = cachedResult; + return true; + } + + if (!_keyframeExtractor.TryExtractKeyframes(filePath, out var result)) + { + return false; + } + + keyframeData = result; + SaveToCache(cachePath, keyframeData); + return true; + } + + private static void SaveToCache(string cachePath, KeyframeData keyframeData) + { + var json = JsonSerializer.Serialize(keyframeData, _jsonOptions); + Directory.CreateDirectory(Path.GetDirectoryName(cachePath) ?? throw new ArgumentException($"Provided path ({cachePath}) is not valid.", nameof(cachePath))); + File.WriteAllText(cachePath, json); + } + + private static string GetCachePath(string keyframeCachePath, string filePath) + { + var lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + ReadOnlySpan filename = (filePath + "_" + lastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename[..1]; + + return Path.Join(keyframeCachePath, prefix, filename); + } + + private static bool TryReadFromCache(string cachePath, [NotNullWhen(true)] out KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize(bytes, _jsonOptions); + return cachedResult != null; + } + + cachedResult = null; + return false; + } +} -- cgit v1.2.3 From 90736ee346e1e78095667d060826c22e57525bb3 Mon Sep 17 00:00:00 2001 From: cvium Date: Sun, 16 Jan 2022 22:10:22 +0100 Subject: Add pagination and fixes --- .../Cache/CacheDecorator.cs | 13 +++++- .../KeyframeExtractionScheduledTask.cs | 52 ++++++++++++++-------- .../Matroska/Extensions/EbmlReaderExtensions.cs | 2 +- .../Matroska/MatroskaKeyframeExtractor.cs | 16 ++++++- 4 files changed, 60 insertions(+), 23 deletions(-) (limited to 'src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs') diff --git a/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs index f5f79ddc5..09816c960 100644 --- a/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs +++ b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs @@ -8,6 +8,7 @@ using Jellyfin.MediaEncoding.Hls.Extractors; using Jellyfin.MediaEncoding.Keyframes; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; +using Microsoft.Extensions.Logging; namespace Jellyfin.MediaEncoding.Hls.Cache; @@ -15,6 +16,8 @@ namespace Jellyfin.MediaEncoding.Hls.Cache; public class CacheDecorator : IKeyframeExtractor { private readonly IKeyframeExtractor _keyframeExtractor; + private readonly ILogger _logger; + private readonly string _keyframeExtractorName; private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; private readonly string _keyframeCachePath; @@ -23,11 +26,15 @@ public class CacheDecorator : IKeyframeExtractor /// /// An instance of the interface. /// An instance of the interface. - public CacheDecorator(IApplicationPaths applicationPaths, IKeyframeExtractor keyframeExtractor) + /// An instance of the interface. + public CacheDecorator(IApplicationPaths applicationPaths, IKeyframeExtractor keyframeExtractor, ILogger logger) { - _keyframeExtractor = keyframeExtractor; ArgumentNullException.ThrowIfNull(applicationPaths); + ArgumentNullException.ThrowIfNull(keyframeExtractor); + _keyframeExtractor = keyframeExtractor; + _logger = logger; + _keyframeExtractorName = keyframeExtractor.GetType().Name; // TODO make the dir configurable _keyframeCachePath = Path.Combine(applicationPaths.DataPath, "keyframes"); } @@ -48,9 +55,11 @@ public class CacheDecorator : IKeyframeExtractor if (!_keyframeExtractor.TryExtractKeyframes(filePath, out var result)) { + _logger.LogDebug("Failed to extract keyframes using {ExtractorName}", _keyframeExtractorName); return false; } + _logger.LogDebug("Successfully extracted keyframes using {ExtractorName}", _keyframeExtractorName); keyframeData = result; SaveToCache(cachePath, keyframeData); return true; diff --git a/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs b/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs index 4b7b3c20b..03acc6911 100644 --- a/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs +++ b/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs @@ -18,6 +18,8 @@ namespace Jellyfin.MediaEncoding.Hls.ScheduledTasks; /// public class KeyframeExtractionScheduledTask : IScheduledTask { + private const int Pagesize = 1000; + private readonly ILocalizationManager _localizationManager; private readonly ILibraryManager _libraryManager; private readonly IKeyframeExtractor[] _keyframeExtractors; @@ -33,7 +35,7 @@ public class KeyframeExtractionScheduledTask : IScheduledTask { _localizationManager = localizationManager; _libraryManager = libraryManager; - _keyframeExtractors = keyframeExtractors.ToArray(); + _keyframeExtractors = keyframeExtractors.OrderByDescending(e => e.IsMetadataBased).ToArray(); } /// @@ -43,7 +45,7 @@ public class KeyframeExtractionScheduledTask : IScheduledTask public string Key => "KeyframeExtraction"; /// - public string Description => "Extracts keyframes from video files to create more precise HLS playlists"; + public string Description => "Extracts keyframes from video files to create more precise HLS playlists. This task may run for a long time."; /// public string Category => _localizationManager.GetLocalizedString("TasksLibraryCategory"); @@ -58,35 +60,49 @@ public class KeyframeExtractionScheduledTask : IScheduledTask IncludeItemTypes = _itemTypes, DtoOptions = new DtoOptions(true), SourceTypes = new[] { SourceType.Library }, - Recursive = true + Recursive = true, + Limit = Pagesize }; - var videos = _libraryManager.GetItemList(query); - var numberOfVideos = videos.Count; + var numberOfVideos = _libraryManager.GetCount(query); + + var startIndex = 0; + var numComplete = 0; - // TODO parallelize with Parallel.ForEach? - for (var i = 0; i < numberOfVideos; i++) + while (startIndex < numberOfVideos) { - var video = videos[i]; - // Only local files supported - if (video.IsFileProtocol && File.Exists(video.Path)) + query.StartIndex = startIndex; + + var videos = _libraryManager.GetItemList(query); + var currentPageCount = videos.Count; + // TODO parallelize with Parallel.ForEach? + for (var i = 0; i < currentPageCount; i++) { - for (var j = 0; j < _keyframeExtractors.Length; j++) + var video = videos[i]; + // Only local files supported + if (video.IsFileProtocol && File.Exists(video.Path)) { - var extractor = _keyframeExtractors[j]; - // The cache decorator will make sure to save them in the data dir - if (extractor.TryExtractKeyframes(video.Path, out _)) + for (var j = 0; j < _keyframeExtractors.Length; j++) { - break; + var extractor = _keyframeExtractors[j]; + // The cache decorator will make sure to save them in the data dir + if (extractor.TryExtractKeyframes(video.Path, out _)) + { + break; + } } } + + // Update progress + numComplete++; + double percent = (double)numComplete / numberOfVideos; + progress.Report(100 * percent); } - // Update progress - double percent = (double)(i + 1) / numberOfVideos; - progress.Report(100 * percent); + startIndex += Pagesize; } + progress.Report(100); return Task.CompletedTask; } diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs index e068cac84..fd170864b 100644 --- a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs @@ -106,7 +106,7 @@ internal static class EbmlReaderExtensions if (!tracksPosition.HasValue || !cuesPosition.HasValue || !infoPosition.HasValue) { - throw new InvalidOperationException("SeekHead is missing or does not contain Info, Tracks and Cues positions"); + throw new InvalidOperationException("SeekHead is missing or does not contain Info, Tracks and Cues positions. SeekHead referencing another SeekHead is not supported"); } return new SeekHead(infoPosition.Value, tracksPosition.Value, cuesPosition.Value); diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs index 8bb1ff00d..501b2bb17 100644 --- a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using Jellyfin.MediaEncoding.Keyframes.Matroska.Extensions; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Models; using NEbml.Core; namespace Jellyfin.MediaEncoding.Keyframes.Matroska; @@ -22,8 +23,19 @@ public static class MatroskaKeyframeExtractor using var reader = new EbmlReader(stream); var seekHead = reader.ReadSeekHead(); - var info = reader.ReadInfo(seekHead.InfoPosition); - var videoTrackNumber = reader.FindFirstTrackNumberByType(seekHead.TracksPosition, MatroskaConstants.TrackTypeVideo); + // External lib does not support seeking backwards (yet) + Info info; + ulong videoTrackNumber; + if (seekHead.InfoPosition < seekHead.TracksPosition) + { + info = reader.ReadInfo(seekHead.InfoPosition); + videoTrackNumber = reader.FindFirstTrackNumberByType(seekHead.TracksPosition, MatroskaConstants.TrackTypeVideo); + } + else + { + videoTrackNumber = reader.FindFirstTrackNumberByType(seekHead.TracksPosition, MatroskaConstants.TrackTypeVideo); + info = reader.ReadInfo(seekHead.InfoPosition); + } var keyframes = new List(); reader.ReadAt(seekHead.CuesPosition); -- cgit v1.2.3