diff options
6 files changed, 135 insertions, 45 deletions
diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 13a381060..a2abafd2a 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1860,7 +1860,7 @@ namespace Emby.Server.Implementations.Library try { var index = item.GetImageIndex(img); - image = await ConvertImageToLocal(item, img, index).ConfigureAwait(false); + image = await ConvertImageToLocal(item, img, index, removeOnFailure: true).ConfigureAwait(false); } catch (ArgumentException) { @@ -2787,7 +2787,7 @@ namespace Emby.Server.Implementations.Library await SavePeopleMetadataAsync(people, cancellationToken).ConfigureAwait(false); } - public async Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex) + public async Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex, bool removeOnFailure) { foreach (var url in image.Path.Split('|')) { @@ -2806,6 +2806,7 @@ namespace Emby.Server.Implementations.Library if (ex.StatusCode.HasValue && (ex.StatusCode.Value == HttpStatusCode.NotFound || ex.StatusCode.Value == HttpStatusCode.Forbidden)) { + _logger.LogDebug(ex, "Error downloading image {Url}", url); continue; } @@ -2813,11 +2814,14 @@ namespace Emby.Server.Implementations.Library } } - // Remove this image to prevent it from retrying over and over - item.RemoveImage(image); - await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false); + if (removeOnFailure) + { + // Remove this image to prevent it from retrying over and over + item.RemoveImage(image); + await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false); + } - throw new InvalidOperationException(); + throw new InvalidOperationException("Unable to convert any images to local"); } public async Task AddVirtualFolder(string name, CollectionTypeOptions? collectionType, LibraryOptions options, bool refreshLibrary) diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index e44c09783..6532f7a34 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -517,8 +517,9 @@ namespace MediaBrowser.Controller.Library /// <param name="item">The item.</param> /// <param name="image">The image.</param> /// <param name="imageIndex">Index of the image.</param> + /// <param name="removeOnFailure">Whether to remove the image from the item on failure.</param> /// <returns>Task.</returns> - Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex); + Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex, bool removeOnFailure = true); /// <summary> /// Gets the items. diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index 81a299015..f34034964 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -9,6 +9,7 @@ using System.Net.Http; using System.Net.Mime; using System.Threading; using System.Threading.Tasks; +using AsyncKeyedLock; using Jellyfin.Data.Enums; using Jellyfin.Data.Events; using Jellyfin.Extensions; @@ -30,6 +31,7 @@ using MediaBrowser.Model.Extensions; using MediaBrowser.Model.IO; using MediaBrowser.Model.Net; using MediaBrowser.Model.Providers; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Book = MediaBrowser.Controller.Entities.Book; using Episode = MediaBrowser.Controller.Entities.TV.Episode; @@ -59,6 +61,13 @@ namespace MediaBrowser.Providers.Manager private readonly ConcurrentDictionary<Guid, double> _activeRefreshes = new(); private readonly CancellationTokenSource _disposeCancellationTokenSource = new(); private readonly PriorityQueue<(Guid ItemId, MetadataRefreshOptions RefreshOptions), RefreshPriority> _refreshQueue = new(); + private readonly IMemoryCache _memoryCache; + + private readonly AsyncKeyedLocker<string> _imageSaveLock = new(o => + { + o.PoolSize = 20; + o.PoolInitialFill = 1; + }); private IImageProvider[] _imageProviders = Array.Empty<IImageProvider>(); private IMetadataService[] _metadataServices = Array.Empty<IMetadataService>(); @@ -81,6 +90,7 @@ namespace MediaBrowser.Providers.Manager /// <param name="libraryManager">The library manager.</param> /// <param name="baseItemManager">The BaseItem manager.</param> /// <param name="lyricManager">The lyric manager.</param> + /// <param name="memoryCache">The memory cache.</param> public ProviderManager( IHttpClientFactory httpClientFactory, ISubtitleManager subtitleManager, @@ -91,7 +101,8 @@ namespace MediaBrowser.Providers.Manager IServerApplicationPaths appPaths, ILibraryManager libraryManager, IBaseItemManager baseItemManager, - ILyricManager lyricManager) + ILyricManager lyricManager, + IMemoryCache memoryCache) { _logger = logger; _httpClientFactory = httpClientFactory; @@ -103,6 +114,7 @@ namespace MediaBrowser.Providers.Manager _subtitleManager = subtitleManager; _baseItemManager = baseItemManager; _lyricManager = lyricManager; + _memoryCache = memoryCache; } /// <inheritdoc/> @@ -150,52 +162,79 @@ namespace MediaBrowser.Providers.Manager /// <inheritdoc/> public async Task SaveImage(BaseItem item, string url, ImageType type, int? imageIndex, CancellationToken cancellationToken) { - var httpClient = _httpClientFactory.CreateClient(NamedClient.Default); - using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false); - - if (response.StatusCode != HttpStatusCode.OK) + using (await _imageSaveLock.LockAsync(url, cancellationToken).ConfigureAwait(false)) { - throw new HttpRequestException("Invalid image received.", null, response.StatusCode); - } + if (_memoryCache.TryGetValue(url, out (string ContentType, byte[] ImageContents)? cachedValue) + && cachedValue is not null) + { + var imageContents = cachedValue.Value.ImageContents; + var cacheStream = new MemoryStream(imageContents, 0, imageContents.Length, false); + await using (cacheStream.ConfigureAwait(false)) + { + await SaveImage( + item, + cacheStream, + cachedValue.Value.ContentType, + type, + imageIndex, + cancellationToken).ConfigureAwait(false); + return; + } + } - var contentType = response.Content.Headers.ContentType?.MediaType; + var httpClient = _httpClientFactory.CreateClient(NamedClient.Default); + using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false); - // Workaround for tvheadend channel icons - // TODO: Isolate this hack into the tvh plugin - if (string.IsNullOrEmpty(contentType)) - { - if (url.Contains("/imagecache/", StringComparison.OrdinalIgnoreCase)) + response.EnsureSuccessStatusCode(); + + var contentType = response.Content.Headers.ContentType?.MediaType; + + // Workaround for tvheadend channel icons + // TODO: Isolate this hack into the tvh plugin + if (string.IsNullOrEmpty(contentType)) { - contentType = "image/png"; + if (url.Contains("/imagecache/", StringComparison.OrdinalIgnoreCase)) + { + contentType = MediaTypeNames.Image.Png; + } + else + { + throw new HttpRequestException("Invalid image received: contentType not set.", null, response.StatusCode); + } } - else + + // TVDb will sometimes serve a rubbish 404 html page with a 200 OK code, because reasons... + if (contentType.Equals(MediaTypeNames.Text.Html, StringComparison.OrdinalIgnoreCase)) { - throw new HttpRequestException("Invalid image received: contentType not set.", null, response.StatusCode); + throw new HttpRequestException("Invalid image received.", null, HttpStatusCode.NotFound); } - } - // TVDb will sometimes serve a rubbish 404 html page with a 200 OK code, because reasons... - if (contentType.Equals(MediaTypeNames.Text.Html, StringComparison.OrdinalIgnoreCase)) - { - throw new HttpRequestException("Invalid image received.", null, HttpStatusCode.NotFound); - } + // some iptv/epg providers don't correctly report media type, extract from url if no extension found + if (string.IsNullOrWhiteSpace(MimeTypes.ToExtension(contentType))) + { + // Strip query parameters from url to get actual path. + contentType = MimeTypes.GetMimeType(new Uri(url).GetLeftPart(UriPartial.Path)); + } - // some iptv/epg providers don't correctly report media type, extract from url if no extension found - if (string.IsNullOrWhiteSpace(MimeTypes.ToExtension(contentType))) - { - contentType = MimeTypes.GetMimeType(url); - } + if (!contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + { + throw new HttpRequestException($"Request returned {contentType} instead of an image type", null, HttpStatusCode.NotFound); + } - var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - await using (stream.ConfigureAwait(false)) - { - await SaveImage( - item, - stream, - contentType, - type, - imageIndex, - cancellationToken).ConfigureAwait(false); + var responseBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var stream = new MemoryStream(responseBytes, 0, responseBytes.Length, false); + await using (stream.ConfigureAwait(false)) + { + _memoryCache.Set(url, (contentType, responseBytes), TimeSpan.FromSeconds(10)); + + await SaveImage( + item, + stream, + contentType, + type, + imageIndex, + cancellationToken).ConfigureAwait(false); + } } } @@ -1115,6 +1154,7 @@ namespace MediaBrowser.Providers.Manager } _disposeCancellationTokenSource.Dispose(); + _imageSaveLock.Dispose(); } _disposed = true; diff --git a/MediaBrowser.Providers/MediaBrowser.Providers.csproj b/MediaBrowser.Providers/MediaBrowser.Providers.csproj index 7a50c6cf4..dfb6319ac 100644 --- a/MediaBrowser.Providers/MediaBrowser.Providers.csproj +++ b/MediaBrowser.Providers/MediaBrowser.Providers.csproj @@ -15,6 +15,7 @@ </ItemGroup> <ItemGroup> + <PackageReference Include="AsyncKeyedLock" /> <PackageReference Include="LrcParser" /> <PackageReference Include="MetaBrainz.MusicBrainz" /> <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" /> diff --git a/src/Jellyfin.LiveTv/Guide/GuideManager.cs b/src/Jellyfin.LiveTv/Guide/GuideManager.cs index 39f174cc2..093970c38 100644 --- a/src/Jellyfin.LiveTv/Guide/GuideManager.cs +++ b/src/Jellyfin.LiveTv/Guide/GuideManager.cs @@ -27,6 +27,8 @@ public class GuideManager : IGuideManager private const string EtagKey = "ProgramEtag"; private const string ExternalServiceTag = "ExternalServiceId"; + private static readonly ParallelOptions _cacheParallelOptions = new() { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 10) }; + private readonly ILogger<GuideManager> _logger; private readonly IConfigurationManager _config; private readonly IFileSystem _fileSystem; @@ -209,6 +211,7 @@ public class GuideManager : IGuideManager _logger.LogInformation("Refreshing guide with {0} days of guide data", guideDays); + var maxCacheDate = DateTime.UtcNow.AddDays(2); foreach (var currentChannel in list) { cancellationToken.ThrowIfCancellationRequested(); @@ -263,6 +266,7 @@ public class GuideManager : IGuideManager if (newPrograms.Count > 0) { _libraryManager.CreateItems(newPrograms, null, cancellationToken); + await PrecacheImages(newPrograms, maxCacheDate).ConfigureAwait(false); } if (updatedPrograms.Count > 0) @@ -272,6 +276,7 @@ public class GuideManager : IGuideManager currentChannel, ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false); + await PrecacheImages(updatedPrograms, maxCacheDate).ConfigureAwait(false); } currentChannel.IsMovie = isMovie; @@ -708,4 +713,41 @@ public class GuideManager : IGuideManager return (item, isNew, isUpdated); } + + private async Task PrecacheImages(IReadOnlyList<BaseItem> programs, DateTime maxCacheDate) + { + await Parallel.ForEachAsync( + programs + .Where(p => p.EndDate.HasValue && p.EndDate.Value < maxCacheDate) + .DistinctBy(p => p.Id), + _cacheParallelOptions, + async (program, cancellationToken) => + { + for (var i = 0; i < program.ImageInfos.Length; i++) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + var imageInfo = program.ImageInfos[i]; + if (!imageInfo.IsLocalFile) + { + try + { + program.ImageInfos[i] = await _libraryManager.ConvertImageToLocal( + program, + imageInfo, + imageIndex: 0, + removeOnFailure: false) + .ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Unable to precache {Url}", imageInfo.Path); + } + } + } + }).ConfigureAwait(false); + } } diff --git a/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs b/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs index 478db6941..6fccce049 100644 --- a/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs +++ b/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs @@ -17,6 +17,7 @@ using MediaBrowser.Controller.Subtitles; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.IO; using MediaBrowser.Providers.Manager; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -572,7 +573,8 @@ namespace Jellyfin.Providers.Tests.Manager Mock.Of<IServerApplicationPaths>(), libraryManager.Object, baseItemManager!, - Mock.Of<ILyricManager>()); + Mock.Of<ILyricManager>(), + Mock.Of<IMemoryCache>()); return providerManager; } |