From 511f1ef882d594df64810d2c36b59ad381fd6e17 Mon Sep 17 00:00:00 2001 From: Shadowghost Date: Fri, 20 Nov 2026 07:23:42 -0500 Subject: Backport pull request #18048 from jellyfin/release-12.z Follow the Schedules Direct API contract when refreshing the guide Original-merge: 85fa90cb758dabfb434fbd00c1128091007cf6a4 Merged-by: Bond-009 Backported-by: Cody Robibero --- .../Library/LibraryManager.cs | 20 +- Jellyfin.Api/Controllers/ImageController.cs | 15 +- .../Net/HttpResponseBodyException.cs | 31 + .../LiveTv/ISchedulesDirectService.cs | 37 +- .../LiveTv/ImageDownloadFailureAction.cs | 17 + MediaBrowser.Providers/Manager/ProviderManager.cs | 23 +- src/Jellyfin.LiveTv/Guide/GuideManager.cs | 48 +- src/Jellyfin.LiveTv/Listings/ListingsManager.cs | 52 +- src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs | 1112 +++++++++++++++++--- .../Listings/SchedulesDirectCachedDay.cs | 29 + .../Listings/SchedulesDirectDtos/DayDto.cs | 29 + .../SchedulesDirectDtos/ProgramDetailsDto.cs | 13 + .../Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs | 34 + .../Listings/SchedulesDirectDtos/SdErrorCode.cs | 237 ++++- .../SchedulesDirectDtos/StatusAccountDto.cs | 22 + .../Listings/SchedulesDirectDtos/StatusDto.cs | 35 + .../SchedulesDirectDtos/StatusLineupDto.cs | 35 + .../SchedulesDirectDtos/SystemStatusDto.cs | 28 + .../Listings/SchedulesDirectLineupCache.cs | 117 ++ .../Listings/SchedulesDirectScheduleCache.cs | 205 ++++ .../Recordings/RecordingsMetadataManager.cs | 15 +- .../Guide/GuideManagerTests.cs | 95 ++ .../Listings/ListingsManagerTests.cs | 44 + .../Listings/SchedulesDirectLineupTests.cs | 130 --- .../Listings/SchedulesDirectTests.cs | 915 ++++++++++++++++ .../Recordings/RecordingsMetadataManagerTests.cs | 49 +- .../Manager/ProviderManagerTests.cs | 40 +- .../Library/LibraryManagerConvertImageTests.cs | 50 + 28 files changed, 3135 insertions(+), 342 deletions(-) create mode 100644 MediaBrowser.Common/Net/HttpResponseBodyException.cs create mode 100644 MediaBrowser.Controller/LiveTv/ImageDownloadFailureAction.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectCachedDay.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusAccountDto.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusDto.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusLineupDto.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SystemStatusDto.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectLineupCache.cs create mode 100644 src/Jellyfin.LiveTv/Listings/SchedulesDirectScheduleCache.cs create mode 100644 tests/Jellyfin.LiveTv.Tests/Guide/GuideManagerTests.cs delete mode 100644 tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectLineupTests.cs create mode 100644 tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectTests.cs create mode 100644 tests/Jellyfin.Server.Implementations.Tests/Library/LibraryManagerConvertImageTests.cs diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 2aa41f020d..359b0f1d0c 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -3769,6 +3769,7 @@ namespace Emby.Server.Implementations.Library public async Task ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex, bool removeOnFailure) { + HttpRequestException? lastHttpException = null; foreach (var url in image.Path.Split('|')) { try @@ -3800,6 +3801,7 @@ namespace Emby.Server.Implementations.Library && (ex.StatusCode.Value == HttpStatusCode.NotFound || ex.StatusCode.Value == HttpStatusCode.Forbidden)) { _logger.LogDebug(ex, "Error downloading image {Url}", url); + lastHttpException = ex; continue; } @@ -3814,13 +3816,17 @@ namespace Emby.Server.Implementations.Library await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false); } - throw new InvalidOperationException(string.Format( - CultureInfo.InvariantCulture, - "Unable to convert any {0} image url in \"{1}\" to a local file for {2} ({3})", - image.Type, - image.Path, - item.Name, - item.Id)); + // Keep the rejection as the inner exception: callers such as the Live TV guide need the + // status code to tell a missing image from a provider that is refusing us. + throw new InvalidOperationException( + string.Format( + CultureInfo.InvariantCulture, + "Unable to convert any {0} image url in \"{1}\" to a local file for {2} ({3})", + image.Type, + image.Path, + item.Name, + item.Id), + lastHttpException); } public async Task AddVirtualFolder(string name, CollectionTypeOptions? collectionType, LibraryOptions options, bool refreshLibrary) diff --git a/Jellyfin.Api/Controllers/ImageController.cs b/Jellyfin.Api/Controllers/ImageController.cs index d492a2f5ba..f04d24cbcc 100644 --- a/Jellyfin.Api/Controllers/ImageController.cs +++ b/Jellyfin.Api/Controllers/ImageController.cs @@ -21,6 +21,7 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.LiveTv; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Branding; using MediaBrowser.Model.Drawing; @@ -50,6 +51,7 @@ public class ImageController : BaseJellyfinApiController private readonly ILogger _logger; private readonly IServerConfigurationManager _serverConfigurationManager; private readonly IApplicationPaths _appPaths; + private readonly ISchedulesDirectService _schedulesDirectService; /// /// Initializes a new instance of the class. @@ -62,6 +64,7 @@ public class ImageController : BaseJellyfinApiController /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. + /// Instance of the interface. public ImageController( IUserManager userManager, ILibraryManager libraryManager, @@ -70,7 +73,8 @@ public class ImageController : BaseJellyfinApiController IFileSystem fileSystem, ILogger logger, IServerConfigurationManager serverConfigurationManager, - IApplicationPaths appPaths) + IApplicationPaths appPaths, + ISchedulesDirectService schedulesDirectService) { _userManager = userManager; _libraryManager = libraryManager; @@ -80,6 +84,7 @@ public class ImageController : BaseJellyfinApiController _logger = logger; _serverConfigurationManager = serverConfigurationManager; _appPaths = appPaths; + _schedulesDirectService = schedulesDirectService; } private static CryptoStream GetFromBase64Stream(Stream inputStream) @@ -1893,7 +1898,13 @@ public class ImageController : BaseJellyfinApiController if (!imageInfo.IsLocalFile && item is not null) { - imageInfo = await _libraryManager.ConvertImageToLocal(item, imageInfo, imageIndex ?? 0).ConfigureAwait(false); + var localImage = await _schedulesDirectService.ConvertImageToLocal(item, imageInfo, imageIndex ?? 0).ConfigureAwait(false); + if (localImage is null) + { + return NotFound(); + } + + imageInfo = localImage; } var options = new ImageProcessingOptions diff --git a/MediaBrowser.Common/Net/HttpResponseBodyException.cs b/MediaBrowser.Common/Net/HttpResponseBodyException.cs new file mode 100644 index 0000000000..2194640b61 --- /dev/null +++ b/MediaBrowser.Common/Net/HttpResponseBodyException.cs @@ -0,0 +1,31 @@ +using System.Net; +using System.Net.Http; + +namespace MediaBrowser.Common.Net; + +/// +/// An that retains the body of the failed response. +/// +/// +/// Providers report why they refused a request in the response body, which +/// discards. +/// +public class HttpResponseBodyException : HttpRequestException +{ + /// + /// Initializes a new instance of the class. + /// + /// The exception message. + /// The status code of the failed response. + /// The body of the failed response. + public HttpResponseBodyException(string message, HttpStatusCode? statusCode, string? responseBody) + : base(message, null, statusCode) + { + ResponseBody = responseBody; + } + + /// + /// Gets the body of the failed response. + /// + public string? ResponseBody { get; } +} diff --git a/MediaBrowser.Controller/LiveTv/ISchedulesDirectService.cs b/MediaBrowser.Controller/LiveTv/ISchedulesDirectService.cs index 6953650952..a0df011811 100644 --- a/MediaBrowser.Controller/LiveTv/ISchedulesDirectService.cs +++ b/MediaBrowser.Controller/LiveTv/ISchedulesDirectService.cs @@ -1,6 +1,8 @@ using System.IO; +using System.Net; using System.Threading; using System.Threading.Tasks; +using MediaBrowser.Controller.Entities; namespace MediaBrowser.Controller.LiveTv; @@ -17,15 +19,36 @@ public interface ISchedulesDirectService Task GetAvailableCountries(CancellationToken cancellationToken); /// - /// Gets a value indicating whether the Schedules Direct daily image download limit is currently active. + /// Gets a value indicating whether an image may be downloaded from the given url. /// - /// true if the image limit has been hit and has not yet reset; otherwise false. - bool IsImageDailyLimitActive(); + /// The image url. + /// false for a Schedules Direct url while the daily image limit is active; otherwise true. + bool CanDownloadImage(string imageUrl); /// - /// Gets a value indicating whether the Schedules Direct service is available. - /// Returns false if a permanent account error has occurred or a transient backoff is active. + /// Reports a successful image download, so that an unexplained failure streak is reset. /// - /// true if the service can accept requests; otherwise false. - bool IsServiceAvailable(); + /// The image url that was downloaded. + void ReportImageDownloadSuccess(string imageUrl); + + /// + /// Reports a failed image download. Schedules Direct answers an exhausted image quota with + /// an HTTP 200 JSON error body rather than a failure status, so the body is what decides + /// whether image acquisition has to stop. + /// + /// The image url that failed. + /// The HTTP status code of the failure, if any. + /// The body of the failed response, if it was readable. + /// What the caller must do with the image. + ImageDownloadFailureAction ReportImageDownloadFailure(string imageUrl, HttpStatusCode? statusCode, string? responseBody); + + /// + /// Downloads a remote image unless Schedules Direct has blocked it, and reports the outcome. + /// Schedules Direct counts every image request, so downloads outside the guide refresh go through here. + /// + /// The item the image belongs to. + /// The remote image. + /// The image index. + /// The local image, or null when the download is blocked. + Task ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex); } diff --git a/MediaBrowser.Controller/LiveTv/ImageDownloadFailureAction.cs b/MediaBrowser.Controller/LiveTv/ImageDownloadFailureAction.cs new file mode 100644 index 0000000000..78a1e554b6 --- /dev/null +++ b/MediaBrowser.Controller/LiveTv/ImageDownloadFailureAction.cs @@ -0,0 +1,17 @@ +namespace MediaBrowser.Controller.LiveTv; + +/// +/// What the caller must do with an image after a failed download. +/// +public enum ImageDownloadFailureAction +{ + /// + /// Keep the image; the download may succeed on a later attempt. + /// + None = 0, + + /// + /// Drop the image. The url is permanently invalid and re-requesting it gets the account blocked. + /// + RemoveImage = 1 +} diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index 855765441b..3179c7a4a8 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -212,7 +212,16 @@ namespace MediaBrowser.Providers.Manager var httpClient = _httpClientFactory.CreateClient(NamedClient.Default); using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + if (!response.IsSuccessStatusCode) + { + // Keep the body: providers report why they refused in it, and a caller that + // only sees the status code cannot tell a missing image from an exhausted quota. + var errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new HttpResponseBodyException( + $"Response status code does not indicate success: {(int)response.StatusCode} ({response.ReasonPhrase}).", + response.StatusCode, + errorBody); + } var contentType = response.Content.Headers.ContentType?.MediaType; @@ -236,7 +245,17 @@ namespace MediaBrowser.Providers.Manager if (!contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) { - throw new HttpRequestException($"Request returned '{contentType}' instead of an image type", null, HttpStatusCode.NotFound); + // A provider can answer 200 with a JSON error instead of the image; Schedules + // Direct reports an exhausted image quota that way, so keep the body. + var errorBody = contentType.Contains("json", StringComparison.OrdinalIgnoreCase) + || contentType.StartsWith("text/", StringComparison.OrdinalIgnoreCase) + ? await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false) + : null; + + throw new HttpResponseBodyException( + $"Request returned '{contentType}' instead of an image type", + HttpStatusCode.NotFound, + errorBody); } var responseBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/Jellyfin.LiveTv/Guide/GuideManager.cs b/src/Jellyfin.LiveTv/Guide/GuideManager.cs index a11f83f2f8..15486081b0 100644 --- a/src/Jellyfin.LiveTv/Guide/GuideManager.cs +++ b/src/Jellyfin.LiveTv/Guide/GuideManager.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; @@ -8,6 +9,7 @@ using Jellyfin.Extensions; using Jellyfin.LiveTv.Configuration; using Jellyfin.LiveTv.Listings; using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; @@ -747,26 +749,17 @@ public class GuideManager : IGuideManager return false; } - private async Task PreCacheImages(IReadOnlyList programs, DateTime maxCacheDate) + internal async Task PreCacheImages(IReadOnlyList programs, DateTime maxCacheDate) { - var sdLimitActive = _schedulesDirectService.IsImageDailyLimitActive(); - await Parallel.ForEachAsync( programs .Where(p => p.EndDate.HasValue && p.EndDate.Value < maxCacheDate) - .Where(p => !sdLimitActive || !p.ImageInfos.All( - img => img.IsLocalFile || img.Path.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase))) + .Where(p => p.ImageInfos.Any(img => !img.IsLocalFile)) .DistinctBy(p => p.Id), _cacheParallelOptions, async (program, cancellationToken) => { - // Re-check: limit may have been set by a parallel task since the LINQ filter ran. - if (_schedulesDirectService.IsImageDailyLimitActive() - && program.ImageInfos.All( - img => img.IsLocalFile || img.Path.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase))) - { - return; - } + List? deadImages = null; for (var i = 0; i < program.ImageInfos.Length; i++) { @@ -781,10 +774,12 @@ public class GuideManager : IGuideManager continue; } - // Skip SD downloads once the daily limit has been hit. - if (imageInfo.Path.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase) - && _schedulesDirectService.IsImageDailyLimitActive()) + // Re-checked per image: the limit may have been hit by a parallel task. A blocked + // url left on the program would still be fetched when a client asks for it. + if (!_schedulesDirectService.CanDownloadImage(imageInfo.Path)) { + deadImages ??= []; + deadImages.Add(imageInfo); continue; } @@ -797,12 +792,35 @@ public class GuideManager : IGuideManager imageIndex: 0, removeOnFailure: false) .ConfigureAwait(false); + _schedulesDirectService.ReportImageDownloadSuccess(imageInfo.Path); } catch (Exception ex) { + // A rejected download never surfaces as an error on its own, so feed the + // response back to the provider to stop us hammering an exhausted account. + var httpException = (ex as HttpRequestException) ?? ex.InnerException as HttpRequestException; + var action = _schedulesDirectService.ReportImageDownloadFailure( + imageInfo.Path, + httpException?.StatusCode, + (httpException as HttpResponseBodyException)?.ResponseBody); + + if (action == ImageDownloadFailureAction.RemoveImage) + { + deadImages ??= []; + deadImages.Add(imageInfo); + } + _logger.LogWarning(ex, "Unable to pre-cache {Url}", imageInfo.Path); } } + + if (deadImages is not null) + { + // Requesting a url Schedules Direct has already rejected is what gets an + // account blocked, so it must not survive to the next refresh. + program.RemoveImages(deadImages); + await program.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, cancellationToken).ConfigureAwait(false); + } }).ConfigureAwait(false); } } diff --git a/src/Jellyfin.LiveTv/Listings/ListingsManager.cs b/src/Jellyfin.LiveTv/Listings/ListingsManager.cs index 7274a3030c..2f4afc2b27 100644 --- a/src/Jellyfin.LiveTv/Listings/ListingsManager.cs +++ b/src/Jellyfin.LiveTv/Listings/ListingsManager.cs @@ -124,6 +124,8 @@ public class ListingsManager : IListingsManager { ArgumentNullException.ThrowIfNull(channel); + Exception? failure = null; + foreach (var (provider, providerInfo) in GetListingProviders()) { if (!IsListingProviderEnabledForTuner(providerInfo, channel.TunerHostId)) @@ -144,19 +146,42 @@ public class ListingsManager : IListingsManager provider.Name, providerInfo.ListingsId ?? string.Empty); - var epgChannels = await GetEpgChannels(provider, providerInfo, true, cancellationToken).ConfigureAwait(false); + List programs; + try + { + // Listing the provider's channels can fail as well, and that must not stop the + // fallback to the next provider either. + var epgChannels = await GetEpgChannels(provider, providerInfo, true, cancellationToken).ConfigureAwait(false); - var epgChannel = GetEpgChannelFromTunerChannel(providerInfo.ChannelMappings, channel, epgChannels); - if (epgChannel is null) + var epgChannel = GetEpgChannelFromTunerChannel(providerInfo.ChannelMappings, channel, epgChannels); + if (epgChannel is null) + { + _logger.LogDebug("EPG channel not found for tuner channel {0}-{1} from {2}-{3}", channel.Number, channel.Name, provider.Name, providerInfo.ListingsId ?? string.Empty); + continue; + } + + programs = (await provider + .GetProgramsAsync(providerInfo, epgChannel.Id, startDateUtc, endDateUtc, cancellationToken).ConfigureAwait(false)) + .ToList(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) { - _logger.LogDebug("EPG channel not found for tuner channel {0}-{1} from {2}-{3}", channel.Number, channel.Name, provider.Name, providerInfo.ListingsId ?? string.Empty); + // Another provider may still be able to serve this channel, so the failure is + // held back rather than thrown, and only surfaces if nothing else answers. + _logger.LogError( + ex, + "Error getting programs for channel {Number}-{Name} from {Provider}", + channel.Number, + channel.Name, + provider.Name); + failure ??= ex; continue; } - var programs = (await provider - .GetProgramsAsync(providerInfo, epgChannel.Id, startDateUtc, endDateUtc, cancellationToken).ConfigureAwait(false)) - .ToList(); - // Replace the value that came from the provider with a normalized value foreach (var program in programs) { @@ -170,7 +195,16 @@ public class ListingsManager : IListingsManager } } - return Enumerable.Empty(); + // Reporting no programs would have the guide refresh delete the ones this channel already + // has, so a channel nothing could be fetched for has to fail instead. + if (failure is not null) + { + throw new InvalidOperationException( + $"No listings provider could supply programs for channel {channel.Number}-{channel.Name}", + failure); + } + + return []; } /// diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs index d2076767b8..801c4f163b 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs @@ -25,6 +25,8 @@ using Jellyfin.LiveTv.Listings.SchedulesDirectDtos; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Authentication; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; @@ -38,31 +40,76 @@ namespace Jellyfin.LiveTv.Listings private const string ApiUrl = "https://json.schedulesdirect.org/20141201"; private const int CountryCacheDays = 7; + // Schedules Direct disables an account that keeps requesting images after the daily + // download limit, so stop after a short streak of rejections instead of retrying forever. + private const int MaxConsecutiveImageFailures = 10; + + // Hard server-side cap on elements in a single programs request. + private const int MaxProgramsPerRequest = 5000; + + // Memory ceiling on the rejected image uris (~2MB of 64-char keys). The daily invalid-uri + // limit stops the account long before this many can accumulate. + private const int MaxRejectedImageUris = 10000; + + // Stations that stopped being queried keep their cached schedules for this long. + private const int ScheduleCacheMaxAgeDays = 30; + + // Long enough to outlive any restart, short enough that the list cannot fill up for good. + private const int RejectedImageUriMaxAgeDays = 30; + + // The spec requires a client to stay away for an hour once the service reports Offline. + private const int OfflineBackoffMinutes = 60; + + private const int LoginBackoffMinutes = 30; + private const int MaxLoginsBackoffHours = 24; + + private const int StatusCacheMinutes = 10; + private readonly ILogger _logger; private readonly IHttpClientFactory _httpClientFactory; private readonly IApplicationPaths _appPaths; + private readonly ILibraryManager _libraryManager; private readonly AsyncNonKeyedLocker _tokenLock = new(1); + private readonly AsyncNonKeyedLocker _statusLock = new(1); + private readonly SchedulesDirectScheduleCache _scheduleCache; + private readonly SchedulesDirectLineupCache _lineupCache; + + // Image uris Schedules Direct answered with IMAGE_NOT_FOUND. Requesting one of these + // again is what trips MAX_IMAGE_INVALID_URI_ERRORS and gets the account blocked. + private readonly ConcurrentDictionary _rejectedImageUris = new(StringComparer.Ordinal); + private readonly Lock _rejectedImageUrisFileLock = new(); + + // Station/date pairs the server has queued for generation, with the retryTime it gave. + private readonly ConcurrentDictionary _scheduleRetryTimes = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _tokens = new(); private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; - private long _lastErrorResponseTicks; + private long _loginBackoffUntilTicks; private volatile bool _accountError; - private bool _disposed = false; + private bool _disposed; private byte[] _countriesCache; private DateOnly? _imageLimitHitDate; private DateOnly? _metadataLimitHitDate; + private int _consecutiveImageFailures; + private long _offlineUntilTicks; + private StatusCacheEntry _status; public SchedulesDirect( ILogger logger, IHttpClientFactory httpClientFactory, - IApplicationPaths appPaths) + IApplicationPaths appPaths, + ILibraryManager libraryManager) { _logger = logger; _httpClientFactory = httpClientFactory; _appPaths = appPaths; + _libraryManager = libraryManager; + _scheduleCache = new SchedulesDirectScheduleCache(logger, appPaths); + _lineupCache = new SchedulesDirectLineupCache(logger, appPaths); _imageLimitHitDate = LoadDailyLimitDate(ImageLimitFilePath); _metadataLimitHitDate = LoadDailyLimitDate(MetadataLimitFilePath); + LoadRejectedImageUris(); } /// @@ -72,6 +119,8 @@ namespace Jellyfin.LiveTv.Listings private string MetadataLimitFilePath => Path.Combine(_appPaths.CachePath, "sd-metadata-limit.txt"); + private string RejectedImageUrisFilePath => Path.Combine(_appPaths.CachePath, "sd-rejected-images.txt"); + /// public string Type => nameof(SchedulesDirect); @@ -93,9 +142,11 @@ namespace Jellyfin.LiveTv.Listings public async Task> GetProgramsAsync(ListingsProviderInfo info, string channelId, DateTime startDateUtc, DateTime endDateUtc, CancellationToken cancellationToken) { + // A failure must not come back as "this channel has no programs": GuideManager only + // skips CleanDatabase when a channel throws, so an empty list deletes the guide. if (IsMetadataLimitActive()) { - return []; + throw new InvalidOperationException("The Schedules Direct daily request limit has been reached."); } ArgumentException.ThrowIfNullOrEmpty(channelId); @@ -107,108 +158,461 @@ namespace Jellyfin.LiveTv.Listings if (string.IsNullOrEmpty(token)) { - _logger.LogWarning("SchedulesDirect token is empty, returning empty program list"); + throw new AuthenticationException("Could not authenticate with Schedules Direct"); + } - return []; + if (!await IsSystemOnline(info, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline."); } var dates = GetScheduleRequestDates(startDateUtc, endDateUtc); - _logger.LogInformation("Channel Station ID is: {ChannelID}", channelId); - var requestList = new List() + + // Ask what changed before downloading anything; a day whose md5 still matches is + // served from the cache and costs no schedule or program request. + var (md5s, unavailableDates) = await GetScheduleMd5s(info, channelId, dates, cancellationToken).ConfigureAwait(false); + + var cachedDays = new List(); + var datesToFetch = new List(); + foreach (var date in dates) + { + if (unavailableDates.Contains(date)) { - new() - { - StationId = channelId, - Date = dates - } - }; + // The hash check already said the server has no schedule for this day, so + // requesting it would only earn the same error again. + continue; + } - _logger.LogDebug("Request string for schedules is: {@RequestString}", requestList); + md5s.TryGetValue(date, out var md5); + var cached = await _scheduleCache.GetAsync(channelId, date, md5, cancellationToken).ConfigureAwait(false); + if (cached?.Schedule is not null) + { + cachedDays.Add(cached); + } + else if (!IsScheduleRetryPending(channelId, date)) + { + datesToFetch.Add(date); + } + } - using var options = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/schedules"); - options.Content = JsonContent.Create(requestList, options: _jsonOptions); - options.Headers.TryAddWithoutValidation("token", token); - var dailySchedules = await Request>(options, true, info, cancellationToken).ConfigureAwait(false); - if (dailySchedules is null) + _logger.LogDebug( + "Channel {ChannelID}: {CachedCount} of {TotalCount} days served from cache", + channelId, + cachedDays.Count, + dates.Count); + + if (datesToFetch.Count > 0) { - return []; + // The hash check may have stopped us (offline, a login limit) or replaced the token, + // and its failure is swallowed, so the schedule request must not go out regardless. + token = await GetToken(info, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrEmpty(token) || IsMetadataLimitActive()) + { + throw new InvalidOperationException("Schedules Direct stopped accepting requests while checking for schedule changes."); + } } - _logger.LogDebug("Found {ScheduleCount} programs on {ChannelID} ScheduleDirect", dailySchedules.Count, channelId); + var freshDays = await DownloadSchedules(info, channelId, datesToFetch, token, cancellationToken).ConfigureAwait(false); - using var programRequestOptions = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/programs"); - programRequestOptions.Headers.TryAddWithoutValidation("token", token); + // A day that is neither cached nor downloaded (queued, a per-day error, a pending retry) + // must not go missing: the guide refresh would delete its programs. The previous copy + // stands in for it, and without one the whole channel fails. + var coveredDates = cachedDays.Concat(freshDays) + .Select(d => d.Schedule?.Metadata?.StartDate?.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)) + .ToHashSet(StringComparer.Ordinal); + foreach (var date in dates) + { + if (unavailableDates.Contains(date) || coveredDates.Contains(date)) + { + continue; + } + + var stale = await _scheduleCache.GetLatestAsync(channelId, date, cancellationToken).ConfigureAwait(false); + if (stale?.Schedule is null) + { + throw new InvalidOperationException( + string.Format( + CultureInfo.InvariantCulture, + "Could not get the schedule for station {0} on {1}", + channelId, + date)); + } + + _logger.LogDebug("Serving the previous schedule for {StationId} {Date} until a new one is available", channelId, date); + cachedDays.Add(stale); + } - var programIds = dailySchedules.SelectMany(d => d.Programs.Select(s => s.ProgramId)).Distinct(); - programRequestOptions.Content = JsonContent.Create(programIds, options: _jsonOptions); + _scheduleCache.Prune(channelId, dates); - var programDetails = await Request>(programRequestOptions, true, info, cancellationToken).ConfigureAwait(false); - if (programDetails is null) + // Only reachable when the server reported every day as unavailable. + var allDays = cachedDays.Select(c => c.Schedule!).Concat(freshDays.Select(f => f.Schedule!)).ToList(); + if (allDays.Count == 0) { return []; } - var programDict = programDetails.ToDictionary(p => p.ProgramId, y => y); - - var programIdsWithImages = programDetails - .Where(p => p.HasImageArtwork) - .Select(p => p.ProgramId) + var programDict = cachedDays.Concat(freshDays) + .SelectMany(d => d.Programs) + .Where(p => !string.IsNullOrEmpty(p.ProgramId)) + .DistinctBy(p => p.ProgramId) + .ToDictionary(p => p.ProgramId, y => y); + + // Artwork uris are ephemeral and must not be cached, and artwork changes are not + // reflected in the schedule md5 either, so the index is requested fresh - but only for + // the programs whose images are actually going to be pre-cached. + var imageProgramIds = allDays.SelectMany(d => d.Programs) + .Where(s => WillBeImageCached(s) + && !string.IsNullOrEmpty(s.ProgramId) + && programDict.TryGetValue(s.ProgramId, out var detail) + && detail.HasImageArtwork) + .Select(s => s.ProgramId) + .Distinct() .ToList(); - var images = await GetImageForPrograms(info, programIdsWithImages, cancellationToken).ConfigureAwait(false); + var images = await GetImageForPrograms(info, imageProgramIds, cancellationToken).ConfigureAwait(false); var programsInfo = new List(); - foreach (ProgramDto schedule in dailySchedules.SelectMany(d => d.Programs)) + foreach (ProgramDto schedule in allDays.SelectMany(d => d.Programs)) { - if (string.IsNullOrEmpty(schedule.ProgramId)) + if (string.IsNullOrEmpty(schedule.ProgramId) + || !programDict.TryGetValue(schedule.ProgramId, out var programDetail)) { continue; } // Only add images which will be pre-cached until we can implement dynamic token fetching - var endDate = schedule.AirDateTime?.AddSeconds(schedule.Duration); - var willBeCached = endDate.HasValue && endDate.Value < DateTime.UtcNow.AddDays(GuideManager.MaxCacheDays); - if (willBeCached && images is not null) + if (WillBeImageCached(schedule)) { - var imageIndex = images.FindIndex(i => - i.ProgramId is not null && schedule.ProgramId.StartsWith(i.ProgramId, StringComparison.Ordinal)); - if (imageIndex > -1) - { - var programEntry = programDict[schedule.ProgramId]; + SetProgramImages(schedule.ProgramId, programDetail, images, token); + } - var allImages = images[imageIndex].Data; - var imagesWithText = allImages.Where(i => string.Equals(i.Text, "yes", StringComparison.OrdinalIgnoreCase)).ToList(); - var imagesWithoutText = allImages.Where(i => string.Equals(i.Text, "no", StringComparison.OrdinalIgnoreCase)).ToList(); + programsInfo.Add(GetProgram(channelId, schedule, programDetail)); + } - const double DesiredAspect = 2.0 / 3; + return programsInfo; + } - programEntry.PrimaryImage = GetProgramImage(ApiUrl, imagesWithText, DesiredAspect, token) ?? - GetProgramImage(ApiUrl, allImages, DesiredAspect, token); + private async Task<(Dictionary Md5s, HashSet UnavailableDates)> GetScheduleMd5s( + ListingsProviderInfo info, + string stationId, + IReadOnlyList dates, + CancellationToken cancellationToken) + { + var result = new Dictionary(StringComparer.Ordinal); + var unavailable = new HashSet(StringComparer.Ordinal); + + var token = await GetToken(info, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrEmpty(token)) + { + return (result, unavailable); + } - const double WideAspect = 16.0 / 9; + var requestList = new List + { + new() { StationId = stationId, Date = dates } + }; - programEntry.ThumbImage = GetProgramImage(ApiUrl, imagesWithText, WideAspect, token); + using var message = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/schedules/md5"); + message.Content = JsonContent.Create(requestList, options: _jsonOptions); + message.Headers.TryAddWithoutValidation("token", token); - // Don't supply the same image twice - if (string.Equals(programEntry.PrimaryImage, programEntry.ThumbImage, StringComparison.Ordinal)) + try + { + var response = await Request>>(message, true, info, cancellationToken) + .ConfigureAwait(false); + if (response is null || !response.TryGetValue(stationId, out var days)) + { + return (result, unavailable); + } + + foreach (var (date, entry) in days) + { + if (entry.Code != 0) + { + // Only a definitive "there is no such day" is worth skipping; a transient + // error still gets the normal request, which may well succeed. + if (IsDefinitiveScheduleError(entry.Code)) { - programEntry.ThumbImage = null; + unavailable.Add(date); } - programEntry.BackdropImage = GetProgramImage(ApiUrl, imagesWithoutText, WideAspect, token); + LogEntryErrors([(entry.Code, date, entry.Message)], "schedule"); + } + else if (!string.IsNullOrEmpty(entry.Md5)) + { + result[date] = entry.Md5; + } + } + } + catch (Exception ex) when (ex is HttpRequestException or JsonException) + { + // Without md5s every requested day is treated as changed, which is the old + // behaviour. Nothing is marked unavailable, so no day is skipped on a failure. + _logger.LogWarning(ex, "Unable to get schedule hashes for station {StationId}", stationId); + } + + return (result, unavailable); + } + + // Whether the code means the day will never be available, as opposed to a transient + // failure that is worth another request. + private static bool IsDefinitiveScheduleError(int code) + => code is (int)SdErrorCode.ScheduleRangeExceeded + or (int)SdErrorCode.ScheduleNotInLineup + or (int)SdErrorCode.StationIdNotFound + or (int)SdErrorCode.StationIdDeleted; + + private bool IsScheduleRetryPending(string stationId, string date) + { + var key = stationId + "|" + date; + if (!_scheduleRetryTimes.TryGetValue(key, out var retryTime)) + { + return false; + } + + if (DateTime.UtcNow < retryTime) + { + _logger.LogDebug( + "Schedules Direct has queued {StationId} {Date}; not requesting it before {RetryTime}", + stationId, + date, + retryTime); + return true; + } + + _scheduleRetryTimes.TryRemove(key, out _); + return false; + } + + private static bool WillBeImageCached(ProgramDto schedule) + { + var endDate = schedule.AirDateTime?.AddSeconds(schedule.Duration); + return endDate.HasValue && endDate.Value < DateTime.UtcNow.AddDays(GuideManager.MaxCacheDays); + } + + private async Task> DownloadSchedules( + ListingsProviderInfo info, + string stationId, + IReadOnlyList dates, + string token, + CancellationToken cancellationToken) + { + if (dates.Count == 0) + { + return []; + } + + var requestList = new List + { + new() { StationId = stationId, Date = dates } + }; + + _logger.LogDebug("Request string for schedules is: {@RequestString}", requestList); + + using var options = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/schedules"); + options.Content = JsonContent.Create(requestList, options: _jsonOptions); + options.Headers.TryAddWithoutValidation("token", token); + + // A station-wide failure such as SCHEDULE_QUEUED arrives as a bare object rather than + // the usual array, so the shape has to be inspected before it is deserialized. + var payload = await Request(options, true, info, cancellationToken, bodyCodeIsError: false).ConfigureAwait(false); + var dailySchedules = payload.ValueKind switch + { + JsonValueKind.Array => payload.Deserialize>(_jsonOptions), + JsonValueKind.Object => [payload.Deserialize(_jsonOptions)], + _ => null + }; + + if (dailySchedules is null) + { + return []; + } + + var errorDays = dailySchedules.Where(d => d.Code.HasValue).ToList(); + RecordQueuedSchedules(stationId, dates, errorDays); + LogEntryErrors(errorDays.Select(d => (d.Code!.Value, d.StationId ?? stationId, d.Message ?? d.Response)), "schedule"); + var days = dailySchedules.Where(d => !d.Code.HasValue).ToList(); + + _logger.LogDebug("Found {ScheduleCount} days on {ChannelID} ScheduleDirect", days.Count, stationId); - // programEntry.bannerImage = GetProgramImage(ApiUrl, data, "Banner", false) ?? - // GetProgramImage(ApiUrl, data, "Banner-L1", false) ?? - // GetProgramImage(ApiUrl, data, "Banner-LO", false) ?? - // GetProgramImage(ApiUrl, data, "Banner-LOT", false); + var programIds = days.SelectMany(d => d.Programs.Select(s => s.ProgramId)) + .Where(id => !string.IsNullOrEmpty(id)) + .Distinct() + .ToList(); + + var programDetails = await GetProgramDetails(info, programIds, token, cancellationToken).ConfigureAwait(false); + var detailsById = programDetails.ToDictionary(p => p.ProgramId, StringComparer.Ordinal); + + var result = new List(days.Count); + foreach (var day in days) + { + var dayPrograms = day.Programs + .Select(p => p.ProgramId) + .Where(id => !string.IsNullOrEmpty(id) && detailsById.ContainsKey(id)) + .Distinct() + .Select(id => detailsById[id]) + .ToList(); + + var cachedDay = new SchedulesDirectCachedDay + { + Md5 = day.Metadata?.Md5, + Schedule = day, + Programs = dayPrograms + }; + + var date = day.Metadata?.StartDate?.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + var missingIds = day.Programs + .Select(p => p.ProgramId) + .Where(id => !string.IsNullOrEmpty(id) && !detailsById.ContainsKey(id)) + .Distinct(StringComparer.Ordinal) + .ToList(); + + if (missingIds.Count == 0) + { + if (date is not null) + { + // Must stay ahead of SetProgramImages: it stamps ephemeral uris and a token + // that expires daily onto these programs, and neither may reach the cache. + await _scheduleCache.SetAsync(stationId, date, cachedDay, cancellationToken).ConfigureAwait(false); } + + result.Add(cachedDay); + continue; } - programsInfo.Add(GetProgram(channelId, schedule, programDict[schedule.ProgramId])); + // A queued program (6001) is not cached: finishing it need not change the day's md5. + // Dropping its airings would delete them, so the previous copy's details stand in; + // without those the day stays uncovered and GetProgramsAsync falls back or fails. + var stale = date is null + ? null + : await _scheduleCache.GetLatestAsync(stationId, date, cancellationToken).ConfigureAwait(false); + var staleById = stale?.Programs + .Where(p => !string.IsNullOrEmpty(p.ProgramId)) + .DistinctBy(p => p.ProgramId, StringComparer.Ordinal) + .ToDictionary(p => p.ProgramId, StringComparer.Ordinal); + if (staleById is not null && missingIds.TrueForAll(staleById.ContainsKey)) + { + cachedDay.Programs = [.. dayPrograms, .. missingIds.Select(id => staleById[id])]; + result.Add(cachedDay); + } } - return programsInfo; + return result; + } + + private void RecordQueuedSchedules(string stationId, IReadOnlyList dates, IReadOnlyList errorDays) + { + // Dates roll out of the guide window without being asked for again, so entries that + // are no longer holding anything back have to be swept here. + var now = DateTime.UtcNow; + foreach (var (key, value) in _scheduleRetryTimes) + { + if (value <= now) + { + _scheduleRetryTimes.TryRemove(key, out _); + } + } + + foreach (var day in errorDays) + { + if (day.Code != (int)SdErrorCode.ScheduleQueued || day.RetryTime is not { } retryTime) + { + continue; + } + + // The response is station-wide and names no date, so it covers everything asked for. + _logger.LogInformation( + "Schedules Direct has queued the schedule for station {StationId}; retrying after {RetryTime}", + day.StationId ?? stationId, + retryTime); + + foreach (var date in dates) + { + _scheduleRetryTimes[(day.StationId ?? stationId) + "|" + date] = retryTime.ToUniversalTime(); + } + } + } + + private async Task> GetProgramDetails( + ListingsProviderInfo info, + IReadOnlyList programIds, + string token, + CancellationToken cancellationToken) + { + var results = new List(programIds.Count); + + for (var i = 0; i < programIds.Count; i += MaxProgramsPerRequest) + { + var batch = programIds.Skip(i).Take(MaxProgramsPerRequest); + + using var message = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/programs"); + message.Headers.TryAddWithoutValidation("token", token); + message.Content = JsonContent.Create(batch, options: _jsonOptions); + + var batchResult = await Request>(message, true, info, cancellationToken).ConfigureAwait(false); + if (batchResult is null) + { + continue; + } + + LogEntryErrors(batchResult.Where(p => p.Code.HasValue).Select(p => (p.Code!.Value, p.ProgramId, p.Message)), "program"); + + // Error entries carry no program data. A queued program (6001) is a soft failure, + // so it is left out, and its day is not cached so the next refresh asks again. + results.AddRange(batchResult.Where(p => !p.Code.HasValue && !string.IsNullOrEmpty(p.ProgramId))); + } + + return results.DistinctBy(p => p.ProgramId, StringComparer.Ordinal).ToList(); + } + + private void SetProgramImages(string programId, ProgramDetailsDto programEntry, IReadOnlyList images, string token) + { + var match = images.FirstOrDefault(i => + i.ProgramId is not null && programId.StartsWith(i.ProgramId, StringComparison.Ordinal)); + if (match is null) + { + return; + } + + var allImages = match.Data; + var imagesWithText = allImages.Where(i => string.Equals(i.Text, "yes", StringComparison.OrdinalIgnoreCase)).ToList(); + var imagesWithoutText = allImages.Where(i => string.Equals(i.Text, "no", StringComparison.OrdinalIgnoreCase)).ToList(); + + const double DesiredAspect = 2.0 / 3; + const double WideAspect = 16.0 / 9; + + programEntry.PrimaryImage = GetProgramImage(ApiUrl, imagesWithText, DesiredAspect, token) ?? + GetProgramImage(ApiUrl, allImages, DesiredAspect, token); + + programEntry.ThumbImage = GetProgramImage(ApiUrl, imagesWithText, WideAspect, token); + + // Don't supply the same image twice + if (string.Equals(programEntry.PrimaryImage, programEntry.ThumbImage, StringComparison.Ordinal)) + { + programEntry.ThumbImage = null; + } + + programEntry.BackdropImage = GetProgramImage(ApiUrl, imagesWithoutText, WideAspect, token); + } + + private void LogEntryErrors(IEnumerable<(int Code, string Id, string Message)> errors, string kind) + { + foreach (var (code, id, message) in errors) + { + _logger.LogWarning( + "Schedules Direct returned an error for {Kind} {Id}: code={Code}, message={Message}", + kind, + id, + code, + message); + + var sdCode = ToKnownErrorCode(code); + if (sdCode.HasValue) + { + ApplyErrorCode(sdCode.Value); + } + } } private static int GetSizeOrder(ImageDataDto image) @@ -299,7 +703,6 @@ namespace Jellyfin.LiveTv.Listings CommunityRating = null, EpisodeTitle = episodeTitle, Audio = audioType, - // IsNew = programInfo.@new ?? false, IsRepeat = programInfo.New is null, IsSeries = string.Equals(details.EntityType, "episode", StringComparison.OrdinalIgnoreCase), ImageUrl = details.PrimaryImage, @@ -417,9 +820,12 @@ namespace Jellyfin.LiveTv.Listings return info; } - private static string GetProgramImage(string apiUrl, IEnumerable images, double desiredAspect, string token) + private string GetProgramImage(string apiUrl, IEnumerable images, double desiredAspect, string token) { + // A uri Schedules Direct has already rejected must never be offered again; repeating + // the request is what trips MAX_IMAGE_INVALID_URI_ERRORS and blocks the account. var match = images + .Where(i => !string.IsNullOrWhiteSpace(i.Uri) && !_rejectedImageUris.ContainsKey(GetImageKey(i.Uri))) .OrderBy(i => Math.Abs(desiredAspect - GetAspectRatio(i))) .ThenByDescending(i => GetSizeOrder(i)) .FirstOrDefault(); @@ -431,11 +837,6 @@ namespace Jellyfin.LiveTv.Listings var uri = match.Uri; - if (string.IsNullOrWhiteSpace(uri)) - { - return null; - } - if (uri.Contains("http", StringComparison.OrdinalIgnoreCase)) { return uri; @@ -444,6 +845,20 @@ namespace Jellyfin.LiveTv.Listings return apiUrl + "/image/" + uri + "?token=" + token; } + private static string GetImageKey(string uriOrUrl) + { + var value = uriOrUrl; + + var imageSegment = value.IndexOf("/image/", StringComparison.OrdinalIgnoreCase); + if (imageSegment >= 0) + { + value = value[(imageSegment + 7)..]; + } + + var query = value.IndexOf('?', StringComparison.Ordinal); + return query < 0 ? value : value[..query]; + } + private static double GetAspectRatio(ImageDataDto i) { int width = 0; @@ -554,6 +969,13 @@ namespace Jellyfin.LiveTv.Listings throw new AuthenticationException("Could not authenticate with Schedules Direct"); } + // The spec puts the status check ahead of all processing, and an empty lineup list is + // indistinguishable from "nothing for this postal code", so this has to surface. + if (!await IsSystemOnline(info, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline. Try again later."); + } + var lineups = new List(); using var options = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/headends?country=" + country + "&postalcode=" + location); @@ -586,7 +1008,9 @@ namespace Jellyfin.LiveTv.Listings private void ResetErrorState(ListingsProviderInfo info) { _accountError = false; - Interlocked.Exchange(ref _lastErrorResponseTicks, 0); + Interlocked.Exchange(ref _loginBackoffUntilTicks, 0); + Interlocked.Exchange(ref _offlineUntilTicks, 0); + _status = null; // Only the account being saved is retried, the tokens of the other accounts stay valid. if (!string.IsNullOrWhiteSpace(info.Username)) @@ -619,10 +1043,21 @@ namespace Jellyfin.LiveTv.Listings return null; } - // Avoid hammering SD after transient login failures (e.g. max attempts / temporary lockout) - if ((DateTime.UtcNow - new DateTime(Interlocked.Read(ref _lastErrorResponseTicks), DateTimeKind.Utc)).TotalMinutes < 30) + // Avoid hammering SD after login failures (temporary lockout, too many logins). + var backoffUntil = Interlocked.Read(ref _loginBackoffUntilTicks); + if (DateTime.UtcNow.Ticks < backoffUntil) + { + _logger.LogWarning( + "Skipping Schedules Direct request because of a recent login failure. Retrying no earlier than {RetryAt}.", + new DateTime(backoffUntil, DateTimeKind.Utc)); + + return null; + } + + // A login during the offline hour counts towards MAX_LOGIN_ATTEMPTS and cannot succeed. + if (IsOfflineBackoffActive()) { - _logger.LogWarning("Skipping Schedules Direct request because of a recent login failure. Retrying no earlier than 30 minutes after it."); + _logger.LogWarning("Skipping Schedules Direct request, the service reported itself offline"); return null; } @@ -663,7 +1098,7 @@ namespace Jellyfin.LiveTv.Listings && (int)ex.StatusCode.Value < 500) { _tokens.Clear(); - Interlocked.Exchange(ref _lastErrorResponseTicks, DateTime.UtcNow.Ticks); + BackOffLogins(TimeSpan.FromMinutes(LoginBackoffMinutes)); } throw; @@ -671,86 +1106,202 @@ namespace Jellyfin.LiveTv.Listings } } + private async Task IsSystemOnline(ListingsProviderInfo info, CancellationToken cancellationToken) + { + if (IsOfflineBackoffActive()) + { + _logger.LogWarning("Skipping Schedules Direct request, the service reported itself offline"); + return false; + } + + var status = await GetStatus(info, cancellationToken).ConfigureAwait(false); + if (status is null) + { + // A failed status check is not a reason to skip the refresh on its own, unless it + // failed with SERVICE_OFFLINE. + return !IsOfflineBackoffActive(); + } + + // The spec does not define the order of the entries, so any of them can be the one + // that takes the service down. + var offline = status.SystemStatus.FirstOrDefault( + i => string.Equals(i.Status, "Offline", StringComparison.OrdinalIgnoreCase)); + if (offline is not null) + { + _logger.LogWarning( + "Schedules Direct reports the service is offline ({Message}). Not retrying for {Minutes} minutes.", + offline.Message, + OfflineBackoffMinutes); + Interlocked.Exchange(ref _offlineUntilTicks, DateTime.UtcNow.AddMinutes(OfflineBackoffMinutes).Ticks); + return false; + } + + if (status.Account?.Expires is { } expires && expires < DateTime.UtcNow.AddDays(7)) + { + _logger.LogWarning("The Schedules Direct subscription expires on {Expires}", expires); + } + + return true; + } + + private async Task GetStatus(ListingsProviderInfo info, CancellationToken cancellationToken) + { + if (TryGetCachedStatus(out var cached)) + { + return cached; + } + + using (await _statusLock.LockAsync(cancellationToken).ConfigureAwait(false)) + { + // Whoever held the lock may have just refreshed it, and a guide refresh asking + // once per channel must not turn into one request per channel. + if (TryGetCachedStatus(out cached)) + { + return cached; + } + + StatusDto status = null; + try + { + var token = await GetToken(info, cancellationToken).ConfigureAwait(false); + + using var message = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/status"); + message.Headers.TryAddWithoutValidation("token", token); + + status = await Request(message, true, info, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or AuthenticationException) + { + _logger.LogWarning(ex, "Unable to get the Schedules Direct service status"); + } + + // A failed check still counts as checked, or every channel retries it. + _status = new StatusCacheEntry(DateTime.UtcNow, status); + return status; + } + } + + private bool TryGetCachedStatus(out StatusDto status) + { + // One reference, so the time of the check and what it found can never disagree. + var entry = _status; + status = entry?.Status; + return entry is not null && (DateTime.UtcNow - entry.CheckedAt).TotalMinutes < StatusCacheMinutes; + } + + private bool ApplyErrorCode(SdErrorCode sdCode) + { + switch (sdCode) + { + case SdErrorCode.AccountExpired: + case SdErrorCode.InvalidHash: + case SdErrorCode.InvalidUser: + case SdErrorCode.AccountLocked: + case SdErrorCode.AppLocked: + case SdErrorCode.AccountInactive: + // Permanent account errors — disable SD for this server lifetime. + _logger.LogError("Schedules Direct account error (code {SdCode}). Disabling SD until server restart.", sdCode); + _tokens.Clear(); + _accountError = true; + return true; + + case SdErrorCode.ServiceOffline: + // The spec requires staying away for an hour once the service reports offline. + _logger.LogError("Schedules Direct is offline (code {SdCode}). Not retrying for {Minutes} minutes.", sdCode, OfflineBackoffMinutes); + _tokens.Clear(); + Interlocked.Exchange(ref _offlineUntilTicks, DateTime.UtcNow.AddMinutes(OfflineBackoffMinutes).Ticks); + return true; + + case SdErrorCode.ServiceBusy: + case SdErrorCode.AccountTempLock: + // Transient login errors — back off for 30 minutes, then allow retry. + _logger.LogError("Schedules Direct transient error (code {SdCode}). Backing off for {Minutes} minutes.", sdCode, LoginBackoffMinutes); + _tokens.Clear(); + BackOffLogins(TimeSpan.FromMinutes(LoginBackoffMinutes)); + return true; + + case SdErrorCode.MaxLoginAttempts: + // Counts logins, so asking for a token again is what earned it. SD support confirmed it + // clears 24 hours after the first login; waiting 24 hours from now is always past that. + _logger.LogError( + "Schedules Direct login limit reached (code {SdCode}). Not logging in again for {Hours} hours.", + sdCode, + MaxLoginsBackoffHours); + _tokens.Clear(); + BackOffLogins(TimeSpan.FromHours(MaxLoginsBackoffHours)); + SetImageLimitHit(); + SetMetadataLimitHit(); + return true; + + case SdErrorCode.MaxIPAttempts: + // Unconfirmed whether this one self-heals, so it stays disabled like an account error. + _logger.LogError( + "Schedules Direct unique IP limit reached (code {SdCode}). Disabling SD until server restart; the user has to contact Schedules Direct support.", + sdCode); + _tokens.Clear(); + _accountError = true; + SetImageLimitHit(); + SetMetadataLimitHit(); + return true; + + case SdErrorCode.MaxImageDownloads: + case SdErrorCode.MaxImageDownloadsTrial: + case SdErrorCode.MaxInvalidImages: + // Stop image requests until SD resets at 00:00 UTC. + _logger.LogError("Schedules Direct image download limit hit (code {SdCode}). Disabling image acquisition until SD reset.", sdCode); + SetImageLimitHit(); + return true; + + case SdErrorCode.MaxLineupChanges: + // Only blocks further lineup edits; guide data is unaffected. + _logger.LogError("Schedules Direct lineup change limit reached (code {SdCode}). Lineup edits are rejected until SD reset.", sdCode); + return true; + + default: + return false; + } + } + private async Task Request( HttpRequestMessage message, bool enableRetry, ListingsProviderInfo providerInfo, CancellationToken cancellationToken, - HttpCompletionOption completionOption = HttpCompletionOption.ResponseContentRead) + bool bodyCodeIsError = true) { using var response = await _httpClientFactory.CreateClient(NamedClient.Default) - .SendAsync(message, completionOption, cancellationToken) + .SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken) .ConfigureAwait(false); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(_jsonOptions, cancellationToken).ConfigureAwait(false); - } - var responseBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - // Try to extract the Schedules Direct error code from the response body. - SdErrorCode? sdCode = null; - try + // SD can report a failure as HTTP 200 with a non-zero code, which would otherwise + // deserialize into an empty object and read as success. + var errorCode = TryGetErrorCode(responseBody); + if (response.IsSuccessStatusCode && (!bodyCodeIsError || errorCode is null or 0)) { - using var doc = JsonDocument.Parse(responseBody); - if (doc.RootElement.TryGetProperty("code", out var codeProp) - && codeProp.TryGetInt32(out var parsedCode) - && Enum.IsDefined((SdErrorCode)parsedCode)) - { - sdCode = (SdErrorCode)parsedCode; - } - } - catch (JsonException) - { - // Response body is not valid JSON; sdCode stays null. + return JsonSerializer.Deserialize(responseBody, _jsonOptions); } + var sdCode = ToKnownErrorCode(errorCode); + _logger.LogError( "Request to {Url} failed with HTTP {StatusCode}, SD code {SdCode}: {Response}", message.RequestUri, (int)response.StatusCode, - sdCode?.ToString() ?? "N/A", + sdCode?.ToString() ?? errorCode?.ToString(CultureInfo.InvariantCulture) ?? "N/A", responseBody); - if (sdCode is SdErrorCode.AccountExpired or SdErrorCode.InvalidHash or SdErrorCode.InvalidUser or SdErrorCode.AccountLocked or SdErrorCode.AppLocked or SdErrorCode.AccountInactive) - { - // Permanent account errors — disable SD for this server lifetime. - _logger.LogError("Schedules Direct account error (code {SdCode}). Disabling SD until server restart.", sdCode); - _tokens.Clear(); - _accountError = true; - } - else if (sdCode is SdErrorCode.ServiceOffline or SdErrorCode.ServiceBusy or SdErrorCode.AccountTempLock) - { - // Transient login errors — back off for 30 minutes, then allow retry. - _logger.LogError("Schedules Direct transient error (code {SdCode}). Backing off for 30 minutes.", sdCode); - _tokens.Clear(); - Interlocked.Exchange(ref _lastErrorResponseTicks, DateTime.UtcNow.Ticks); - } - else if (sdCode is SdErrorCode.MaxLoginAttempts or SdErrorCode.MaxIPAttempts) - { - // 24 hour bans - stop image and metadata requests until SD reset at 00:00 UTC. - _logger.LogError("Schedules Direct service limit error (code {SdCode}). Disabling until SD reset.", sdCode); - SetImageLimitHit(); - SetMetadataLimitHit(); - } - else if (sdCode is SdErrorCode.MaxImageDownloads or SdErrorCode.MaxImageDownloadsTrial) + if (sdCode.HasValue && ApplyErrorCode(sdCode.Value)) { - // Max image downloads — stop image requests until SD resets at 00:00 UTC. - _logger.LogError("Schedules Direct image download limit hit (code {SdCode}). Disabling image acquisition until SD reset.", sdCode); - SetImageLimitHit(); - } - else if (sdCode is SdErrorCode.MaxScheduleRequests) - { - // Max schedule/metadata requests — stop metadata requests until SD resets at 00:00 UTC. - _logger.LogError("Schedules Direct metadata download limit hit (code {SdCode}). Disabling metadata acquisition until SD reset.", sdCode); - SetMetadataLimitHit(); + // Handled by the error code. } else if (enableRetry && (int)response.StatusCode < 500 - && (sdCode == SdErrorCode.TokenExpired || (response.StatusCode == HttpStatusCode.Forbidden && sdCode is null))) + && (sdCode is SdErrorCode.TokenExpired or SdErrorCode.TokenInvalid or SdErrorCode.TokenMissing + || (response.StatusCode == HttpStatusCode.Forbidden && errorCode is null))) { - // Token expired — clear tokens and retry with a fresh token. - // Also retry on 403 with no parseable SD code (legacy/unexpected auth failure). + // Token rejected — clear tokens and retry with a fresh token. Also retry on 403 + // carrying no code at all; a code we simply do not know is not an auth failure. _tokens.Clear(); using var retryMessage = new HttpRequestMessage(message.Method, message.RequestUri); retryMessage.Content = message.Content; @@ -807,9 +1358,26 @@ namespace Jellyfin.LiveTv.Listings if (!response.IsSuccessStatusCode) { + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var errorCode = TryGetErrorCode(responseBody); + var sdCode = ToKnownErrorCode(errorCode); + if (sdCode.HasValue) + { + ApplyErrorCode(sdCode.Value); + } + _logger.LogError( - "Error adding lineup to account: {Response}", - await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + "Error adding lineup {Id} to account, SD code {SdCode}: {Response}", + info.ListingsId, + sdCode?.ToString() ?? errorCode?.ToString(CultureInfo.InvariantCulture) ?? "N/A", + responseBody); + + // Reporting success here leaves the user with a listings provider that silently + // has no lineup, so the failure has to surface. + throw new HttpRequestException( + string.Format(CultureInfo.InvariantCulture, "Could not add lineup {0} to the Schedules Direct account", info.ListingsId), + null, + response.StatusCode); } } @@ -913,23 +1481,225 @@ namespace Jellyfin.LiveTv.Listings } /// - public bool IsServiceAvailable() + public bool CanDownloadImage(string imageUrl) + => !IsSchedulesDirectUrl(imageUrl) + || (!IsImageDailyLimitActive() && !_rejectedImageUris.ContainsKey(GetImageKey(imageUrl))); + + /// + public void ReportImageDownloadSuccess(string imageUrl) { - if (_accountError) + if (IsSchedulesDirectUrl(imageUrl)) { - return false; + Interlocked.Exchange(ref _consecutiveImageFailures, 0); } + } - if ((DateTime.UtcNow - new DateTime(Interlocked.Read(ref _lastErrorResponseTicks), DateTimeKind.Utc)).TotalMinutes < 30) + /// + public ImageDownloadFailureAction ReportImageDownloadFailure(string imageUrl, HttpStatusCode? statusCode, string responseBody) + { + if (!IsSchedulesDirectUrl(imageUrl)) { - return false; + return ImageDownloadFailureAction.None; } - return true; + var errorCode = TryGetErrorCode(responseBody); + if (errorCode is null) + { + // No readable code, so stop after a short streak of unexplained rejections rather + // than keep retrying an account that may already be refusing us. + if (Interlocked.Increment(ref _consecutiveImageFailures) == MaxConsecutiveImageFailures) + { + _logger.LogError( + "Schedules Direct image downloads keep failing without a readable error code (last status {StatusCode}). Disabling image acquisition until SD reset.", + statusCode); + SetImageLimitHit(); + } + + return ImageDownloadFailureAction.None; + } + + Interlocked.Exchange(ref _consecutiveImageFailures, 0); + + var sdCode = ToKnownErrorCode(errorCode); + if (sdCode is null) + { + return ImageDownloadFailureAction.None; + } + + // SD reports an exhausted image quota with an HTTP 200 body, so the code decides. + ApplyErrorCode(sdCode.Value); + + switch (sdCode.Value) + { + // The uri is gone. Remembering it is what keeps the next refresh from asking for + // it again and walking the account into MAX_IMAGE_INVALID_URI_ERRORS. + case SdErrorCode.ImageNotFound: + RejectImageUri(imageUrl); + return ImageDownloadFailureAction.RemoveImage; + + // The account is already being warned about invalid uris; this url is not + // necessarily one of them, so drop it for now but do not blacklist it. + case SdErrorCode.MaxInvalidImages: + return ImageDownloadFailureAction.RemoveImage; + + // The token baked into the url is stale; the next refresh mints a fresh one. + case SdErrorCode.TokenMissing: + case SdErrorCode.TokenInvalid: + case SdErrorCode.TokenExpired: + _tokens.Clear(); + return ImageDownloadFailureAction.RemoveImage; + + default: + return ImageDownloadFailureAction.None; + } } /// - public bool IsImageDailyLimitActive() + public async Task ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex) + { + ArgumentNullException.ThrowIfNull(image); + + var remotePath = image.Path; + if (!CanDownloadImage(remotePath)) + { + return null; + } + + try + { + var localImage = await _libraryManager.ConvertImageToLocal(item, image, imageIndex).ConfigureAwait(false); + ReportImageDownloadSuccess(remotePath); + return localImage; + } + catch (Exception ex) when ((ex as HttpRequestException ?? ex.InnerException as HttpRequestException) is { } httpException) + { + ReportImageDownloadFailure(remotePath, httpException.StatusCode, (httpException as HttpResponseBodyException)?.ResponseBody); + throw; + } + } + + private void RejectImageUri(string imageUrl) + { + var key = GetImageKey(imageUrl); + if (string.IsNullOrEmpty(key)) + { + return; + } + + // Stop learning rather than forgetting: clearing here would re-offer every known-bad + // uri at once, which is the burst that trips MAX_IMAGE_INVALID_URI_ERRORS. + if (_rejectedImageUris.Count < MaxRejectedImageUris && _rejectedImageUris.TryAdd(key, 0)) + { + // Persisted, or a restart would offer every rejected uri again in one refresh. + var line = DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) + "\t" + key + Environment.NewLine; + try + { + lock (_rejectedImageUrisFileLock) + { + Directory.CreateDirectory(Path.GetDirectoryName(RejectedImageUrisFilePath)!); + File.AppendAllText(RejectedImageUrisFilePath, line); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to persist rejected SD image uri to {Path}", RejectedImageUrisFilePath); + } + } + } + + private void LoadRejectedImageUris() + { + var path = RejectedImageUrisFilePath; + if (!File.Exists(path)) + { + return; + } + + try + { + var cutoff = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-RejectedImageUriMaxAgeDays)); + var kept = new List(); + var lines = File.ReadAllLines(path); + foreach (var line in lines) + { + var separator = line.IndexOf('\t', StringComparison.Ordinal); + if (separator <= 0 + || !DateOnly.TryParseExact(line.AsSpan(0, separator), "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date) + || date < cutoff + || _rejectedImageUris.Count >= MaxRejectedImageUris) + { + continue; + } + + if (_rejectedImageUris.TryAdd(line[(separator + 1)..], 0)) + { + kept.Add(line); + } + } + + if (kept.Count != lines.Length) + { + File.WriteAllLines(path, kept); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to load rejected SD image uris from {Path}", path); + } + } + + private static bool IsSchedulesDirectUrl(string url) + => url is not null && url.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase); + + private static int? TryGetErrorCode(string responseBody) + { + // Only an object can carry a code; this also spares parsing large program arrays twice. + if (string.IsNullOrWhiteSpace(responseBody) || !responseBody.AsSpan().TrimStart().StartsWith('{')) + { + return null; + } + + try + { + using var doc = JsonDocument.Parse(responseBody); + if (doc.RootElement.ValueKind == JsonValueKind.Object + && doc.RootElement.TryGetProperty("code", out var codeProp) + && codeProp.TryGetInt32(out var parsedCode)) + { + return parsedCode; + } + } + catch (JsonException) + { + // Not an SD error payload. + } + + return null; + } + + private bool IsOfflineBackoffActive() + => DateTime.UtcNow.Ticks < Interlocked.Read(ref _offlineUntilTicks); + + private void BackOffLogins(TimeSpan duration) + { + // Keep the later deadline, so a generic 30 minute back-off cannot shorten the 24 hours of MAX_LOGIN_ATTEMPTS. + var until = DateTime.UtcNow.Add(duration).Ticks; + long current; + do + { + current = Interlocked.Read(ref _loginBackoffUntilTicks); + if (current >= until) + { + return; + } + } + while (Interlocked.CompareExchange(ref _loginBackoffUntilTicks, until, current) != current); + } + + private static SdErrorCode? ToKnownErrorCode(int? code) + => code.HasValue && Enum.IsDefined((SdErrorCode)code.Value) ? (SdErrorCode)code.Value : null; + + private bool IsImageDailyLimitActive() { if (!_imageLimitHitDate.HasValue) { @@ -939,6 +1709,7 @@ namespace Jellyfin.LiveTv.Listings if (_imageLimitHitDate.Value < DateOnly.FromDateTime(DateTime.UtcNow)) { _imageLimitHitDate = null; + Interlocked.Exchange(ref _consecutiveImageFailures, 0); TryDeleteFile(ImageLimitFilePath); return false; } @@ -1016,6 +1787,11 @@ namespace Jellyfin.LiveTv.Listings } } + if (!await IsSystemOnline(info, CancellationToken.None).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline. Try again later."); + } + if (validateListings) { ArgumentException.ThrowIfNullOrEmpty(info.ListingsId); @@ -1042,20 +1818,27 @@ namespace Jellyfin.LiveTv.Listings return []; } + // An empty channel list maps no channel, which the guide refresh reads as "no programs" + // and deletes the stored ones, so a failure has to surface here as well. var token = await GetToken(info, cancellationToken).ConfigureAwait(false); if (string.IsNullOrEmpty(token)) { - return []; + throw new AuthenticationException("Could not authenticate with Schedules Direct"); } - using var options = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/lineups/" + listingsId); - options.Headers.TryAddWithoutValidation("token", token); + if (!await IsSystemOnline(info, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline."); + } - var root = await Request(options, true, info, cancellationToken).ConfigureAwait(false); + // Stations that have left every lineup would otherwise keep their schedules forever. + _scheduleCache.PruneStale(TimeSpan.FromDays(ScheduleCacheMaxAgeDays)); + + var root = await GetLineup(info, listingsId, cancellationToken).ConfigureAwait(false); if (root is null) { - return new List(); + return []; } _logger.LogInformation("Found {ChannelCount} channels on the lineup on ScheduleDirect", root.Map.Count); @@ -1093,6 +1876,44 @@ namespace Jellyfin.LiveTv.Listings return list; } + private async Task GetLineup(ListingsProviderInfo info, string listingsId, CancellationToken cancellationToken) + { + var status = await GetStatus(info, cancellationToken).ConfigureAwait(false); + var lineup = status?.Lineups.FirstOrDefault( + i => string.Equals(i.Lineup ?? i.Id, listingsId, StringComparison.OrdinalIgnoreCase)); + + if (lineup?.IsDeleted == true) + { + _logger.LogWarning( + "The Schedules Direct lineup {ListingsId} has been deleted at the headend. Pick a new lineup for this listings provider.", + listingsId); + } + + var modified = lineup?.Modified; + if (modified.HasValue) + { + var cached = await _lineupCache.GetAsync(listingsId, modified.Value, cancellationToken).ConfigureAwait(false); + if (cached is not null) + { + _logger.LogDebug("Serving the Schedules Direct lineup {ListingsId} from cache", listingsId); + return cached; + } + } + + var token = await GetToken(info, cancellationToken).ConfigureAwait(false); + + using var options = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/lineups/" + listingsId); + options.Headers.TryAddWithoutValidation("token", token); + + var root = await Request(options, true, info, cancellationToken).ConfigureAwait(false); + if (root is not null && modified.HasValue) + { + await _lineupCache.SetAsync(listingsId, modified.Value, root, cancellationToken).ConfigureAwait(false); + } + + return root; + } + /// public void Dispose() { @@ -1114,9 +1935,12 @@ namespace Jellyfin.LiveTv.Listings if (disposing) { _tokenLock?.Dispose(); + _statusLock?.Dispose(); } _disposed = true; } + + private sealed record StatusCacheEntry(DateTime CheckedAt, StatusDto Status); } } diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectCachedDay.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectCachedDay.cs new file mode 100644 index 0000000000..c5dae1682e --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectCachedDay.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +namespace Jellyfin.LiveTv.Listings; + +/// +/// One station's schedule for one day, together with the details of its programs. +/// +/// +/// Artwork is deliberately not cached: the spec states the image uri is ephemeral and must not be +/// stored, and artwork changes are not reflected in the schedule md5 this entry is keyed by. +/// +public class SchedulesDirectCachedDay +{ + /// + /// Gets or sets the md5 Schedules Direct reported for this day's schedule. + /// + public string? Md5 { get; set; } + + /// + /// Gets or sets the schedule. + /// + public DayDto? Schedule { get; set; } + + /// + /// Gets or sets the details of the programs in the schedule. + /// + public IReadOnlyList Programs { get; set; } = []; +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs index 094f9a3194..4959e12675 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs @@ -21,6 +21,35 @@ namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos [JsonPropertyName("programs")] public IReadOnlyList Programs { get; set; } = Array.Empty(); + /// + /// Gets or sets the error code. Set when this entry is a per-station error + /// (for example SCHEDULE_RANGE_EXCEEDED) instead of a schedule. + /// + [JsonPropertyName("code")] + public int? Code { get; set; } + + /// + /// Gets or sets the error message that goes with . + /// + /// + /// Not every error entry carries one; STATIONID_DELETED only sets . + /// + [JsonPropertyName("message")] + public string? Message { get; set; } + + /// + /// Gets or sets the symbolic name of , for example STATIONID_DELETED. + /// + [JsonPropertyName("response")] + public string? Response { get; set; } + + /// + /// Gets or sets the earliest time a queued schedule (SCHEDULE_QUEUED) may be requested + /// again. The spec requires the client to wait at least this long. + /// + [JsonPropertyName("retryTime")] + public DateTime? RetryTime { get; set; } + /// /// Gets or sets the metadata schedule. /// diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs index 7bfc4bc8be..0b883e44aa 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs @@ -21,6 +21,19 @@ namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos [JsonPropertyName("programID")] public string? ProgramId { get; set; } + /// + /// Gets or sets the error code. Set when this entry is a per-program error + /// (for example INVALID_PROGRAMID) instead of a program. + /// + [JsonPropertyName("code")] + public int? Code { get; set; } + + /// + /// Gets or sets the error message that goes with . + /// + [JsonPropertyName("message")] + public string? Message { get; set; } + /// /// Gets or sets the list of titles. /// diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs new file mode 100644 index 0000000000..6acd2172a9 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs @@ -0,0 +1,34 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// +/// The md5 and last modified date of one station's schedule for one day. +/// +public class ScheduleMd5Dto +{ + /// + /// Gets or sets the error code. Non-zero when the day is unavailable. + /// + [JsonPropertyName("code")] + public int Code { get; set; } + + /// + /// Gets or sets the message that goes with . + /// + [JsonPropertyName("message")] + public string? Message { get; set; } + + /// + /// Gets or sets the time the schedule was last modified. + /// + [JsonPropertyName("lastModified")] + public DateTime? LastModified { get; set; } + + /// + /// Gets or sets the md5 of the schedule. A change means the schedule must be downloaded again. + /// + [JsonPropertyName("md5")] + public string? Md5 { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs index fffbfb9a58..b44ddc2bb4 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs @@ -1,19 +1,161 @@ -#pragma warning disable CA1008 // Enums should have zero value - namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; /// -/// Schedules Direct API error codes. See https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#error-response for details. +/// Schedules Direct API error codes. See https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#error-codes for details. /// +/// +/// A complete mirror of the documented table. Codes the client does not act on are still listed, +/// because membership is what lets a response be logged by name instead of by number. +/// public enum SdErrorCode { /// - /// Schedules Direct unavailable/out of service. + /// No error. + /// + Ok = 0, + + /// + /// Unable to decode JSON. + /// + InvalidJson = 1001, + + /// + /// Did not receive a User-Agent in the request header. + /// + UserAgentRequired = 1003, + + /// + /// Token required but not provided in the request header. + /// + TokenMissing = 1004, + + /// + /// Did not recognize the submitted client. + /// + UnknownClient = 1005, + + /// + /// The maximum number of elements in a single request is 5000. + /// + MaxChunkExceeded = 1006, + + /// + /// The request sent to the server was empty. + /// + EmptyRequest = 1007, + + /// + /// The request is improperly formatted. + /// + IncorrectRequest = 1008, + + /// + /// The maximum number of elements in a single request is 500. + /// + MaxChunkExceededMetadata = 1009, + + /// + /// Token is not 32 characters. + /// + TokenInvalid = 1010, + + /// + /// The Content-Type header must be application/json. + /// + IncorrectContentType = 1011, + + /// + /// Did not receive request. + /// + RequiredRequestMissing = 2002, + + /// + /// Searching for lineups requires a 3-letter country parameter. + /// + CountryRequired = 2004, + + /// + /// Searching for lineups requires a postal code parameter. + /// + PostalCodeRequired = 2005, + + /// + /// The request is missing the personID / nameID in the path. + /// + PersonIdRequired = 2020, + + /// + /// The country parameter must be ISO-3166-1 alpha 3. + /// + InvalidCountryParameter = 2050, + + /// + /// Unknown request; available data may be requested for COUNTRIES. + /// + UnknownRequest = 2054, + + /// + /// Unexpected debug connection from client. + /// + InvalidDebugParameter = 2055, + + /// + /// Lineup already in account. + /// + DuplicateLineup = 2100, + + /// + /// Lineup not in account; add it before requesting its mapping. + /// + LineupNotFound = 2101, + + /// + /// Invalid lineup requested; check the country/postal code combination. + /// + UnknownLineup = 2102, + + /// + /// Delete of a lineup that is not in the account. + /// + InvalidLineupDelete = 2103, + + /// + /// Lineup must be formatted COUNTRY-LINEUP-DEVICE or COUNTRY-OTA-POSTALCODE. + /// + LineupWrongFormat = 2104, + + /// + /// The requested lineup has been deleted from the server. + /// + LineupDeleted = 2106, + + /// + /// The requested country is mis-typed or has no valid data. + /// + InvalidCountry = 2108, + + /// + /// The requested personID does not exist. + /// + InvalidPersonId = 2109, + + /// + /// The requested stationID is not in any of the configured lineups. + /// + StationIdNotFound = 2200, + + /// + /// The requested stationID has been marked as deleted and will be purged. + /// + StationIdDeleted = 2201, + + /// + /// Server offline for maintenance. /// ServiceOffline = 3000, /// - /// Schedules Direct busy. + /// Server is busy processing other requests. Retry. /// ServiceBusy = 3001, @@ -23,72 +165,125 @@ public enum SdErrorCode AccountExpired = 4001, /// - /// Invalid password hash. + /// Password hash must be a lowercase 40 character sha1_hex of the password. /// InvalidHash = 4002, /// - /// Invalid user or password. + /// Invalid username or password. /// InvalidUser = 4003, /// - /// Account temporarily locked due to login failures. + /// Too many login failures. Locked for 15 minutes. /// AccountTempLock = 4004, /// - /// Account permanently locked due to abuse. + /// Access to the account via JSON has been disabled; the user must contact SD support. /// AccountLocked = 4005, /// - /// Token has expired. Request a new one. + /// Token has expired. Request a new token. /// TokenExpired = 4006, /// - /// Application locked out. + /// Application not authorized to use the data service. /// AppLocked = 4007, /// - /// Account not active. + /// Account inactive. /// AccountInactive = 4008, /// - /// Maximum login attempts exceeded. + /// Exceeded the maximum number of logins in 24 hours; clears 24 hours after the first login. /// MaxLoginAttempts = 4009, /// - /// Maximum unique IP attempts reached. + /// Exceeded the maximum number of unique IP addresses in 24 hours; the user must contact SD support. /// MaxIPAttempts = 4010, /// - /// Lineup change maximum reached. + /// Exceeded the maximum number of lineup changes for today. /// - MaxScheduleRequests = 4100, + MaxLineupChanges = 4100, /// - /// Requested image not found. + /// Exceeded the number of lineups this account may hold. + /// + MaxLineups = 4101, + + /// + /// No lineups have been added to this account. + /// + NoLineups = 4102, + + /// + /// Could not find the requested image. /// ImageNotFound = 5000, /// - /// Maximum image downloads reached for the day. + /// Maximum image downloads reached. The counter resets at 00:00Z. /// MaxImageDownloads = 5002, /// - /// Trial specific maximum image downloads reached for the day. + /// Maximum image downloads for a trial user reached. The counter resets at 00:00Z. /// MaxImageDownloadsTrial = 5003, /// - /// Maximum schedule/metadata requests reached for the day. + /// Exceeded the maximum number of invalid image URIs in 24 hours. The application is + /// requesting URIs which do not exist and the account is about to be blocked. + /// + MaxInvalidImages = 5004, + + /// + /// The requested programID does not exist. Permanent failure, do not ask again. + /// + InvalidProgramId = 6000, + + /// + /// The requested programID is queued for generation at the server. Soft failure, retry later. + /// + ProgramQueued = 6001, + + /// + /// The requested schedule should be available but was not found. + /// + ScheduleNotFound = 7000, + + /// + /// The server cannot determine whether the requested schedule is valid; the user has to open + /// a support ticket. + /// + InvalidScheduleRequest = 7010, + + /// + /// The requested date is outside the range the server holds for that station. + /// + ScheduleRangeExceeded = 7020, + + /// + /// The requested schedule is not in any of the configured lineups. + /// + ScheduleNotInLineup = 7030, + + /// + /// The schedule is queued for generation and is not ready yet. The client must wait until the + /// retryTime carried by the response before asking for it again. + /// + ScheduleQueued = 7100, + + /// + /// Unknown error; the user has to open a support ticket. /// - MaxInvalidImages = 5004 + UnknownError = 9999 } diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusAccountDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusAccountDto.cs new file mode 100644 index 0000000000..fca28e0511 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusAccountDto.cs @@ -0,0 +1,22 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// +/// The account block of the /status response. +/// +public class StatusAccountDto +{ + /// + /// Gets or sets the time the subscription expires. + /// + [JsonPropertyName("expires")] + public DateTime? Expires { get; set; } + + /// + /// Gets or sets the maximum number of lineups the account may hold. + /// + [JsonPropertyName("maxLineups")] + public int? MaxLineups { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusDto.cs new file mode 100644 index 0000000000..ce6b2ccc22 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusDto.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// +/// The /status response. +/// +public class StatusDto +{ + /// + /// Gets or sets the account details. + /// + [JsonPropertyName("account")] + public StatusAccountDto? Account { get; set; } + + /// + /// Gets or sets the lineups on the account, with the time each was last modified. + /// + [JsonPropertyName("lineups")] + public IReadOnlyList Lineups { get; set; } = []; + + /// + /// Gets or sets the system status entries. The spec does not define their order, so every + /// entry has to be inspected. + /// + [JsonPropertyName("systemStatus")] + public IReadOnlyList SystemStatus { get; set; } = []; + + /// + /// Gets or sets the response code. + /// + [JsonPropertyName("code")] + public int Code { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusLineupDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusLineupDto.cs new file mode 100644 index 0000000000..9a3559e406 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusLineupDto.cs @@ -0,0 +1,35 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// +/// One lineup entry of the /status response. +/// +public class StatusLineupDto +{ + /// + /// Gets or sets the lineup id. + /// + [JsonPropertyName("lineup")] + public string? Lineup { get; set; } + + /// + /// Gets or sets the lineup id as reported for deleted lineups, which use a different name. + /// + [JsonPropertyName("ID")] + public string? Id { get; set; } + + /// + /// Gets or sets the time the lineup was last modified. A client only has to download the + /// lineup again when this is newer than the copy it holds. + /// + [JsonPropertyName("modified")] + public DateTime? Modified { get; set; } + + /// + /// Gets or sets a value indicating whether the lineup has been deleted at the headend. + /// + [JsonPropertyName("isDeleted")] + public bool IsDeleted { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SystemStatusDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SystemStatusDto.cs new file mode 100644 index 0000000000..00909218f3 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SystemStatusDto.cs @@ -0,0 +1,28 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// +/// One entry of the system status. +/// +public class SystemStatusDto +{ + /// + /// Gets or sets the time the status was posted. + /// + [JsonPropertyName("date")] + public DateTime? Date { get; set; } + + /// + /// Gets or sets the status. "Offline" means the client must disconnect for an hour. + /// + [JsonPropertyName("status")] + public string? Status { get; set; } + + /// + /// Gets or sets the message describing the status. + /// + [JsonPropertyName("message")] + public string? Message { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectLineupCache.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectLineupCache.cs new file mode 100644 index 0000000000..a24961f0aa --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectLineupCache.cs @@ -0,0 +1,117 @@ +using System; +using System.Globalization; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Extensions.Json; +using Jellyfin.LiveTv.Listings.SchedulesDirectDtos; +using MediaBrowser.Common.Configuration; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.LiveTv.Listings; + +/// +/// Caches a lineup's channel map, keyed by the "modified" date Schedules Direct reports for it. +/// +/// +/// The spec's steady-state flow is to compare the lineup's modified date in /status against the +/// copy the client holds and only download the lineup again when the server's is newer. +/// +public class SchedulesDirectLineupCache +{ + private readonly ILogger _logger; + private readonly string _cacheRoot; + private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + /// The application paths. + public SchedulesDirectLineupCache(ILogger logger, IApplicationPaths appPaths) + { + _logger = logger; + _cacheRoot = Path.Combine(appPaths.CachePath, "schedulesdirect", "lineups"); + } + + /// + /// Reads the cached lineup if it was stored under the same modified date. + /// + /// The lineup id. + /// The modified date Schedules Direct currently reports. + /// The cancellation token. + /// The cached lineup, or null when it is missing or stale. + public async Task GetAsync(string lineupId, DateTime modified, CancellationToken cancellationToken) + { + var path = GetPath(lineupId, modified); + if (!File.Exists(path)) + { + return null; + } + + try + { + var stream = File.OpenRead(path); + await using (stream.ConfigureAwait(false)) + { + return await JsonSerializer.DeserializeAsync(stream, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Discarding unreadable Schedules Direct lineup cache entry {Path}", path); + return null; + } + } + + /// + /// Stores a freshly downloaded lineup and drops the copies it supersedes. + /// + /// The lineup id. + /// The modified date Schedules Direct reports for it. + /// The lineup to store. + /// The cancellation token. + /// A task representing the write. + public async Task SetAsync(string lineupId, DateTime modified, ChannelDto lineup, CancellationToken cancellationToken) + { + var path = GetPath(lineupId, modified); + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var stream = File.Create(path); + await using (stream.ConfigureAwait(false)) + { + await JsonSerializer.SerializeAsync(stream, lineup, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + + foreach (var stale in Directory.EnumerateFiles(Path.GetDirectoryName(path)!, "*.json")) + { + if (!string.Equals(stale, path, StringComparison.Ordinal)) + { + File.Delete(stale); + } + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to cache Schedules Direct lineup {Path}", path); + } + } + + private static string GetSafeName(string value) + => string.Create(value.Length, value, (span, source) => + { + for (var i = 0; i < source.Length; i++) + { + span[i] = char.IsAsciiLetterOrDigit(source[i]) ? source[i] : '_'; + } + }); + + private string GetPath(string lineupId, DateTime modified) + => Path.Combine( + _cacheRoot, + GetSafeName(lineupId), + modified.ToUniversalTime().ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture) + ".json"); +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectScheduleCache.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectScheduleCache.cs new file mode 100644 index 0000000000..e1a9e56cc4 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectScheduleCache.cs @@ -0,0 +1,205 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Extensions.Json; +using MediaBrowser.Common.Configuration; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.LiveTv.Listings; + +/// +/// Caches one station's schedule for one day, keyed by the md5 Schedules Direct reports for it. +/// +/// +/// The API requires clients to ask /schedules/md5 what changed and re-download only that. A day +/// whose md5 still matches needs no /schedules, /programs or /metadata/programs request at all, +/// which is what keeps a guide refresh inside the account's daily quotas. +/// +public class SchedulesDirectScheduleCache +{ + private readonly ILogger _logger; + private readonly string _cacheRoot; + private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + /// The application paths. + public SchedulesDirectScheduleCache(ILogger logger, IApplicationPaths appPaths) + { + _logger = logger; + _cacheRoot = Path.Combine(appPaths.CachePath, "schedulesdirect", "schedules"); + } + + /// + /// Reads the cached day if it was stored under the same md5. + /// + /// The station id. + /// The date, formatted yyyy-MM-dd. + /// The md5 Schedules Direct currently reports, or null if unknown. + /// The cancellation token. + /// The cached entry, or null when it is missing or stale. + public async Task GetAsync(string stationId, string date, string? md5, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(md5)) + { + return null; + } + + var cached = await ReadAsync(stationId, date, cancellationToken).ConfigureAwait(false); + return string.Equals(cached?.Md5, md5, StringComparison.Ordinal) ? cached : null; + } + + /// + /// Reads the cached day whatever md5 it was stored under, to stand in for a day that could not + /// be downloaded. + /// + /// The station id. + /// The date, formatted yyyy-MM-dd. + /// The cancellation token. + /// The cached entry, or null when there is none. + public Task GetLatestAsync(string stationId, string date, CancellationToken cancellationToken) + => ReadAsync(stationId, date, cancellationToken); + + /// + /// Stores a freshly downloaded day. + /// + /// The station id. + /// The date, formatted yyyy-MM-dd. + /// The day to store. + /// The cancellation token. + /// A task representing the write. + public async Task SetAsync(string stationId, string date, SchedulesDirectCachedDay day, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(day.Md5)) + { + // Without an md5 there is nothing to validate the entry against later. + return; + } + + var path = GetPath(stationId, date); + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var stream = File.Create(path); + await using (stream.ConfigureAwait(false)) + { + await JsonSerializer.SerializeAsync(stream, day, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to cache Schedules Direct schedule {Path}", path); + } + } + + /// + /// Deletes cached days that are no longer requested, so the cache cannot grow without bound. + /// + /// The station id. + /// The dates to keep. + public void Prune(string stationId, IEnumerable keepDates) + { + var directory = Path.Combine(_cacheRoot, GetSafeName(stationId)); + if (!Directory.Exists(directory)) + { + return; + } + + var keep = new HashSet(keepDates.Select(GetSafeName), StringComparer.Ordinal); + + try + { + foreach (var file in Directory.EnumerateFiles(directory, "*.json")) + { + if (!keep.Contains(Path.GetFileNameWithoutExtension(file))) + { + File.Delete(file); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Failed to prune Schedules Direct cache in {Directory}", directory); + } + } + + /// + /// Deletes entries that have not been touched for a while, so that stations which have left + /// the lineup do not keep their cache forever. + /// + /// The age past which an untouched entry is deleted. + public void PruneStale(TimeSpan maxAge) + { + if (!Directory.Exists(_cacheRoot)) + { + return; + } + + var cutoff = DateTime.UtcNow - maxAge; + + try + { + foreach (var directory in Directory.EnumerateDirectories(_cacheRoot)) + { + foreach (var file in Directory.EnumerateFiles(directory, "*.json")) + { + if (File.GetLastWriteTimeUtc(file) < cutoff) + { + File.Delete(file); + } + } + + if (!Directory.EnumerateFileSystemEntries(directory).Any()) + { + Directory.Delete(directory); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Failed to prune stale Schedules Direct cache entries in {Directory}", _cacheRoot); + } + } + + private async Task ReadAsync(string stationId, string date, CancellationToken cancellationToken) + { + var path = GetPath(stationId, date); + if (!File.Exists(path)) + { + return null; + } + + try + { + var stream = File.OpenRead(path); + await using (stream.ConfigureAwait(false)) + { + return await JsonSerializer.DeserializeAsync(stream, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Discarding unreadable Schedules Direct cache entry {Path}", path); + return null; + } + } + + private static string GetSafeName(string value) + => string.Create(value.Length, value, (span, source) => + { + for (var i = 0; i < source.Length; i++) + { + span[i] = char.IsAsciiLetterOrDigit(source[i]) ? source[i] : '_'; + } + }); + + private string GetPath(string stationId, string date) + => Path.Combine(_cacheRoot, GetSafeName(stationId), GetSafeName(date) + ".json"); +} diff --git a/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs b/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs index e0e5a00fb9..d9e9112fbe 100644 --- a/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs +++ b/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs @@ -30,6 +30,7 @@ public class RecordingsMetadataManager private readonly ILogger _logger; private readonly IConfigurationManager _config; private readonly ILibraryManager _libraryManager; + private readonly ISchedulesDirectService _schedulesDirectService; /// /// Initializes a new instance of the class. @@ -37,14 +38,17 @@ public class RecordingsMetadataManager /// The . /// The . /// The . + /// The . public RecordingsMetadataManager( ILogger logger, IConfigurationManager config, - ILibraryManager libraryManager) + ILibraryManager libraryManager, + ISchedulesDirectService schedulesDirectService) { _logger = logger; _config = config; _libraryManager = libraryManager; + _schedulesDirectService = schedulesDirectService; } /// @@ -469,7 +473,14 @@ public class RecordingsMetadataManager { if (!image.IsLocalFile) { - image = await _libraryManager.ConvertImageToLocal(program, image, 0).ConfigureAwait(false); + var localImage = await _schedulesDirectService.ConvertImageToLocal(program, image, 0).ConfigureAwait(false); + if (localImage is null) + { + _logger.LogDebug("Not saving recording image {Url}: Schedules Direct has blocked it", image.Path); + return; + } + + image = localImage; } var imageSaveFilenameWithoutExtension = image.Type switch diff --git a/tests/Jellyfin.LiveTv.Tests/Guide/GuideManagerTests.cs b/tests/Jellyfin.LiveTv.Tests/Guide/GuideManagerTests.cs new file mode 100644 index 0000000000..a09768df86 --- /dev/null +++ b/tests/Jellyfin.LiveTv.Tests/Guide/GuideManagerTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using Jellyfin.LiveTv.Guide; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.LiveTv; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.IO; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace Jellyfin.LiveTv.Tests.Guide; + +public sealed class GuideManagerTests : IDisposable +{ + private const string ImageUrl = "https://json.schedulesdirect.org/20141201/image/assets/p1_b.jpg?token=abc"; + private const string ImageNotFoundBody = "{\"response\":\"IMAGE_NOT_FOUND\",\"code\":5000,\"message\":\"test\"}"; + + private readonly ILibraryManager? _previousLibraryManager = BaseItem.LibraryManager; + private readonly Mock _libraryManager = new(); + private readonly Mock _schedulesDirect = new(); + + public GuideManagerTests() + { + // UpdateToRepositoryAsync goes through the static. + BaseItem.LibraryManager = _libraryManager.Object; + } + + public void Dispose() + { + BaseItem.LibraryManager = _previousLibraryManager!; + } + + [Fact] + public async Task PreCacheImages_ImageNotFound_ReportsTheBodyAndRemovesTheImage() + { + var program = CreateProgram(); + _schedulesDirect.Setup(s => s.CanDownloadImage(ImageUrl)).Returns(true); + _schedulesDirect.Setup(s => s.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.NotFound, ImageNotFoundBody)) + .Returns(ImageDownloadFailureAction.RemoveImage); + + // ConvertImageToLocal wraps the rejection, which is the only place the SD error body survives. + _libraryManager.Setup(l => l.ConvertImageToLocal(program, It.IsAny(), 0, false)) + .ThrowsAsync(new InvalidOperationException( + "test", + new HttpResponseBodyException("test", HttpStatusCode.NotFound, ImageNotFoundBody))); + + await CreateGuideManager().PreCacheImages([program], DateTime.UtcNow.AddDays(GuideManager.MaxCacheDays)); + + _schedulesDirect.Verify(s => s.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.NotFound, ImageNotFoundBody), Times.Once); + Assert.Empty(program.ImageInfos); + } + + [Fact] + public async Task PreCacheImages_ImageBlocked_RemovesItWithoutDownloading() + { + // A blocked url left on the program would be fetched from SD when a client displays it. + var program = CreateProgram(); + _schedulesDirect.Setup(s => s.CanDownloadImage(ImageUrl)).Returns(false); + + await CreateGuideManager().PreCacheImages([program], DateTime.UtcNow.AddDays(GuideManager.MaxCacheDays)); + + _libraryManager.Verify( + l => l.ConvertImageToLocal(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Assert.Empty(program.ImageInfos); + } + + private static LiveTvProgram CreateProgram() + => new() + { + Id = Guid.NewGuid(), + Name = "Test", + EndDate = DateTime.UtcNow.AddHours(1), + ImageInfos = [new ItemImageInfo { Path = ImageUrl, Type = ImageType.Primary }] + }; + + private GuideManager CreateGuideManager() + => new( + NullLogger.Instance, + Mock.Of(), + Mock.Of(), + Mock.Of(), + _libraryManager.Object, + Mock.Of(), + Mock.Of(), + Mock.Of(), + _schedulesDirect.Object, + null!); +} diff --git a/tests/Jellyfin.LiveTv.Tests/Listings/ListingsManagerTests.cs b/tests/Jellyfin.LiveTv.Tests/Listings/ListingsManagerTests.cs index 40934d9c6b..ebeacf9115 100644 --- a/tests/Jellyfin.LiveTv.Tests/Listings/ListingsManagerTests.cs +++ b/tests/Jellyfin.LiveTv.Tests/Listings/ListingsManagerTests.cs @@ -1,4 +1,8 @@ using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Jellyfin.LiveTv.Configuration; using Jellyfin.LiveTv.Listings; using MediaBrowser.Common.Configuration; @@ -47,4 +51,44 @@ public class ListingsManagerTests _config.GetLiveTvConfiguration().ListingProviders, p => p.Id.Equals(id, StringComparison.Ordinal)); } + + [Fact] + public async Task GetProgramsAsync_FirstProviderCannotListChannels_FallsBackToTheNext() + { + // Arrange + var failing = new Mock(); + failing.SetupGet(x => x.Type).Returns("failing"); + failing.Setup(x => x.GetChannels(It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("offline")); + + var working = new Mock(); + working.SetupGet(x => x.Type).Returns("working"); + working.Setup(x => x.GetChannels(It.IsAny(), It.IsAny())) + .ReturnsAsync(new List { new() { Id = "5", Number = "5" } }); + working.Setup(x => x.GetProgramsAsync(It.IsAny(), "5", It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new[] { new ProgramInfo { Id = "p1" } }); + + Mock.Get(_config) + .Setup(x => x.GetConfiguration(It.IsAny())) + .Returns(new LiveTvOptions + { + ListingProviders = + [ + new ListingsProviderInfo { Id = "a", Type = "failing" }, + new ListingsProviderInfo { Id = "b", Type = "working" } + ] + }); + + var manager = new ListingsManager(_logger, _config, _taskManager, _tunerHostManager, [failing.Object, working.Object]); + + // Act + var programs = await manager.GetProgramsAsync( + new ChannelInfo { Id = "5", Number = "5", TunerHostId = "tuner" }, + DateTime.UtcNow, + DateTime.UtcNow.AddDays(1), + CancellationToken.None); + + // Assert + Assert.Single(programs); + } } diff --git a/tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectLineupTests.cs b/tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectLineupTests.cs deleted file mode 100644 index 04c7b05c3d..0000000000 --- a/tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectLineupTests.cs +++ /dev/null @@ -1,130 +0,0 @@ -using System; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using MediaBrowser.Common.Configuration; -using MediaBrowser.Model.LiveTv; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using Moq.Protected; -using Xunit; -using SchedulesDirectProvider = Jellyfin.LiveTv.Listings.SchedulesDirect; - -namespace Jellyfin.LiveTv.Tests.Listings; - -public class SchedulesDirectLineupTests -{ - private const string InvalidUserResponse = "{\"response\":\"INVALID_USER\",\"code\":4003,\"message\":\"Invalid user.\",\"serverID\":\"AWS-SD-web.1\"}"; - - private static readonly ListingsProviderInfo _info = new() { Username = "user", Password = "password" }; - - [Fact] - public async Task GetLineups_ValidCredentials_ReturnsLineups() - { - var tokenResponse = await CreateSuccessfulLogin(); - using var provider = CreateProvider(tokenResponse, await GetHeadendsResponse()); - - var lineups = await provider.GetLineups(_info, "USA", "90210"); - - Assert.NotEmpty(lineups); - Assert.Contains(lineups, i => string.Equals(i.Id, "USA-OTA-90210", StringComparison.Ordinal)); - Assert.Contains(lineups, i => string.Equals(i.Name, "Antenna", StringComparison.Ordinal)); - } - - [Fact] - public async Task GetLineups_LoginFails_Throws() - { - using var provider = CreateProvider(CreateFailedLogin(), await GetHeadendsResponse()); - - // An empty lineup list is indistinguishable from "no lineups for this location", so a - // failed login has to surface as an error instead. - await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); - } - - [Fact] - public async Task Validate_LoginFails_Throws() - { - using var provider = CreateProvider(CreateFailedLogin(), await GetHeadendsResponse()); - - await Assert.ThrowsAnyAsync(() => provider.Validate(_info, true, false)); - } - - [Fact] - public async Task Validate_AfterAccountError_RecoversWithoutRestart() - { - var login = CreateFailedLogin(); - using var provider = CreateProvider(login, await GetHeadendsResponse()); - - await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); - - // The account error disables Schedules Direct; saving the provider again is the user - // correcting their credentials, and that has to recover without a server restart. - login.Status = HttpStatusCode.OK; - login.Body = await GetTokenResponse(); - - await provider.Validate(_info, true, false); - - Assert.NotEmpty(await provider.GetLineups(_info, "USA", "90210")); - } - - private static async Task CreateSuccessfulLogin() - => new() { Status = HttpStatusCode.OK, Body = await GetTokenResponse() }; - - private static Response CreateFailedLogin() - => new() { Status = HttpStatusCode.BadRequest, Body = InvalidUserResponse }; - - private static Task GetTokenResponse() - => File.ReadAllTextAsync("Test Data/SchedulesDirect/token_live_response.json", TestContext.Current.CancellationToken); - - private static Task GetHeadendsResponse() - => File.ReadAllTextAsync("Test Data/SchedulesDirect/headends_response.json", TestContext.Current.CancellationToken); - - private static SchedulesDirectProvider CreateProvider(Response login, string headendsResponse) - { - var messageHandler = new Mock(); - messageHandler.Protected() - .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) - .Returns((m, _) => - { - var path = m.RequestUri!.AbsolutePath; - if (path.EndsWith("/token", StringComparison.Ordinal)) - { - return Task.FromResult(new HttpResponseMessage(login.Status) - { - Content = new StringContent(login.Body) - }); - } - - if (path.EndsWith("/headends", StringComparison.Ordinal)) - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(headendsResponse) - }); - } - - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)); - }); - - var httpClientFactory = new Mock(); - httpClientFactory.Setup(x => x.CreateClient(It.IsAny())) - .Returns(() => new HttpClient(messageHandler.Object)); - - var appPaths = new Mock(); - appPaths.SetupGet(x => x.CachePath).Returns(Path.GetTempPath()); - - return new SchedulesDirectProvider( - NullLogger.Instance, - httpClientFactory.Object, - appPaths.Object); - } - - private sealed class Response - { - public HttpStatusCode Status { get; set; } - - public string Body { get; set; } = string.Empty; - } -} diff --git a/tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectTests.cs b/tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectTests.cs new file mode 100644 index 0000000000..51d855cf8a --- /dev/null +++ b/tests/Jellyfin.LiveTv.Tests/Listings/SchedulesDirectTests.cs @@ -0,0 +1,915 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.LiveTv; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.LiveTv; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Moq.Protected; +using Xunit; +using SchedulesDirectProvider = Jellyfin.LiveTv.Listings.SchedulesDirect; + +namespace Jellyfin.LiveTv.Tests.Listings; + +public class SchedulesDirectTests : IDisposable +{ + private const string InvalidUserResponse = "{\"response\":\"INVALID_USER\",\"code\":4003,\"message\":\"Invalid user.\",\"serverID\":\"AWS-SD-web.1\"}"; + private const string ImageUrl = "https://json.schedulesdirect.org/20141201/image/assets/p1_b.jpg?token=abc"; + + private static readonly ListingsProviderInfo _info = new() { Username = "user", Password = "password" }; + + private readonly Mock _libraryManager = new(); + + // The daily limit files live under CachePath, so every provider needs its own directory or + // one test leaves a limit behind for the next. + private readonly string _cachePath = Path.Combine(Path.GetTempPath(), "jf-sd-" + Guid.NewGuid().ToString("N")); + + [Fact] + public async Task GetLineups_ValidCredentials_ReturnsLineups() + { + var tokenResponse = await CreateSuccessfulLogin(); + using var provider = CreateProvider(tokenResponse, await GetHeadendsResponse()); + + var lineups = await provider.GetLineups(_info, "USA", "90210"); + + Assert.NotEmpty(lineups); + Assert.Contains(lineups, i => string.Equals(i.Id, "USA-OTA-90210", StringComparison.Ordinal)); + Assert.Contains(lineups, i => string.Equals(i.Name, "Antenna", StringComparison.Ordinal)); + } + + [Fact] + public async Task GetLineups_LoginFails_Throws() + { + using var provider = CreateProvider(CreateFailedLogin(), await GetHeadendsResponse()); + + // An empty lineup list is indistinguishable from "no lineups for this location", so a + // failed login has to surface as an error instead. + await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); + } + + [Fact] + public async Task Validate_LoginFails_Throws() + { + using var provider = CreateProvider(CreateFailedLogin(), await GetHeadendsResponse()); + + await Assert.ThrowsAnyAsync(() => provider.Validate(_info, true, false)); + } + + [Fact] + public async Task Validate_AfterAccountError_RecoversWithoutRestart() + { + var login = CreateFailedLogin(); + using var provider = CreateProvider(login, await GetHeadendsResponse()); + + await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); + + // The account error disables Schedules Direct; saving the provider again is the user + // correcting their credentials, and that has to recover without a server restart. + login.Status = HttpStatusCode.OK; + login.Body = await GetTokenResponse(); + + await provider.Validate(_info, true, false); + + Assert.NotEmpty(await provider.GetLineups(_info, "USA", "90210")); + } + + [Fact] + public async Task GetLineups_TokenRejectedWithHttp200_DisablesService() + { + // The account lockout this guards against arrived as HTTP 200 carrying code 4005. + var login = new Response + { + Status = HttpStatusCode.OK, + Body = "{\"response\":\"JSON_ACCOUNT_ACCESS_DISABLED\",\"code\":4005,\"serverID\":\"20141201.web\",\"message\":\"Access to account via JSON has been disabled.\"}" + }; + using var provider = CreateProvider(login, await GetHeadendsResponse()); + + await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); + + // The account error disables Schedules Direct: a retry must not reach the network again. + login.Body = await GetTokenResponse(); + await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); + } + + [Theory] + [InlineData(5002, "MAX_IMAGE_DOWNLOADS")] + [InlineData(5003, "MAX_IMAGE_DOWNLOADS_TRIAL")] + public async Task ReportImageDownloadFailure_ImageLimitCode_BlocksOnFirstReport(int code, string response) + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + var action = provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.NotFound, ErrorBody(code, response)); + + Assert.Equal(ImageDownloadFailureAction.None, action); + Assert.False(provider.CanDownloadImage(ImageUrl)); + } + + [Fact] + public async Task ReportImageDownloadFailure_ImageNotFound_DropsUrlWithoutBlockingTheDay() + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + var action = provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.NotFound, ErrorBody(5000, "IMAGE_NOT_FOUND")); + + // The spec requires that url is never requested again, but the day is not blocked and + // every other image stays downloadable. + Assert.Equal(ImageDownloadFailureAction.RemoveImage, action); + Assert.False(provider.CanDownloadImage(ImageUrl)); + Assert.True(provider.CanDownloadImage("https://json.schedulesdirect.org/20141201/image/assets/p2_b.jpg?token=abc")); + } + + [Fact] + public async Task ReportImageDownloadFailure_InvalidUriLimit_BlocksAndDropsUrl() + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + var action = provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.NotFound, ErrorBody(5004, "MAX_IMAGE_INVALID_URI_ERRORS")); + + Assert.Equal(ImageDownloadFailureAction.RemoveImage, action); + Assert.False(provider.CanDownloadImage(ImageUrl)); + } + + [Theory] + [InlineData(1004, "TOKEN_MISSING")] + [InlineData(4006, "TOKEN_EXPIRED")] + public async Task ReportImageDownloadFailure_TokenCode_DoesNotBlockTheDay(int code, string response) + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + var action = provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.Forbidden, ErrorBody(code, response)); + + Assert.Equal(ImageDownloadFailureAction.RemoveImage, action); + Assert.True(provider.CanDownloadImage(ImageUrl)); + } + + [Fact] + public async Task ReportImageDownloadFailure_UnreadableBody_FallsBackToFailureStreak() + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + for (var i = 0; i < 9; i++) + { + provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.Forbidden, "gateway error"); + Assert.True(provider.CanDownloadImage(ImageUrl)); + } + + provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.Forbidden, "gateway error"); + + Assert.False(provider.CanDownloadImage(ImageUrl)); + } + + [Fact] + public async Task ReportImageDownloadSuccess_ResetsUnexplainedFailureStreak() + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + for (var i = 0; i < 9; i++) + { + provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.Forbidden, null); + } + + provider.ReportImageDownloadSuccess(ImageUrl); + provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.Forbidden, null); + + Assert.True(provider.CanDownloadImage(ImageUrl)); + } + + [Fact] + public async Task ReportImageDownloadFailure_ForeignUrl_IsIgnored() + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + var action = provider.ReportImageDownloadFailure("https://example.com/image.jpg", HttpStatusCode.NotFound, ErrorBody(5002, "MAX_IMAGE_DOWNLOADS")); + + Assert.Equal(ImageDownloadFailureAction.None, action); + Assert.True(provider.CanDownloadImage(ImageUrl)); + Assert.True(provider.CanDownloadImage("https://example.com/image.jpg")); + } + + [Fact] + public async Task ConvertImageToLocal_Blocked_DoesNotDownload() + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.NotFound, ErrorBody(5002, "MAX_IMAGE_DOWNLOADS")); + + var image = await provider.ConvertImageToLocal(new LiveTvProgram(), new ItemImageInfo { Path = ImageUrl, Type = ImageType.Primary }, 0); + + Assert.Null(image); + _libraryManager.Verify( + l => l.ConvertImageToLocal(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task ConvertImageToLocal_ImageNotFound_StopsOfferingTheUri() + { + using var provider = CreateProvider(await CreateSuccessfulLogin(), await GetHeadendsResponse()); + + // LibraryManager wraps the rejection, which is the only place the SD error body survives. + _libraryManager.Setup(l => l.ConvertImageToLocal(It.IsAny(), It.IsAny(), 0, true)) + .ThrowsAsync(new InvalidOperationException( + "test", + new HttpResponseBodyException("test", HttpStatusCode.NotFound, ErrorBody(5000, "IMAGE_NOT_FOUND")))); + + await Assert.ThrowsAsync( + () => provider.ConvertImageToLocal(new LiveTvProgram(), new ItemImageInfo { Path = ImageUrl, Type = ImageType.Primary }, 0)); + + Assert.False(provider.CanDownloadImage(ImageUrl)); + } + + [Fact] + public async Task GetProgramsAsync_UnchangedMd5_ServesSecondRefreshFromCache() + { + var server = new FakeServer(); + using var provider = CreateProvider(server); + + var first = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + Assert.NotEmpty(first); + + var schedulesAfterFirst = server.Counts["/schedules"]; + var programsAfterFirst = server.Counts.GetValueOrDefault("/programs"); + Assert.True(schedulesAfterFirst > 0); + Assert.True(programsAfterFirst > 0); + + var second = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + // The md5 is unchanged, so only the hash check should have gone to the server. + Assert.Equal(first.Count(), second.Count()); + Assert.Equal(schedulesAfterFirst, server.Counts["/schedules"]); + Assert.Equal(programsAfterFirst, server.Counts.GetValueOrDefault("/programs")); + Assert.Equal(2, server.Counts["/schedules/md5"]); + } + + [Fact] + public async Task GetProgramsAsync_ChangedMd5_DownloadsTheScheduleAgain() + { + var server = new FakeServer(); + using var provider = CreateProvider(server); + + await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + var afterFirst = server.Counts["/schedules"]; + + server.Md5Salt = "changed"; + await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.True(server.Counts["/schedules"] > afterFirst); + } + + [Fact] + public async Task GetProgramsAsync_SystemOffline_ThrowsInsteadOfReportingNoPrograms() + { + var server = new FakeServer { SystemStatus = "Offline" }; + using var provider = CreateProvider(server); + + // An empty list means "this channel has no programs", and the guide refresh acts on that + // by deleting the ones it already has. A failure has to surface as one. + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + + Assert.Equal(0, server.Counts.GetValueOrDefault("/schedules")); + } + + [Fact] + public async Task GetProgramsAsync_OfflineCodeWithHttp200_ThrowsInsteadOfReportingNoPrograms() + { + // A 200 carrying an error code used to deserialize into an empty status that read as online. + var server = new FakeServer { StatusCode = 3000 }; + using var provider = CreateProvider(server); + + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + + Assert.Equal(0, server.Counts.GetValueOrDefault("/schedules")); + } + + [Fact] + public async Task GetProgramsAsync_ImageLimitHit_StillReturnsPrograms() + { + // The image limit must not be mistaken for a fetch failure: the listings are still good, + // only the artwork is unavailable. + var server = new FakeServer { HasImageArtwork = true }; + using var provider = CreateProvider(server); + + provider.ReportImageDownloadFailure(ImageUrl, HttpStatusCode.NotFound, ErrorBody(5002, "MAX_IMAGE_DOWNLOADS")); + Assert.False(provider.CanDownloadImage(ImageUrl)); + + var programs = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.NotEmpty(programs); + Assert.All(programs, p => Assert.Null(p.ImageUrl)); + Assert.Equal(0, server.Counts.GetValueOrDefault("/metadata/programs")); + } + + [Fact] + public async Task GetProgramsAsync_Md5SaysDateIsOutOfRange_DoesNotRequestTheSchedule() + { + // 7020 is what the hash check exists to catch: asking /schedules for the day anyway just + // earns the same error, which is why the spec points at the md5 endpoint for the range. + var server = new FakeServer { Md5ErrorCode = 7020 }; + using var provider = CreateProvider(server); + + var programs = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.Empty(programs); + Assert.Equal(1, server.Counts["/schedules/md5"]); + Assert.Equal(0, server.Counts.GetValueOrDefault("/schedules")); + } + + [Fact] + public async Task GetProgramsAsync_ScheduleQueued_HonoursRetryTime() + { + // The queued response is a bare object rather than the usual array, and it carries the + // earliest time the schedule may be asked for again. + var server = new FakeServer { ScheduleRetryTime = DateTime.UtcNow.AddMinutes(10) }; + using var provider = CreateProvider(server); + + // A queued schedule is a failure, not an empty channel: reporting no programs would have + // the guide refresh delete the ones this channel already has. + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + var afterFirst = server.Counts["/schedules"]; + + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + + Assert.Equal(afterFirst, server.Counts["/schedules"]); + } + + [Fact] + public async Task GetProgramsAsync_ScheduleQueuedRetryTimePassed_AsksAgain() + { + var server = new FakeServer { ScheduleRetryTime = DateTime.UtcNow.AddSeconds(-1) }; + using var provider = CreateProvider(server); + + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + var afterFirst = server.Counts["/schedules"]; + + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + + Assert.True(server.Counts["/schedules"] > afterFirst); + } + + [Fact] + public async Task GetProgramsAsync_CacheHit_StillRefreshesArtwork() + { + // Artwork uris are ephemeral and their changes are not reflected in the schedule md5, so + // the spec forbids storing them; the index has to be requested again on a cache hit. + var server = new FakeServer { HasImageArtwork = true }; + using var provider = CreateProvider(server); + + var first = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + Assert.All(first, p => Assert.Contains("p1_b.jpg", p.ImageUrl, StringComparison.Ordinal)); + + var metadataAfterFirst = server.Counts["/metadata/programs"]; + + var second = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.Equal(1, server.Counts["/schedules"]); + Assert.True(server.Counts["/metadata/programs"] > metadataAfterFirst); + Assert.All(second, p => Assert.Contains("p1_b.jpg", p.ImageUrl, StringComparison.Ordinal)); + } + + [Fact] + public async Task GetProgramsAsync_AfterImageNotFound_StopsOfferingTheUri() + { + var server = new FakeServer { HasImageArtwork = true }; + using var provider = CreateProvider(server); + + var first = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + var imageUrl = first.First().ImageUrl; + Assert.NotNull(imageUrl); + + Assert.Equal( + ImageDownloadFailureAction.RemoveImage, + provider.ReportImageDownloadFailure(imageUrl, HttpStatusCode.NotFound, ErrorBody(5000, "IMAGE_NOT_FOUND"))); + + // Handing the same dead uri back every refresh is what walks an account into + // MAX_IMAGE_INVALID_URI_ERRORS, so it must not survive the rejection. + var second = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.All(second, p => Assert.Null(p.ImageUrl)); + } + + [Fact] + public async Task GetProgramsAsync_OneDayQueuedNextToCachedDays_ServesThePreviousCopy() + { + var server = new FakeServer(); + using var provider = CreateProvider(server); + + var first = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + // One day changed and is now queued; the others still match the cache. Leaving the queued + // day out would have the guide refresh delete its programs. + server.ChangedDate = server.RequestedDates[0]; + server.ScheduleRetryTime = DateTime.UtcNow.AddMinutes(10); + var second = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.Equal(2, server.Counts["/schedules"]); + Assert.Equal(first.Select(p => p.Id).Order(), second.Select(p => p.Id).Order()); + } + + [Fact] + public async Task GetProgramsAsync_ProgramQueued_DoesNotCacheTheIncompleteDay() + { + var server = new FakeServer { ProgramErrorCode = 6001 }; + using var provider = CreateProvider(server); + + // With no previous copy to stand in, a queued program fails the channel rather than + // dropping its airings, which the guide refresh would delete. + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + + // The program finishing generation need not change the day's md5, so the day must be + // downloaded again rather than served from the cache without it. + server.ProgramErrorCode = 0; + var second = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.NotEmpty(second); + Assert.Equal(2, server.Counts["/programs"]); + } + + [Theory] + [InlineData(null)] + [InlineData("EP000000020001")] + public async Task GetProgramsAsync_ProgramQueuedAfterGoodRefresh_KeepsItsAirings(string? queuedProgramId) + { + var server = new FakeServer { SecondProgram = true }; + using var provider = CreateProvider(server); + + var first = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + Assert.Contains(first, p => p.Id!.Contains("EP000000020001", StringComparison.Ordinal)); + + // The day changed and one (or every) program on it is still being generated. + server.Md5Salt = "changed"; + server.ProgramErrorCode = 6001; + server.QueuedProgramId = queuedProgramId; + var second = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.Equal(first.Select(p => p.Id).Order(), second.Select(p => p.Id).Order()); + } + + [Theory] + [InlineData(HttpStatusCode.OK, 3000, "SERVICE_OFFLINE")] + [InlineData(HttpStatusCode.Forbidden, 4009, "TOO_MANY_LOGINS")] + public async Task GetProgramsAsync_StopCodeOnMd5_DoesNotRequestSchedules(HttpStatusCode status, int code, string response) + { + var server = new FakeServer { Md5RejectionBody = ErrorBody(code, response), Md5RejectionStatus = status }; + using var provider = CreateProvider(server); + + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + + Assert.Equal(0, server.Counts.GetValueOrDefault("/schedules")); + } + + [Theory] + [InlineData(1004, "TOKEN_MISSING")] + [InlineData(1010, "INVALID_TOKEN")] + public async Task GetProgramsAsync_TokenRejected_LogsInAgain(int code, string response) + { + var server = new FakeServer { Md5RejectionBody = ErrorBody(code, response) }; + using var provider = CreateProvider(server); + + var programs = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + + Assert.NotEmpty(programs); + Assert.Equal(2, server.Counts["/token"]); + } + + [Fact] + public async Task GetProgramsAsync_ServiceOffline_DoesNotLogInAgainDuringTheOfflineHour() + { + var server = new FakeServer { StatusCode = 3000 }; + using var provider = CreateProvider(server); + + for (var i = 0; i < 3; i++) + { + await Assert.ThrowsAnyAsync( + () => provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken)); + } + + // MAX_LOGIN_ATTEMPTS counts logins, so each channel of a refresh must not log in again. + Assert.Equal(1, server.Counts["/token"]); + } + + [Fact] + public async Task GetProgramsAsync_AfterImageNotFoundAndRestart_StopsOfferingTheUri() + { + var server = new FakeServer { HasImageArtwork = true }; + + using (var provider = CreateProvider(server)) + { + var first = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + provider.ReportImageDownloadFailure(first.First().ImageUrl!, HttpStatusCode.NotFound, ErrorBody(5000, "IMAGE_NOT_FOUND")); + } + + // A second provider over the same cache directory stands in for a restart. + using (var provider = CreateProvider(server)) + { + var second = await provider.GetProgramsAsync(_info, "20454", DateTime.UtcNow, DateTime.UtcNow.AddDays(1), TestContext.Current.CancellationToken); + Assert.All(second, p => Assert.Null(p.ImageUrl)); + } + } + + [Theory] + [InlineData(HttpStatusCode.Forbidden, 4009, "TOO_MANY_LOGINS")] + [InlineData(HttpStatusCode.OK, 4009, "TOO_MANY_LOGINS")] + [InlineData(HttpStatusCode.Forbidden, 4010, "TOO_MANY_UNIQUE_IPS")] + public async Task GetLineups_AccountLimitReached_StopsLoggingIn(HttpStatusCode status, int code, string response) + { + var login = new Response + { + Status = status, + Body = ErrorBody(code, response) + }; + using var provider = CreateProvider(login, await GetHeadendsResponse()); + + await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); + + // These codes count logins, so continuing to ask for a token is what earned them: a + // retry must not go back to /token even once the credentials would work. + login.Status = HttpStatusCode.OK; + login.Body = await GetTokenResponse(); + await Assert.ThrowsAnyAsync(() => provider.GetLineups(_info, "USA", "90210")); + } + + [Fact] + public async Task GetChannels_SystemOffline_ThrowsInsteadOfReportingNoChannels() + { + var server = new FakeServer { SystemStatus = "Offline" }; + using var provider = CreateProvider(server); + + // No channels means no channel is mapped, which the guide refresh reads as "no programs". + await Assert.ThrowsAnyAsync( + () => provider.GetChannels(new ListingsProviderInfo { Username = "user", Password = "password", ListingsId = "USA-OTA-90210" }, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task GetChannels_LoginBackedOff_ThrowsInsteadOfReportingNoChannels() + { + var login = new Response { Status = HttpStatusCode.Forbidden, Body = ErrorBody(4009, "TOO_MANY_LOGINS") }; + using var provider = CreateProvider(login, await GetHeadendsResponse()); + var info = new ListingsProviderInfo { Username = "user", Password = "password", ListingsId = "USA-OTA-90210" }; + + await Assert.ThrowsAnyAsync(() => provider.GetChannels(info, TestContext.Current.CancellationToken)); + + // The second call never reaches /token, so this is the null-token path. + await Assert.ThrowsAnyAsync(() => provider.GetChannels(info, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task GetChannels_UnchangedModifiedDate_ServesTheLineupFromCache() + { + var server = new FakeServer(); + using var provider = CreateProvider(server); + var info = new ListingsProviderInfo { Username = "user", Password = "password", ListingsId = "USA-OTA-90210" }; + + var first = await provider.GetChannels(info, TestContext.Current.CancellationToken); + Assert.NotEmpty(first); + Assert.Equal(1, server.Counts["/lineups/USA-OTA-90210"]); + + var second = await provider.GetChannels(info, TestContext.Current.CancellationToken); + + Assert.Equal(first.Count, second.Count); + Assert.Equal(1, server.Counts["/lineups/USA-OTA-90210"]); + } + + [Fact] + public async Task GetChannels_NewerModifiedDate_DownloadsTheLineupAgain() + { + var server = new FakeServer(); + var info = new ListingsProviderInfo { Username = "user", Password = "password", ListingsId = "USA-OTA-90210" }; + + using (var provider = CreateProvider(server)) + { + await provider.GetChannels(info, TestContext.Current.CancellationToken); + } + + server.LineupModified = server.LineupModified.AddDays(1); + + // A second provider over the same cache directory stands in for a restart, which is also + // what makes the status re-read rather than coming from its short-lived cache. + using (var provider = CreateProvider(server)) + { + await provider.GetChannels(info, TestContext.Current.CancellationToken); + } + + Assert.Equal(2, server.Counts["/lineups/USA-OTA-90210"]); + } + + [Fact] + public async Task Validate_LineupCannotBeAdded_Throws() + { + var server = new FakeServer { LineupPutStatus = HttpStatusCode.BadRequest }; + using var provider = CreateProvider(server); + + // Silently reporting success leaves the user with a provider that has no lineup. + await Assert.ThrowsAnyAsync( + () => provider.Validate(new ListingsProviderInfo { Username = "user", Password = "password", ListingsId = "USA-OTA-90210" }, true, true)); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing && Directory.Exists(_cachePath)) + { + Directory.Delete(_cachePath, true); + } + } + + // Schedules Direct answers an exhausted image quota with HTTP 200 and a body like this, which + // ProviderManager surfaces as a synthetic 404 because it is not an image content type. + private static string ErrorBody(int code, string response) + => string.Format( + CultureInfo.InvariantCulture, + "{{\"response\":\"{0}\",\"code\":{1},\"serverID\":\"20141201.debug\",\"message\":\"test\",\"datetime\":\"2026-09-14T00:00:00Z\"}}", + response, + code); + + private static async Task CreateSuccessfulLogin() + => new() { Status = HttpStatusCode.OK, Body = await GetTokenResponse() }; + + private static Response CreateFailedLogin() + => new() { Status = HttpStatusCode.BadRequest, Body = InvalidUserResponse }; + + private static Task GetTokenResponse() + => File.ReadAllTextAsync("Test Data/SchedulesDirect/token_live_response.json", TestContext.Current.CancellationToken); + + private static Task GetHeadendsResponse() + => File.ReadAllTextAsync("Test Data/SchedulesDirect/headends_response.json", TestContext.Current.CancellationToken); + + private SchedulesDirectProvider CreateProvider(Response login, string headendsResponse) + { + var messageHandler = new Mock(); + messageHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((m, _) => + { + var path = m.RequestUri!.AbsolutePath; + if (path.EndsWith("/token", StringComparison.Ordinal)) + { + return Task.FromResult(new HttpResponseMessage(login.Status) + { + Content = new StringContent(login.Body) + }); + } + + if (path.EndsWith("/headends", StringComparison.Ordinal)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(headendsResponse) + }); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)); + }); + + var httpClientFactory = new Mock(); + httpClientFactory.Setup(x => x.CreateClient(It.IsAny())) + .Returns(() => new HttpClient(messageHandler.Object)); + + var appPaths = new Mock(); + appPaths.SetupGet(x => x.CachePath).Returns(_cachePath); + + return new SchedulesDirectProvider( + NullLogger.Instance, + httpClientFactory.Object, + appPaths.Object, + _libraryManager.Object); + } + + private SchedulesDirectProvider CreateProvider(FakeServer server) + { + var messageHandler = new Mock(); + messageHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((m, ct) => server.Respond(m, ct)); + + var httpClientFactory = new Mock(); + httpClientFactory.Setup(x => x.CreateClient(It.IsAny())) + .Returns(() => new HttpClient(messageHandler.Object)); + + var appPaths = new Mock(); + appPaths.SetupGet(x => x.CachePath).Returns(_cachePath); + + return new SchedulesDirectProvider( + NullLogger.Instance, + httpClientFactory.Object, + appPaths.Object, + _libraryManager.Object); + } + + /// + /// A minimal stand-in for the Schedules Direct API that counts what was asked of it. + /// + private sealed class FakeServer + { + public Dictionary Counts { get; } = new(StringComparer.Ordinal); + + public string SystemStatus { get; set; } = "Online"; + + public int StatusCode { get; set; } + + public string Md5Salt { get; set; } = "base"; + + public HttpStatusCode LineupPutStatus { get; set; } = HttpStatusCode.OK; + + /// + /// Gets or sets a per-day error code for the hash endpoint, as SCHEDULE_RANGE_EXCEEDED is + /// reported. Zero means every day is available. + /// + public int Md5ErrorCode { get; set; } + + /// + /// Gets or sets the retryTime of a SCHEDULE_QUEUED response, which the spec documents as a + /// bare object rather than the usual array. + /// + public DateTime? ScheduleRetryTime { get; set; } + + public bool HasImageArtwork { get; set; } + + /// + /// Gets or sets a date whose schedule md5 differs from the one first reported. + /// + public string? ChangedDate { get; set; } + + public List RequestedDates { get; private set; } = []; + + /// + /// Gets or sets a per-program error code for the programs endpoint, such as PROGRAMID_QUEUED. + /// + public int ProgramErrorCode { get; set; } + + /// + /// Gets or sets a body the hash endpoint rejects the next request's token with, as HTTP 403. + /// + public string? Md5RejectionBody { get; set; } + + public HttpStatusCode Md5RejectionStatus { get; set; } = HttpStatusCode.Forbidden; + + /// + /// Gets or sets a value indicating whether each day airs a second program. + /// + public bool SecondProgram { get; set; } + + /// + /// Gets or sets the only program applies to; null means all of them. + /// + public string? QueuedProgramId { get; set; } + + public string ImageUri { get; set; } = "assets/p1_b.jpg"; + + public DateTime LineupModified { get; set; } = new(2026, 9, 14, 0, 0, 0, DateTimeKind.Utc); + + public async Task Respond(HttpRequestMessage message, CancellationToken cancellationToken) + { + var path = message.RequestUri!.AbsolutePath.Replace("/20141201", string.Empty, StringComparison.Ordinal).TrimEnd('/'); + Counts[path] = Counts.GetValueOrDefault(path) + 1; + + string body; + switch (path) + { + case "/token": + body = "{\"code\":0,\"message\":\"OK\",\"serverID\":\"test\",\"token\":\"f3fca79989cafe7dead71beefedc812b\"}"; + break; + + case "/status": + body = "{\"code\":" + StatusCode.ToString(CultureInfo.InvariantCulture) + ",\"account\":{\"maxLineups\":4},\"lineups\":[{\"lineup\":\"USA-OTA-90210\",\"modified\":\"" + + LineupModified.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) + + "\"}],\"systemStatus\":[{\"status\":\"" + SystemStatus + "\",\"message\":\"test\"}]}"; + break; + + case "/schedules/md5": + if (Md5RejectionBody is not null) + { + var rejection = Md5RejectionBody; + Md5RejectionBody = null; + return new HttpResponseMessage(Md5RejectionStatus) { Content = new StringContent(rejection) }; + } + + RequestedDates = await ReadDates(message, cancellationToken).ConfigureAwait(false); + body = BuildMd5Response(RequestedDates); + break; + + case "/schedules": + body = BuildSchedulesResponse(await ReadDates(message, cancellationToken).ConfigureAwait(false)); + break; + + case "/programs": + var ids = JsonSerializer.Deserialize>(await message.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false))!; + body = "[" + string.Join(',', ids.Select(BuildProgramEntry)) + "]"; + break; + + case "/metadata/programs": + body = HasImageArtwork + ? "[{\"programID\":\"EP000000010001\",\"data\":[{\"width\":\"120\",\"height\":\"180\",\"uri\":\"" + + ImageUri + "\",\"aspect\":\"2x3\",\"category\":\"Iconic\",\"text\":\"yes\",\"tier\":\"Series\"}]}]" + : "[]"; + break; + + case "/lineups/USA-OTA-90210": + if (message.Method == HttpMethod.Get) + { + body = "{\"map\":[{\"stationID\":\"20454\",\"channel\":\"5\"}],\"stations\":[{\"stationID\":\"20454\",\"name\":\"Test\",\"callsign\":\"TST\"}]}"; + break; + } + + return new HttpResponseMessage(LineupPutStatus) + { + Content = new StringContent(LineupPutStatus == HttpStatusCode.OK + ? "{\"code\":0,\"message\":\"OK\"}" + : "{\"response\":\"MAX_LINEUPS\",\"code\":4101,\"message\":\"Exceeded number of lineups.\"}") + }; + + case "/lineups": + body = "{\"code\":0,\"lineups\":[]}"; + break; + + default: + return new HttpResponseMessage(HttpStatusCode.NotFound) { Content = new StringContent("{}") }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; + } + + private static async Task> ReadDates(HttpRequestMessage message, CancellationToken cancellationToken) + { + var requestBody = await message.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + using var doc = JsonDocument.Parse(requestBody); + + var dates = new List(); + foreach (var station in doc.RootElement.EnumerateArray()) + { + foreach (var date in station.GetProperty("date").EnumerateArray()) + { + dates.Add(date.GetString()!); + } + } + + return dates; + } + + private string BuildMd5Response(List dates) + { + var days = dates.ConvertAll(d => Md5ErrorCode == 0 + ? "\"" + d + "\":{\"code\":0,\"message\":\"OK\",\"lastModified\":\"2026-09-14T00:00:00Z\",\"md5\":\"" + GetMd5(d) + "\"}" + : "\"" + d + "\":{\"code\":" + Md5ErrorCode.ToString(CultureInfo.InvariantCulture) + ",\"message\":\"Date requested not within range.\"}"); + + return "{\"20454\":{" + string.Join(',', days) + "}}"; + } + + private string GetMd5(string date) + => (string.Equals(date, ChangedDate, StringComparison.Ordinal) ? "changed" : Md5Salt) + date; + + private string BuildProgramEntry(string id) + => ProgramErrorCode != 0 && (QueuedProgramId is null || string.Equals(id, QueuedProgramId, StringComparison.Ordinal)) + ? "{\"programID\":\"" + id + "\",\"code\":" + ProgramErrorCode.ToString(CultureInfo.InvariantCulture) + ",\"message\":\"test\"}" + : "{\"programID\":\"" + id + "\",\"titles\":[{\"title120\":\"Test\"}],\"hasImageArtwork\":" + + (HasImageArtwork ? "true" : "false") + "}"; + + private string BuildSchedulesResponse(List dates) + { + if (ScheduleRetryTime is { } retryTime) + { + return "{\"response\":\"SCHEDULE_QUEUED\",\"code\":7100,\"serverID\":\"test\",\"message\":\"Queued.\",\"stationID\":\"20454\",\"retryTime\":\"" + + retryTime.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) + "\"}"; + } + + var days = dates.ConvertAll(d => + "{\"stationID\":\"20454\",\"programs\":[{\"programID\":\"EP000000010001\",\"airDateTime\":\"" + d + + "T12:00:00Z\",\"duration\":1800,\"md5\":\"x\"}" + + (SecondProgram ? ",{\"programID\":\"EP000000020001\",\"airDateTime\":\"" + d + "T14:00:00Z\",\"duration\":1800,\"md5\":\"y\"}" : string.Empty) + + "],\"metadata\":{\"modified\":\"2026-09-14T00:00:00Z\",\"md5\":\"" + + GetMd5(d) + "\",\"startDate\":\"" + d + "\"}}"); + + return "[" + string.Join(',', days) + "]"; + } + } + + private sealed class Response + { + public HttpStatusCode Status { get; set; } + + public string Body { get; set; } = string.Empty; + } +} diff --git a/tests/Jellyfin.LiveTv.Tests/Recordings/RecordingsMetadataManagerTests.cs b/tests/Jellyfin.LiveTv.Tests/Recordings/RecordingsMetadataManagerTests.cs index 14ce470fb4..193f794f1d 100644 --- a/tests/Jellyfin.LiveTv.Tests/Recordings/RecordingsMetadataManagerTests.cs +++ b/tests/Jellyfin.LiveTv.Tests/Recordings/RecordingsMetadataManagerTests.cs @@ -10,6 +10,7 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.Entities; using MediaBrowser.Model.LiveTv; using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -19,6 +20,8 @@ namespace Jellyfin.LiveTv.Tests.Recordings; public sealed class RecordingsMetadataManagerTests { + private const string ImageUrl = "https://json.schedulesdirect.org/20141201/image/assets/p1_b.jpg?token=abc"; + private readonly string _tempDir = Path.Combine(Path.GetTempPath(), "jellyfin-test-" + Guid.NewGuid()); @@ -43,7 +46,8 @@ public sealed class RecordingsMetadataManagerTests var manager = new RecordingsMetadataManager( NullLogger.Instance, config.Object, - libraryManager.Object); + libraryManager.Object, + Mock.Of()); var timer = new TimerInfo { Name = "Test Recording", ProgramId = null }; @@ -61,4 +65,47 @@ public sealed class RecordingsMetadataManagerTests Assert.InRange(parsed, beforeUtc, afterUtc); } + + [Fact] + public async Task SaveRecordingMetadata_RemoteImage_GoesThroughSchedulesDirect() + { + // Schedules Direct counts every image request, so a recording must not fetch one behind its back. + var (manager, libraryManager, schedulesDirect, recordingPath) = CreateImageManager(); + + await manager.SaveRecordingMetadata(new TimerInfo { Name = "Test Recording", ProgramId = "p1" }, recordingPath, null); + + schedulesDirect.Verify(s => s.ConvertImageToLocal(It.IsAny(), It.Is(i => i.Path == ImageUrl), 0), Times.Once); + libraryManager.Verify( + l => l.ConvertImageToLocal(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + private (RecordingsMetadataManager Manager, Mock LibraryManager, Mock SchedulesDirect, string RecordingPath) CreateImageManager() + { + Directory.CreateDirectory(_tempDir); + var recordingPath = Path.Combine(_tempDir, "test-recording.ts"); + + var config = new Mock(); + config.Setup(c => c.GetConfiguration("livetv")) + .Returns(new LiveTvOptions { SaveRecordingNFO = false, SaveRecordingImages = true }); + + var program = new LiveTvProgram + { + Name = "Test", + ImageInfos = [new ItemImageInfo { Path = ImageUrl, Type = ImageType.Primary }] + }; + var libraryManager = new Mock(); + libraryManager + .Setup(l => l.GetItemList(It.IsAny())) + .Returns([program]); + + var schedulesDirect = new Mock(); + var manager = new RecordingsMetadataManager( + NullLogger.Instance, + config.Object, + libraryManager.Object, + schedulesDirect.Object); + + return (manager, libraryManager, schedulesDirect, recordingPath); + } } diff --git a/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs b/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs index d3b82cefcd..da2e9c53d8 100644 --- a/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs +++ b/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs @@ -2,10 +2,13 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Net.Http; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; +using MediaBrowser.Common.Net; using MediaBrowser.Controller; using MediaBrowser.Controller.BaseItemManager; using MediaBrowser.Controller.Configuration; @@ -17,12 +20,14 @@ using MediaBrowser.Controller.MediaSegments; using MediaBrowser.Controller.Providers; using MediaBrowser.Controller.Subtitles; using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Providers.Manager; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Moq; +using Moq.Protected; using Xunit; // Allow Moq to see internal class @@ -488,6 +493,36 @@ namespace Jellyfin.Providers.Tests.Manager Assert.Equal(ItemCount - 1, processed.Count); } + [Theory] + [InlineData(HttpStatusCode.OK, "application/json", HttpStatusCode.NotFound)] + [InlineData(HttpStatusCode.Forbidden, "application/json", HttpStatusCode.Forbidden)] + public async Task SaveImage_ProviderRefusesWithBody_KeepsTheBody(HttpStatusCode status, string contentType, HttpStatusCode expectedStatus) + { + // Schedules Direct reports an exhausted image quota as HTTP 200 with a JSON body, and + // only the code in that body tells the guide refresh to stop requesting images. + const string Body = "{\"response\":\"MAX_IMAGE_DOWNLOADS\",\"code\":5002}"; + + var messageHandler = new Mock(); + messageHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(() => new HttpResponseMessage(status) + { + Content = new StringContent(Body, Encoding.UTF8, contentType) + }); + + var httpClientFactory = new Mock(); + httpClientFactory.Setup(x => x.CreateClient(It.IsAny())) + .Returns(() => new HttpClient(messageHandler.Object)); + + using var providerManager = GetProviderManager(httpClientFactory: httpClientFactory.Object); + + var ex = await Assert.ThrowsAsync( + () => providerManager.SaveImage(new Movie(), "https://json.schedulesdirect.org/20141201/image/a.jpg", ImageType.Primary, null, CancellationToken.None)); + + Assert.Equal(expectedStatus, ex.StatusCode); + Assert.Equal(Body, ex.ResponseBody); + } + private static void GetMetadataProviders_CanRefreshMetadata_Tester( string providerType, bool expected, @@ -666,7 +701,8 @@ namespace Jellyfin.Providers.Tests.Manager ServerConfiguration? serverConfiguration = null, LibraryOptions? libraryOptions = null, IBaseItemManager? baseItemManager = null, - ILibraryManager? libraryManager = null) + ILibraryManager? libraryManager = null, + IHttpClientFactory? httpClientFactory = null) { var serverConfigurationManager = new Mock(MockBehavior.Strict); serverConfigurationManager.Setup(i => i.Configuration) @@ -683,7 +719,7 @@ namespace Jellyfin.Providers.Tests.Manager } var providerManager = new ProviderManager( - Mock.Of(), + httpClientFactory ?? Mock.Of(), Mock.Of(), serverConfigurationManager.Object, Mock.Of(), diff --git a/tests/Jellyfin.Server.Implementations.Tests/Library/LibraryManagerConvertImageTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Library/LibraryManagerConvertImageTests.cs new file mode 100644 index 0000000000..d374847e54 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Library/LibraryManagerConvertImageTests.cs @@ -0,0 +1,50 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using AutoFixture; +using AutoFixture.AutoMoq; +using Emby.Naming.Common; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Movies; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.Entities; +using Moq; +using Xunit; +using ServerLibraryManager = Emby.Server.Implementations.Library.LibraryManager; + +namespace Jellyfin.Server.Implementations.Tests.Library; + +public class LibraryManagerConvertImageTests +{ + [Theory] + [InlineData(HttpStatusCode.NotFound)] + [InlineData(HttpStatusCode.Forbidden)] + public async Task ConvertImageToLocal_RejectedDownload_KeepsTheResponseAsInnerException(HttpStatusCode status) + { + // The Live TV guide reads the provider's error body off the inner exception to stop + // requesting images from an account that is refusing them. + const string Body = "{\"response\":\"MAX_IMAGE_DOWNLOADS\",\"code\":5002}"; + var rejection = new HttpResponseBodyException("test", status, Body); + + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + fixture.Register(() => new NamingOptions()); + var configuration = fixture.Freeze>(); + configuration.Setup(c => c.Configuration).Returns(new ServerConfiguration()); + configuration.Setup(c => c.ApplicationPaths.ProgramDataPath).Returns("/data"); + var providerManager = fixture.Freeze>(); + providerManager.Setup(p => p.SaveImage(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(rejection); + var manager = fixture.Create(); + + var item = new Movie { Id = Guid.NewGuid(), Name = "Test" }; + var image = new ItemImageInfo { Path = "https://example.com/image.jpg", Type = ImageType.Primary }; + + var ex = await Assert.ThrowsAsync(() => manager.ConvertImageToLocal(item, image, 0, removeOnFailure: false)); + + Assert.Same(rejection, ex.InnerException); + } +} -- cgit v1.2.3