using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.TV;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Providers;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.IO;
using Microsoft.Extensions.Logging;
using TMDbLib.Objects.Search;
namespace MediaBrowser.Providers.Plugins.Tmdb.TV
{
///
/// Creates virtual (metadata-only) entries for missing and unaired episodes.
///
public class TmdbMissingEpisodeProvider : ICustomMetadataProvider, IHasItemChangeMonitor, IHasOrder
{
private readonly TmdbClientManager _tmdbClientManager;
private readonly ILibraryManager _libraryManager;
private readonly IFileSystem _fileSystem;
private readonly IProviderManager _providerManager;
private readonly ILogger _logger;
///
/// Initializes a new instance of the class.
///
/// The .
/// The .
/// The .
/// The .
/// The .
public TmdbMissingEpisodeProvider(
TmdbClientManager tmdbClientManager,
ILibraryManager libraryManager,
IFileSystem fileSystem,
IProviderManager providerManager,
ILogger logger)
{
_tmdbClientManager = tmdbClientManager;
_libraryManager = libraryManager;
_fileSystem = fileSystem;
_providerManager = providerManager;
_logger = logger;
}
///
public string Name => TmdbUtils.ProviderName;
///
// Run after the remote series provider so the TMDb id and other metadata are available.
public int Order => 100;
///
public bool HasChanged(BaseItem item, IDirectoryService directoryService)
{
// Reporting a change makes this provider (and only this provider) run during an otherwise incremental refresh.
if (Plugin.Instance?.Configuration is null)
{
return false;
}
return item is Series series && series.HasProviderId(MetadataProvider.Tmdb);
}
///
public async Task FetchAsync(Series item, MetadataRefreshOptions options, CancellationToken cancellationToken)
{
var configuration = Plugin.Instance?.Configuration;
var importUnaired = (configuration?.ImportUnairedEpisodes).GetValueOrDefault();
var importMissing = (configuration?.ImportMissingEpisodes).GetValueOrDefault();
// The provider is inactive for this series when both global imports are off, or the series'
// library has not been opted in. In either case remove every virtual episode (unaired and
// missing alike) it previously created, so disabling the feature cleans up on the next scan.
if ((!importUnaired && !importMissing) || !IsEnabledForLibrary(item))
{
if (!PruneAllVirtualEpisodes(item))
{
return ItemUpdateType.None;
}
item.Children = null;
return ItemUpdateType.MetadataImport;
}
var tmdbId = item.GetProviderId(MetadataProvider.Tmdb);
if (string.IsNullOrEmpty(tmdbId)
|| !int.TryParse(tmdbId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seriesTmdbId)
|| seriesTmdbId <= 0)
{
return ItemUpdateType.None;
}
var language = item.GetPreferredMetadataLanguage();
var countryCode = item.GetPreferredMetadataCountryCode();
var imageLanguages = TmdbUtils.GetImageLanguagesParam(language, countryCode);
var tmdbSeries = await _tmdbClientManager
.GetSeriesAsync(seriesTmdbId, language, imageLanguages, countryCode, cancellationToken)
.ConfigureAwait(false);
if (tmdbSeries?.Seasons is null)
{
return ItemUpdateType.None;
}
var today = DateTime.UtcNow.Date;
var importSpecials = (configuration?.ImportSpecials).GetValueOrDefault();
var gracePeriodDays = Math.Max(0, (configuration?.UpcomingEpisodeGracePeriodDays).GetValueOrDefault());
// Track every (season, episode) number that already exists (physical or virtual) so we never
// create a duplicate.
// When missing episodes are disabled, this pass also prunes virtual episodes that aired more
// than the grace period ago, as well as any specials when specials are not wanted.
var (existingEpisodes, updatableEpisodes) = GetExistingEpisodes(item, !importMissing, today, gracePeriodDays, importSpecials, out var prunedEpisodes);
var seasonsByNumber = item.GetRecursiveChildren(i => i is Season)
.OfType()
.Where(s => s.IndexNumber.HasValue)
.GroupBy(s => s.IndexNumber!.Value)
.ToDictionary(g => g.Key, g => g.First());
var addedEpisodes = false;
var updatedEpisodes = false;
foreach (var seasonInfo in tmdbSeries.Seasons)
{
cancellationToken.ThrowIfCancellationRequested();
var seasonNumber = seasonInfo.SeasonNumber;
var tmdbSeason = await _tmdbClientManager
.GetSeasonAsync(seriesTmdbId, seasonNumber, language, imageLanguages, countryCode, cancellationToken)
.ConfigureAwait(false);
if (tmdbSeason?.Episodes is null)
{
continue;
}
foreach (var tmdbEpisode in tmdbSeason.Episodes)
{
var episodeNumber = (int)tmdbEpisode.EpisodeNumber;
var premiereDate = GetPremiereDate(tmdbEpisode);
// Skips undated episodes, unaired (upcoming) ones unless upcoming import is enabled,
// already aired ones unless missing import is enabled, and unaired specials entirely.
if (!ShouldImportEpisode(premiereDate, today, importUnaired, importMissing, seasonNumber == 0, importSpecials))
{
continue;
}
var key = (seasonNumber, episodeNumber);
// Already have a virtual episode this provider created, keep metadata in sync with TMDb.
if (updatableEpisodes.TryGetValue(key, out var existingEpisode))
{
var season = await GetOrCreateSeasonAsync(item, seasonNumber, tmdbSeason.Name, seasonsByNumber, cancellationToken).ConfigureAwait(false);
var changed = UpdateVirtualEpisode(existingEpisode, tmdbEpisode, premiereDate);
if (!existingEpisode.ParentId.Equals(season.Id))
{
existingEpisode.SetParent(season);
existingEpisode.SeasonId = season.Id;
existingEpisode.SeasonName = season.Name;
changed = true;
}
if (string.IsNullOrEmpty(existingEpisode.PresentationUniqueKey))
{
existingEpisode.PresentationUniqueKey = existingEpisode.CreatePresentationUniqueKey();
changed = true;
}
if (changed)
{
await existingEpisode.UpdateToRepositoryAsync(ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false);
updatedEpisodes = true;
}
// Backfill the still for placeholders created before images were fetched.
if (await EnsureEpisodeImageAsync(existingEpisode, tmdbEpisode, cancellationToken).ConfigureAwait(false))
{
updatedEpisodes = true;
}
continue;
}
if (!existingEpisodes.Add(key))
{
continue;
}
var targetSeason = await GetOrCreateSeasonAsync(item, seasonNumber, tmdbSeason.Name, seasonsByNumber, cancellationToken).ConfigureAwait(false);
var newEpisode = AddVirtualEpisode(item, targetSeason, tmdbEpisode, premiereDate);
await EnsureEpisodeImageAsync(newEpisode, tmdbEpisode, cancellationToken).ConfigureAwait(false);
addedEpisodes = true;
}
}
var alignedSeasons = await AlignVirtualSeasonSortNamesAsync(seasonsByNumber.Values, cancellationToken).ConfigureAwait(false);
if (!addedEpisodes && !prunedEpisodes && !updatedEpisodes && !alignedSeasons)
{
return ItemUpdateType.None;
}
// Invalidate the cached children so that the season creation / cleanup that runs later in
// SeriesMetadataService.AfterMetadataRefresh observes the newly created (and pruned) episodes.
item.Children = null;
return ItemUpdateType.MetadataImport;
}
///
/// Returns the series' season with the given number, creating (and refreshing) a virtual season
/// when the whole season is missing from the library.
///
private async Task GetOrCreateSeasonAsync(Series series, int seasonNumber, string? seasonName, Dictionary seasonsByNumber, CancellationToken cancellationToken)
{
if (seasonsByNumber.TryGetValue(seasonNumber, out var existingSeason))
{
return existingSeason;
}
_logger.LogInformation("Creating virtual season {SeasonNumber} for series {SeriesName}", seasonNumber, series.Name);
var season = new Season
{
Name = seasonName,
IndexNumber = seasonNumber,
Id = _libraryManager.GetNewItemId(
series.Id.ToString("N", CultureInfo.InvariantCulture) + "Season" + seasonNumber.ToString(CultureInfo.InvariantCulture),
typeof(Season)),
IsVirtualItem = true,
SeriesId = series.Id,
SeriesName = series.Name,
SeriesPresentationUniqueKey = series.GetPresentationUniqueKey()
};
series.AddChild(season);
await season.RefreshMetadata(new MetadataRefreshOptions(new DirectoryService(_fileSystem)), cancellationToken).ConfigureAwait(false);
seasonsByNumber[seasonNumber] = season;
return season;
}
///
/// Mirrors physical seasons' name-based sort convention onto virtual seasons so they interleave by
/// number instead of jumping ahead. See for the details.
///
/// The series' seasons (physical and virtual).
/// The cancellation token.
/// true if any virtual season was updated; otherwise false.
private async Task AlignVirtualSeasonSortNamesAsync(IEnumerable seasons, CancellationToken cancellationToken)
{
var seasonList = seasons.ToList();
var template = BuildSeasonSortNameTemplate(seasonList);
if (template is null)
{
// No physical season sorts by name: virtual seasons already share the bare-index key space.
return false;
}
var updated = false;
foreach (var season in seasonList)
{
if (!season.IsVirtualItem || !season.IndexNumber.HasValue)
{
continue;
}
var desired = template(season.IndexNumber.Value);
if (string.Equals(season.ForcedSortName, desired, StringComparison.Ordinal))
{
continue;
}
_logger.LogInformation(
"Aligning sort name of virtual season {SeasonNumber} in series {SeriesName} to {SortName}",
season.IndexNumber,
season.SeriesName,
desired);
season.ForcedSortName = desired;
await season.UpdateToRepositoryAsync(ItemUpdateType.MetadataEdit, cancellationToken).ConfigureAwait(false);
updated = true;
}
return updated;
}
///
/// Builds a factory that maps a season number to a forced sort name mirroring a physical,
/// name-sorted sibling season, or null when no physical season sorts by name.
///
/// The series' seasons (physical and virtual).
/// A season-number-to-sort-name factory, or null if there is nothing to mirror.
internal static Func? BuildSeasonSortNameTemplate(IEnumerable seasons)
{
// Season.CreateSortName sorts by the bare padded index ("0003"), but season NFOs give physical
// seasons a name-based forced sort ("Season 01" -> "season 0000000001"). The digit-leading key
// sorts ahead of the letter-leading one, so mirror the sibling's token with each season number.
var reference = seasons.FirstOrDefault(s =>
!s.IsVirtualItem && s.IndexNumber.HasValue && !string.IsNullOrEmpty(s.ForcedSortName));
if (reference is null)
{
return null;
}
var forced = reference.ForcedSortName!;
// Locate the last run of digits (the season number) in the sibling's forced sort name.
var end = -1;
var start = -1;
for (var i = forced.Length - 1; i >= 0; i--)
{
if (char.IsDigit(forced[i]))
{
end = end < 0 ? i : end;
start = i;
}
else if (end >= 0)
{
break;
}
}
if (end < 0)
{
// Sibling has no numeric component to swap; leave virtual seasons on the bare-index key.
return null;
}
var prefix = forced[..start];
var suffix = forced[(end + 1)..];
var width = end - start + 1;
// The exact zero-padding is cosmetic: ModifySortChunks pads every digit run to 10 characters,
// so "Season 3" and "Season 03" collapse to the same sort key. Keeping the sibling's width just
// makes the stored value read naturally.
return number => prefix
+ number.ToString(CultureInfo.InvariantCulture).PadLeft(width, '0')
+ suffix;
}
private bool IsEnabledForLibrary(BaseItem item)
{
var enabledLibraries = Plugin.Instance?.Configuration.EnabledMissingEpisodeLibraries;
if (enabledLibraries is null || enabledLibraries.Length == 0)
{
return false;
}
// A series can live under more than one collection folder; opting in any one of them is
// enough. An item that belongs to no collection folder cannot be opted in at all.
return _libraryManager.GetCollectionFolders(item).Any(folder =>
enabledLibraries.Contains(folder.Id.ToString("N", CultureInfo.InvariantCulture), StringComparer.OrdinalIgnoreCase));
}
private (HashSet<(int Season, int Episode)> Keys, Dictionary<(int Season, int Episode), Episode> Updatable) GetExistingEpisodes(Series series, bool pruneAgedOut, DateTime today, int gracePeriodDays, bool importSpecials, out bool pruned)
{
var keys = new HashSet<(int Season, int Episode)>();
var updatable = new Dictionary<(int Season, int Episode), Episode>();
var physicalKeys = new HashSet<(int Season, int Episode)>();
var ourVirtuals = new List<((int Season, int Episode) Key, Episode Episode)>();
pruned = false;
// Enumerate by parent rather than via Series.GetEpisodes: on an initial scan the episodes'
// SeriesPresentationUniqueKey is not set yet, so the presentation-key based query would miss
// them. GetRecursiveChildren walks the actual child tree and sees them regardless.
foreach (var episode in series.GetRecursiveChildren(i => i is Episode).OfType())
{
// The series is refreshed before its episodes during an initial scan, so a freshly
// resolved physical episode may not have its numbers populated yet. Resolve them from
// the path (in memory, mirroring CreateSeasonsAsync) so we can dedupe against episodes
// the user actually has files for instead of creating virtual duplicates.
if (episode.IsFileProtocol && (!episode.ParentIndexNumber.HasValue || !episode.IndexNumber.HasValue))
{
try
{
_libraryManager.FillMissingEpisodeNumbersFromPath(episode, false);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error resolving episode number from path for {Path}", episode.Path);
}
}
// Virtual episodes this provider created are candidates for metadata sync (and pruning).
var isOurs = episode.IsVirtualItem && episode.HasProviderId(MetadataProvider.Tmdb);
if (ShouldPrune(episode, pruneAgedOut, today, gracePeriodDays, importSpecials))
{
DeleteEpisode(episode, "no longer upcoming and missing episodes are disabled");
pruned = true;
continue;
}
if (episode.ParentIndexNumber.HasValue && episode.IndexNumber.HasValue)
{
var key = (episode.ParentIndexNumber.Value, episode.IndexNumber.Value);
keys.Add(key);
// Defer the ours/physical reconciliation: an episode's virtual counterpart and its
// physical file can appear in either order while walking the tree, so we can only
// decide which of our virtual episodes are superseded once every episode is seen.
if (isOurs)
{
ourVirtuals.Add((key, episode));
}
else if (!episode.IsVirtualItem)
{
physicalKeys.Add(key);
}
}
}
// A physical file now exists for one of our placeholders: delete the placeholder here rather
// than updating it (and then leaving RemoveObsoleteEpisodes to delete it moments later). The
// physical key already blocks re-creation via the dedupe set above.
foreach (var (key, episode) in ourVirtuals)
{
if (physicalKeys.Contains(key))
{
DeleteEpisode(episode, "a physical episode now exists for this slot");
pruned = true;
}
else
{
// Virtual episodes this provider created are candidates for metadata sync.
updatable[key] = episode;
}
}
return (keys, updatable);
}
///
/// Removes every virtual episode this provider previously created in the series.
///
/// The series to clean up.
/// true if any episode was removed; otherwise false.
private bool PruneAllVirtualEpisodes(Series series)
{
var pruned = false;
foreach (var episode in series.GetRecursiveChildren(i => i is Episode).OfType())
{
if (episode.IsVirtualItem && episode.HasProviderId(MetadataProvider.Tmdb))
{
DeleteEpisode(episode, "the TMDb missing episode provider is disabled for this library");
pruned = true;
}
}
return pruned;
}
private void DeleteEpisode(Episode episode, string reason)
{
_logger.LogInformation(
"Removing virtual episode S{SeasonNumber}E{EpisodeNumber} in series {SeriesName}: {Reason}",
episode.ParentIndexNumber,
episode.IndexNumber,
episode.SeriesName,
reason);
_libraryManager.DeleteItem(
episode,
new DeleteOptions { DeleteFileLocation = false },
false);
}
///
/// Determines whether a TMDb episode should be imported as a virtual item, based on its air date
/// and the enabled options. Undated episodes are never imported; unaired (today or later) episodes
/// require ; already aired episodes require .
/// Specials (season 0) are only imported when is enabled.
///
/// The episode air date (UTC), or null if unknown.
/// The current UTC date.
/// Whether unaired (upcoming) episodes should be imported.
/// Whether already aired missing episodes should be imported.
/// Whether the episode belongs to the specials season (season 0).
/// Whether specials should be included.
/// true if the episode should be imported; otherwise false.
internal static bool ShouldImportEpisode(DateTime? premiereDate, DateTime today, bool importUnaired, bool importMissing, bool isSpecial, bool importSpecials)
{
if (!premiereDate.HasValue)
{
return false;
}
// Specials are only imported when the user opts in.
if (isSpecial && !importSpecials)
{
return false;
}
var isUnaired = premiereDate.Value.Date >= today;
return isUnaired ? importUnaired : importMissing;
}
///
/// Determines whether an existing virtual episode created by this provider (carries a TMDb id)
/// should be pruned. Specials are removed entirely unless is
/// enabled. Otherwise, when missing episodes are not wanted, an entry is pruned once its air date
/// is more than in the past; the grace period keeps recently
/// aired episodes in place to allow for the delay between an episode airing and its file being
/// added to the library.
///
/// The episode to evaluate.
/// Whether aged-out virtual episodes should be pruned (missing import disabled).
/// The current UTC date.
/// The number of days an aired episode is retained before pruning.
/// Whether specials should be kept.
/// true if the episode should be pruned; otherwise false.
internal static bool ShouldPrune(Episode episode, bool pruneAgedOut, DateTime today, int gracePeriodDays, bool importSpecials)
{
if (!episode.IsVirtualItem || !episode.HasProviderId(MetadataProvider.Tmdb))
{
return false;
}
// Specials are removed entirely unless the user opts in.
if (episode.ParentIndexNumber == 0 && !importSpecials)
{
return true;
}
// When missing episodes are not wanted, prune placeholders for episodes that aired more than
// the grace period ago.
return pruneAgedOut
&& episode.PremiereDate.HasValue
&& episode.PremiereDate.Value.Date < today.AddDays(-gracePeriodDays);
}
internal static DateTime? GetPremiereDate(TvSeasonEpisode tmdbEpisode)
{
return tmdbEpisode.AirDate.HasValue
? DateTime.SpecifyKind(tmdbEpisode.AirDate.Value, DateTimeKind.Local).ToUniversalTime()
: null;
}
internal static bool UpdateVirtualEpisode(Episode episode, TvSeasonEpisode tmdbEpisode, DateTime? premiereDate)
{
var changed = false;
if (!string.IsNullOrEmpty(tmdbEpisode.Name) && !string.Equals(episode.Name, tmdbEpisode.Name, StringComparison.Ordinal))
{
episode.Name = tmdbEpisode.Name;
changed = true;
}
if (!string.IsNullOrEmpty(tmdbEpisode.Overview) && !string.Equals(episode.Overview, tmdbEpisode.Overview, StringComparison.Ordinal))
{
episode.Overview = tmdbEpisode.Overview;
changed = true;
}
if (premiereDate.HasValue && episode.PremiereDate != premiereDate)
{
episode.PremiereDate = premiereDate;
episode.ProductionYear = tmdbEpisode.AirDate?.Year;
changed = true;
}
return changed;
}
private Episode AddVirtualEpisode(Series series, Season season, TvSeasonEpisode tmdbEpisode, DateTime? premiereDate)
{
var seasonNumber = season.IndexNumber.GetValueOrDefault();
var episodeNumber = (int)tmdbEpisode.EpisodeNumber;
// Leaving Path unset makes the item a virtual (metadata-only) episode.
var episode = new Episode
{
Name = tmdbEpisode.Name,
IndexNumber = episodeNumber,
ParentIndexNumber = seasonNumber,
Id = _libraryManager.GetNewItemId(
series.Id.ToString("N", CultureInfo.InvariantCulture)
+ "Season" + seasonNumber.ToString(CultureInfo.InvariantCulture)
+ "Episode" + episodeNumber.ToString(CultureInfo.InvariantCulture),
typeof(Episode)),
IsVirtualItem = true,
PremiereDate = premiereDate,
ProductionYear = tmdbEpisode.AirDate?.Year,
Overview = tmdbEpisode.Overview,
SeasonId = season.Id,
SeasonName = season.Name,
SeriesId = series.Id,
SeriesName = series.Name,
SeriesPresentationUniqueKey = series.GetPresentationUniqueKey()
};
episode.PresentationUniqueKey = episode.CreatePresentationUniqueKey();
if (tmdbEpisode.Id > 0)
{
episode.SetProviderId(MetadataProvider.Tmdb, tmdbEpisode.Id.ToString(CultureInfo.InvariantCulture));
}
_logger.LogInformation(
"Creating virtual episode S{SeasonNumber}E{EpisodeNumber} for series {SeriesName}",
seasonNumber,
episodeNumber,
series.Name);
season.AddChild(episode);
return episode;
}
///
/// Downloads the TMDb still for a virtual episode that has no image yet, so it does not fall back
/// to the season/series image.
///
/// The virtual episode.
/// The matching TMDb episode.
/// The cancellation token.
/// true if a still was downloaded and saved; otherwise false.
private async Task EnsureEpisodeImageAsync(Episode episode, TvSeasonEpisode tmdbEpisode, CancellationToken cancellationToken)
{
// The still ships with the season episode list, so use it directly instead of a per-episode lookup.
if (episode.HasImage(ImageType.Primary, 0) || string.IsNullOrEmpty(tmdbEpisode.StillPath))
{
return false;
}
var stillUrl = _tmdbClientManager.GetStillUrl(tmdbEpisode.StillPath);
if (string.IsNullOrEmpty(stillUrl))
{
return false;
}
try
{
// SaveImage sets the image path on the item but does not persist it, so save afterwards.
await _providerManager.SaveImage(episode, stillUrl, ImageType.Primary, null, cancellationToken).ConfigureAwait(false);
await episode.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, cancellationToken).ConfigureAwait(false);
return true;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Error downloading still for virtual episode S{SeasonNumber}E{EpisodeNumber} of {SeriesName}",
episode.ParentIndexNumber,
episode.IndexNumber,
episode.SeriesName);
return false;
}
}
}
}