diff options
Diffstat (limited to 'Emby.Server.Implementations/Library')
7 files changed, 364 insertions, 131 deletions
diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index dd8c883684..e6fa94fbef 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -15,7 +16,6 @@ using Emby.Naming.Common; using Emby.Naming.TV; using Emby.Naming.Video; using Emby.Server.Implementations.Library.Resolvers; -using Emby.Server.Implementations.Library.Validators; using Emby.Server.Implementations.Playlists; using Emby.Server.Implementations.ScheduledTasks.Tasks; using Emby.Server.Implementations.Sorting; @@ -35,7 +35,6 @@ using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; -using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Playlists; using MediaBrowser.Controller.Providers; @@ -75,7 +74,6 @@ namespace Emby.Server.Implementations.Library private readonly Lazy<IProviderManager> _providerManagerFactory; private readonly Lazy<IUserViewManager> _userViewManagerFactory; private readonly IServerApplicationHost _appHost; - private readonly IMediaEncoder _mediaEncoder; private readonly IFileSystem _fileSystem; private readonly IItemRepository _itemRepository; private readonly IItemPersistenceService _persistenceService; @@ -88,6 +86,7 @@ namespace Emby.Server.Implementations.Library private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; private readonly ILocalizationManager _localization; + private readonly IDirectoryService _directoryService; private readonly FastConcurrentLru<Guid, BaseItem> _cache; private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; private readonly IMediaStreamRepository _mediaStreamRepository; @@ -122,7 +121,6 @@ namespace Emby.Server.Implementations.Library /// <param name="fileSystem">The file system.</param> /// <param name="providerManagerFactory">The provider manager.</param> /// <param name="userViewManagerFactory">The user view manager.</param> - /// <param name="mediaEncoder">The media encoder.</param> /// <param name="itemRepository">The item repository.</param> /// <param name="persistenceService">The item persistence service.</param> /// <param name="nextUpService">The next up service.</param> @@ -148,7 +146,6 @@ namespace Emby.Server.Implementations.Library IFileSystem fileSystem, Lazy<IProviderManager> providerManagerFactory, Lazy<IUserViewManager> userViewManagerFactory, - IMediaEncoder mediaEncoder, IItemRepository itemRepository, IItemPersistenceService persistenceService, INextUpService nextUpService, @@ -174,7 +171,6 @@ namespace Emby.Server.Implementations.Library _fileSystem = fileSystem; _providerManagerFactory = providerManagerFactory; _userViewManagerFactory = userViewManagerFactory; - _mediaEncoder = mediaEncoder; _itemRepository = itemRepository; _persistenceService = persistenceService; _nextUpService = nextUpService; @@ -189,6 +185,7 @@ namespace Emby.Server.Implementations.Library _pathManager = pathManager; _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; _localization = localization; + _directoryService = directoryService; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -1210,6 +1207,12 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc /> + public Guid GetPersonId(string name) + { + return GetItemByNameId<Person>(Person.GetPath(name)); + } + + /// <inheritdoc /> public Person? GetPerson(string name) { var path = Person.GetPath(name); @@ -1222,6 +1225,33 @@ namespace Emby.Server.Implementations.Library return null; } + /// <inheritdoc /> + public Person GetOrCreatePerson(string name) + { + var existing = GetPerson(name); + if (existing is not null) + { + return existing; + } + + var path = Person.GetPath(name); + var info = Directory.CreateDirectory(path); + var item = new Person + { + Name = name, + Id = GetItemByNameId<Person>(path), + DateCreated = info.CreationTimeUtc, + DateModified = info.LastWriteTimeUtc, + Path = path + }; + + item.PresentationUniqueKey = item.CreatePresentationUniqueKey(); + + CreateItem(item, null); + + return item; + } + /// <summary> /// Gets the studio. /// </summary> @@ -1354,15 +1384,6 @@ namespace Emby.Server.Implementations.Library return GetNewItemIdInternal(path, typeof(T), forceCaseInsensitiveId); } - /// <inheritdoc /> - public Task ValidatePeopleAsync(IProgress<double> progress, CancellationToken cancellationToken) - { - // Ensure the location is available. - Directory.CreateDirectory(_configurationManager.ApplicationPaths.PeoplePath); - - return new PeopleValidator(this, _logger, _fileSystem).ValidatePeople(cancellationToken, progress); - } - /// <summary> /// Reloads the root media folder. /// </summary> @@ -1489,6 +1510,10 @@ namespace Emby.Server.Implementations.Library var numComplete = 0; var numTasks = tasks.Count; + _logger.LogInformation("Running {TaskCount} post-scan task(s)", numTasks); + + var phaseStart = Stopwatch.GetTimestamp(); + foreach (var task in tasks) { // Prevent access to modified closure @@ -1506,20 +1531,45 @@ namespace Emby.Server.Implementations.Library progress.Report(innerPercent); }); - _logger.LogDebug("Running post-scan task {0}", task.GetType().Name); + var taskName = task.GetType().Name; + var taskStart = Stopwatch.GetTimestamp(); + + _logger.LogInformation( + "Running post-scan task {TaskNumber}/{TaskCount}: {TaskName}", + currentNumComplete + 1, + numTasks, + taskName); try { await task.Run(innerProgress, cancellationToken).ConfigureAwait(false); + + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} completed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } catch (OperationCanceledException) { - _logger.LogInformation("Post-scan task cancelled: {0}", task.GetType().Name); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} cancelled after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); throw; } catch (Exception ex) { - _logger.LogError(ex, "Error running post-scan task"); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogError( + ex, + "Post-scan task {TaskName} failed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } numComplete++; @@ -1528,6 +1578,12 @@ namespace Emby.Server.Implementations.Library progress.Report(percent * 100); } + var phaseElapsed = Stopwatch.GetElapsedTime(phaseStart); + _logger.LogInformation( + "All post-scan tasks completed after {Minutes} minute(s) and {Seconds} seconds", + Math.Truncate(phaseElapsed.TotalMinutes), + phaseElapsed.Seconds); + _persistenceService.UpdateInheritedValues(); progress.Report(100); @@ -1745,9 +1801,9 @@ namespace Emby.Server.Implementations.Library return _countService.GetItemCountsForNameItem(kind, id, relatedItemKinds, query); } - public Dictionary<Guid, int> GetChildCountBatch(IReadOnlyList<Guid> parentIds, Guid? userId) + public Dictionary<Guid, int> GetChildCountBatch(IReadOnlyList<Guid> parentIds, User? user) { - return _countService.GetChildCountBatch(parentIds, userId); + return _countService.GetChildCountBatch(parentIds, user); } /// <inheritdoc/> @@ -1984,18 +2040,10 @@ namespace Emby.Server.Implementations.Library { // Playlists and BoxSets store their contents in LinkedChildren and never // populate AncestorIds for those items, so a recursive AncestorIds query - // would return zero rows. Resolve to the linked child IDs up front and - // route through the existing indexed ItemIds filter. - query.ItemIds = folder.LinkedChildren - .Where(lc => lc.ItemId.HasValue && !lc.ItemId.Value.IsEmpty()) - .Select(lc => lc.ItemId!.Value) - .ToArray(); - - // Empty linked-children should still return empty rather than scanning everything. - if (query.ItemIds.Length == 0) - { - query.ItemIds = [Guid.NewGuid()]; - } + // would return zero rows. Filter by the descendant set instead, which follows + // the links and keeps descending, so a linked folder contributes what is below + // it as well - the episodes of a Series added to a collection, for example. + query.DescendantOfId = folder.Id; } else { @@ -3728,6 +3776,10 @@ namespace Emby.Server.Implementations.Library AddMediaPathInternal(name, path, false); } } + + // The libraries root was listed before this folder existed, so drop that listing: + // anything still reading it resolves the library set without the new folder. + _directoryService.Invalidate(virtualFolderPath); } finally { @@ -3754,27 +3806,14 @@ namespace Emby.Server.Implementations.Library var itemUpdateType = ItemUpdateType.MetadataDownload; var saveEntity = false; - var createEntity = false; var personEntity = GetPerson(person.Name); if (personEntity is null) { try { - var path = Person.GetPath(person.Name); - var info = Directory.CreateDirectory(path); - personEntity = new Person() - { - Name = person.Name, - Id = GetItemByNameId<Person>(path), - DateCreated = info.CreationTimeUtc, - DateModified = info.LastWriteTimeUtc, - Path = path - }; - - personEntity.PresentationUniqueKey = personEntity.CreatePresentationUniqueKey(); + personEntity = GetOrCreatePerson(person.Name); saveEntity = true; - createEntity = true; } catch (Exception ex) { @@ -3808,11 +3847,6 @@ namespace Emby.Server.Implementations.Library if (saveEntity) { - if (createEntity) - { - CreateItems([personEntity], null, CancellationToken.None); - } - await RunMetadataSavers(personEntity, itemUpdateType).ConfigureAwait(false); personEntity.DateLastSaved = DateTime.UtcNow; @@ -3852,7 +3886,9 @@ namespace Emby.Server.Implementations.Library } var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName) + ?? throw new FileNotFoundException( + string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); CreateShortcut(virtualFolderPath, pathInfo); @@ -3873,7 +3909,9 @@ namespace Emby.Server.Implementations.Library ArgumentNullException.ThrowIfNull(mediaPath); var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName) + ?? throw new FileNotFoundException( + string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); var libraryOptions = CollectionFolder.GetLibraryOptions(virtualFolderPath); @@ -3912,9 +3950,9 @@ namespace Emby.Server.Implementations.Library var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var path = Path.Combine(rootFolderPath, name); + var path = FileSystemHelper.GetChildPath(rootFolderPath, name); - if (!Directory.Exists(path)) + if (path is null || !Directory.Exists(path)) { throw new FileNotFoundException("The media folder does not exist"); } @@ -3924,6 +3962,7 @@ namespace Emby.Server.Implementations.Library try { Directory.Delete(path, true); + _directoryService.Invalidate(path); } finally { @@ -3978,9 +4017,9 @@ namespace Emby.Server.Implementations.Library ArgumentException.ThrowIfNullOrEmpty(mediaPath); var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName); - if (!Directory.Exists(virtualFolderPath)) + if (virtualFolderPath is null || !Directory.Exists(virtualFolderPath)) { throw new FileNotFoundException( string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); @@ -3993,6 +4032,7 @@ namespace Emby.Server.Implementations.Library if (!string.IsNullOrEmpty(shortcut)) { _fileSystem.DeleteFile(shortcut); + _directoryService.Invalidate(shortcut); } var libraryOptions = CollectionFolder.GetLibraryOptions(virtualFolderPath); @@ -4036,6 +4076,7 @@ namespace Emby.Server.Implementations.Library } _fileSystem.CreateShortcut(lnk, _appHost.ReverseVirtualPath(path)); + _directoryService.Invalidate(lnk); RemoveContentTypeOverrides(path); } diff --git a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs index 6624d0125f..a8bd832cc8 100644 --- a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs +++ b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs @@ -129,6 +129,17 @@ namespace Emby.Server.Implementations.Library.Resolvers.TV var tmdbId = justName.GetAttributeValue("tmdbid"); item.TrySetProviderId(MetadataProvider.Tmdb, tmdbId); + + // Anime databases model a single cour as its own entry, so a multi-season + // series maps to one of these ids per season rather than one per series. + var anidbId = justName.GetAttributeValue("anidbid"); + item.TrySetProviderId("AniDB", anidbId); + + var aniListId = justName.GetAttributeValue("anilistid"); + item.TrySetProviderId("AniList", aniListId); + + var aniSearchId = justName.GetAttributeValue("anisearchid"); + item.TrySetProviderId("AniSearch", aniSearchId); } } } diff --git a/Emby.Server.Implementations/Library/Search/SearchManager.cs b/Emby.Server.Implementations/Library/Search/SearchManager.cs index 0e180753a6..306a8673d5 100644 --- a/Emby.Server.Implementations/Library/Search/SearchManager.cs +++ b/Emby.Server.Implementations/Library/Search/SearchManager.cs @@ -112,13 +112,12 @@ public class SearchManager : ISearchManager return externalResults; } - var internalResults = await internalTask.ConfigureAwait(false); if (_internalProviders.Length > 0) { _logger.LogDebug("No results from external providers, using internal provider results"); } - return internalResults; + return await internalTask.ConfigureAwait(false); } private async Task<IReadOnlyList<SearchResult>> FilterByUserAccessAsync( @@ -144,17 +143,16 @@ public class SearchManager : ISearchManager baseQuery = _queryHelpers.ApplyAccessFiltering(dbContext, baseQuery, accessFilter); - var allowedCount = await baseQuery.CountAsync(cancellationToken).ConfigureAwait(false); - if (allowedCount == candidates.Count) - { - return candidates; - } - var allowedIds = await baseQuery .Select(e => e.Id) .ToHashSetAsync(cancellationToken) .ConfigureAwait(false); + if (allowedIds.Count == candidates.Count) + { + return candidates; + } + var filtered = candidates.Where(c => allowedIds.Contains(c.ItemId)).ToList(); if (filtered.Count < candidates.Count) { diff --git a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs new file mode 100644 index 0000000000..75aea0eab6 --- /dev/null +++ b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs @@ -0,0 +1,42 @@ +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations.Entities; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; + +namespace Emby.Server.Implementations.Library.SimilarItems; + +/// <summary> +/// Builds the access filter that decides which items a similar-items lookup may return for a user. +/// </summary> +internal static class SimilarItemsAccessFilter +{ + private static readonly BaseItemKind[] _itemByNameKinds = + [ + BaseItemKind.Person, + BaseItemKind.Genre, + BaseItemKind.MusicGenre, + BaseItemKind.MusicArtist, + BaseItemKind.Studio + ]; + + /// <summary> + /// Builds an access filter carrying the user's library access and parental restrictions. + /// </summary> + /// <param name="user">The user the lookup runs for.</param> + /// <param name="libraryManager">The library manager.</param> + /// <returns>The access filter.</returns> + public static InternalItemsQuery Build(User user, ILibraryManager libraryManager) + { + // IncludeItemTypes is read only for the by-name exemption here; the caller applies this + // filter through ApplyAccessFiltering, which does not translate it into a type restriction. + var accessFilter = new InternalItemsQuery(user) + { + IncludeItemTypes = _itemByNameKinds + }; + + // ConfigureUserAccess populates TopParentIds for the libraries the user may open. + libraryManager.ConfigureUserAccess(accessFilter, user); + + return accessFilter; + } +} diff --git a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs index 4e482c174a..fd5f292ebe 100644 --- a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs +++ b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs @@ -7,8 +7,10 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Database.Implementations.Enums; +using Jellyfin.Extensions; using Jellyfin.Extensions.Json; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller; @@ -16,11 +18,13 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Persistence; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Model.Querying; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library.SimilarItems; @@ -35,6 +39,8 @@ public class SimilarItemsManager : ISimilarItemsManager private readonly ILibraryManager _libraryManager; private readonly IFileSystem _fileSystem; private readonly IServerConfigurationManager _serverConfigurationManager; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IItemQueryHelpers _queryHelpers; private ISimilarItemsProvider[] _similarItemsProviders = []; /// <summary> @@ -45,18 +51,24 @@ public class SimilarItemsManager : ISimilarItemsManager /// <param name="libraryManager">The library manager.</param> /// <param name="fileSystem">The file system.</param> /// <param name="serverConfigurationManager">The server configuration manager.</param> + /// <param name="dbProvider">The database context factory.</param> + /// <param name="queryHelpers">The shared item query helpers.</param> public SimilarItemsManager( ILogger<SimilarItemsManager> logger, IServerApplicationPaths appPaths, ILibraryManager libraryManager, IFileSystem fileSystem, - IServerConfigurationManager serverConfigurationManager) + IServerConfigurationManager serverConfigurationManager, + IDbContextFactory<JellyfinDbContext> dbProvider, + IItemQueryHelpers queryHelpers) { _logger = logger; _appPaths = appPaths; _libraryManager = libraryManager; _fileSystem = fileSystem; _serverConfigurationManager = serverConfigurationManager; + _dbProvider = dbProvider; + _queryHelpers = queryHelpers; } /// <inheritdoc/> @@ -230,11 +242,64 @@ public class SimilarItemsManager : ISimilarItemsManager } } - return allResults + var ordered = allResults .OrderByDescending(x => x.Score) .Select(x => x.Item) .Take(requestedLimit) .ToList(); + + return await FilterByLibraryAccessAsync(ordered, user, cancellationToken).ConfigureAwait(false); + } + + private async Task<IReadOnlyList<BaseItem>> FilterByLibraryAccessAsync( + IReadOnlyList<BaseItem> candidates, + User? user, + CancellationToken cancellationToken) + { + if (candidates.Count == 0 || user is null) + { + return candidates; + } + + var accessFilter = SimilarItemsAccessFilter.Build(user, _libraryManager); + + // No accessible libraries means nothing to compare against, and an empty TopParentIds set + // would disable the filter rather than reject everything. + if (accessFilter.TopParentIds.Length == 0) + { + return candidates; + } + + Guid[] candidateIds = [.. candidates.Select(c => c.Id)]; + + var dbContext = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var baseQuery = dbContext.BaseItems + .AsNoTracking() + .WhereOneOrMany(candidateIds, e => e.Id); + + baseQuery = _queryHelpers.ApplyAccessFiltering(dbContext, baseQuery, accessFilter); + + var allowedCount = await baseQuery.CountAsync(cancellationToken).ConfigureAwait(false); + if (allowedCount == candidates.Count) + { + return candidates; + } + + var allowedIds = await baseQuery + .Select(e => e.Id) + .ToHashSetAsync(cancellationToken) + .ConfigureAwait(false); + + var filtered = candidates.Where(c => allowedIds.Contains(c.Id)).ToList(); + _logger.LogDebug( + "Dropped {Dropped} of {Total} similar-item candidates due to user access filtering", + candidates.Count - filtered.Count, + candidates.Count); + + return filtered; + } } /// <inheritdoc/> @@ -376,19 +441,39 @@ public class SimilarItemsManager : ISimilarItemsManager var batchResults = await batchProvider.GetBatchSimilarItemsAsync(baselineItems, query, cancellationToken).ConfigureAwait(false); + // Filter once across every category rather than per baseline, so a batch provider costs one + // access query no matter how many categories it produced. + var allItems = batchResults.Values.SelectMany(items => items).DistinctBy(item => item.Id).ToList(); + var allowed = await FilterByLibraryAccessAsync(allItems, query.User, cancellationToken).ConfigureAwait(false); + + HashSet<Guid>? allowedIds = allowed.Count == allItems.Count + ? null + : [.. allowed.Select(item => item.Id)]; + var recommendations = new List<SimilarItemsRecommendation>(baselineItems.Count); foreach (var baseline in baselineItems) { - if (batchResults.TryGetValue(baseline.Id, out var similar) && similar.Count > 0) + if (!batchResults.TryGetValue(baseline.Id, out var similar) || similar.Count == 0) + { + continue; + } + + if (allowedIds is not null) { - recommendations.Add(new SimilarItemsRecommendation + similar = similar.Where(item => allowedIds.Contains(item.Id)).ToList(); + if (similar.Count == 0) { - BaselineItemName = baseline.Name, - CategoryId = baseline.Id, - RecommendationType = recommendationType, - Items = similar - }); + continue; + } } + + recommendations.Add(new SimilarItemsRecommendation + { + BaselineItemName = baseline.Name, + CategoryId = baseline.Id, + RecommendationType = recommendationType, + Items = similar + }); } return recommendations; diff --git a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs index fa7112eb90..690466be70 100644 --- a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Threading; @@ -61,6 +62,9 @@ public class ArtistsValidator var count = names.Count; var refreshed = 0; + var liveIds = new HashSet<Guid>(); + var unresolved = 0; + foreach (var name in names) { try @@ -73,13 +77,20 @@ public class ArtistsValidator // Fall back to GetArtist if not found (creates new item if needed) item ??= _libraryManager.GetArtist(name); - var isNew = !existingArtistIds.Contains(item.Id); - var neverRefreshed = item.DateLastRefreshed == default; - if (isNew || neverRefreshed) + // A name with no item is nothing to refresh, and nothing to keep alive either. + if (item is not null) { - await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); - refreshed++; + liveIds.Add(item.Id); + + var isNew = !existingArtistIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; + + if (isNew || neverRefreshed) + { + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } } catch (OperationCanceledException) @@ -88,6 +99,7 @@ public class ArtistsValidator } catch (Exception ex) { + unresolved++; _logger.LogError(ex, "Error refreshing {ArtistName}", name); } @@ -101,13 +113,26 @@ public class ArtistsValidator _logger.LogInformation("Refreshed metadata for {RefreshedCount} new artists out of {TotalCount} total", refreshed, count); + // Every name that threw is a name whose artist is missing from the live set, and deleting against + // a live set with holes in it deletes artists the library still refers to. Leave the sweep to a + // run that got a clean read of them. + if (unresolved > 0) + { + _logger.LogWarning( + "Not removing dead artists: {Count} of {TotalCount} names could not be resolved this run", + unresolved, + count); + + progress.Report(100); + return; + } + var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery { IncludeItemTypes = [BaseItemKind.MusicArtist], - IsDeadArtist = true, IsLocked = false - }).Cast<MusicArtist>() - .Where(item => item.IsAccessedByName) + }).OfType<MusicArtist>() + .Where(item => item.IsAccessedByName && !liveIds.Contains(item.Id)) .ToList(); foreach (var item in deadEntities) diff --git a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs index 078a0b921d..7d53f40ce7 100644 --- a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs @@ -1,12 +1,12 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.IO; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library.Validators; @@ -17,112 +17,143 @@ namespace Emby.Server.Implementations.Library.Validators; public class PeopleValidator { /// <summary> - /// The _library manager. + /// The library manager. /// </summary> private readonly ILibraryManager _libraryManager; /// <summary> - /// The _logger. + /// The logger. /// </summary> - private readonly ILogger _logger; - - private readonly IFileSystem _fileSystem; + private readonly ILogger<PeopleValidator> _logger; /// <summary> /// Initializes a new instance of the <see cref="PeopleValidator" /> class. /// </summary> /// <param name="libraryManager">The library manager.</param> /// <param name="logger">The logger.</param> - /// <param name="fileSystem">The file system.</param> - public PeopleValidator(ILibraryManager libraryManager, ILogger logger, IFileSystem fileSystem) + public PeopleValidator(ILibraryManager libraryManager, ILogger<PeopleValidator> logger) { _libraryManager = libraryManager; _logger = logger; - _fileSystem = fileSystem; } /// <summary> /// Validates the people. /// </summary> - /// <param name="cancellationToken">The cancellation token.</param> /// <param name="progress">The progress.</param> + /// <param name="cancellationToken">The cancellation token.</param> /// <returns>Task.</returns> - public async Task ValidatePeople(CancellationToken cancellationToken, IProgress<double> progress) + public async Task Run(IProgress<double> progress, CancellationToken cancellationToken) { // Before the refresh below walks them: a credit no item maps to any more stands for nothing, // and while it is there the person it names cannot reach the dead-person sweep either. var numOrphaned = _libraryManager.DeleteOrphanedCredits(); if (numOrphaned > 0) { - _logger.LogDebug("Deleted {Amount} credits no item maps to", numOrphaned); + _logger.LogInformation("Deleted {Amount} credits no item maps to", numOrphaned); } - var people = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); - - var numComplete = 0; - - var numPeople = people.Count; + var names = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); + var existingPersonIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + IncludeItemTypes = [BaseItemKind.Person] + }).ToHashSet(); - IProgress<double> subProgress = new Progress<double>((val) => progress.Report(val / 2)); + var (newNames, deadIds) = PartitionCreditsByPersonId(names, _libraryManager.GetPersonId, existingPersonIds); - _logger.LogDebug("Will refresh {Amount} people", numPeople); + var numComplete = 0; + var count = names.Count; + var refreshed = 0; - foreach (var person in people) + foreach (var name in names) { cancellationToken.ThrowIfCancellationRequested(); try { - var item = _libraryManager.GetPerson(person); - if (item is null) - { - _logger.LogWarning("Failed to get person: {Name}", person); - continue; - } + var item = _libraryManager.GetOrCreatePerson(name); + var isNew = !existingPersonIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; - var options = new MetadataRefreshOptions(new DirectoryService(_fileSystem)) + if (isNew || neverRefreshed) { - ImageRefreshMode = MetadataRefreshMode.ValidationOnly, - MetadataRefreshMode = MetadataRefreshMode.ValidationOnly - }; - - await item.RefreshMetadata(options, cancellationToken).ConfigureAwait(false); + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } catch (OperationCanceledException) { + // Don't clutter the log throw; } catch (Exception ex) { - _logger.LogError(ex, "Error validating IBN entry {Person}", person); + _logger.LogError(ex, "Error refreshing {PersonName}", name); } - // Update progress numComplete++; double percent = numComplete; - percent /= numPeople; + percent /= count; + percent *= 100; - subProgress.Report(100 * percent); + progress.Report(percent); } - var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery - { - IncludeItemTypes = [BaseItemKind.Person], - IsDeadPerson = true, - IsLocked = false - }); + _logger.LogInformation( + "Refreshed metadata for {RefreshedCount} people out of {TotalCount} total, {NewCount} of which had no item yet", + refreshed, + count, + newNames.Count); - subProgress = new Progress<double>((val) => progress.Report((val / 2) + 50)); + // A person somebody locked is theirs, not ours, however little the library still credits them. + var deadEntities = deadIds + .Select(_libraryManager.GetItemById) + .OfType<Person>() + .Where(item => !item.IsLocked) + .ToList(); - var i = 0; - foreach (var item in deadEntities.Chunk(500)) + foreach (var item in deadEntities) { - _libraryManager.DeleteItemsUnsafeFast(item, true); - subProgress.Report(100f / deadEntities.Count * (i++ * 100)); + _logger.LogInformation("Deleting dead {ItemType} {ItemId} {ItemName}", item.GetType().Name, item.Id.ToString("N", CultureInfo.InvariantCulture), item.Name); } + _libraryManager.DeleteItemsUnsafeFast(deadEntities, deleteSourceFiles: true); + progress.Report(100); + } + + /// <summary> + /// Splits the person items into the ones a credit still calls for and the ones nothing does. + /// </summary> + /// <param name="creditNames">Every name credited on an item, from the people table.</param> + /// <param name="getPersonId">Maps a credit name to the id its person item has.</param> + /// <param name="existingPersonIds">The ids of the person items that exist.</param> + /// <returns>The credits needing an item, and the ids of the items nothing credits.</returns> + internal static (List<string> NewNames, List<Guid> DeadIds) PartitionCreditsByPersonId( + IReadOnlyList<string> creditNames, + Func<string, Guid> getPersonId, + IReadOnlySet<Guid> existingPersonIds) + { + ArgumentNullException.ThrowIfNull(creditNames); + ArgumentNullException.ThrowIfNull(getPersonId); + ArgumentNullException.ThrowIfNull(existingPersonIds); + + var newNames = new List<string>(); + var liveIds = new HashSet<Guid>(); + + foreach (var name in creditNames) + { + var personId = getPersonId(name); + + // Distinct credit names can normalize onto one id; only the first of them needs an item. + if (liveIds.Add(personId) && !existingPersonIds.Contains(personId)) + { + newNames.Add(name); + } + } + + var deadIds = existingPersonIds.Where(id => !liveIds.Contains(id)).ToList(); - _logger.LogInformation("People validation complete, deleted {Orphaned} orphaned credits", numOrphaned); + return (newNames, deadIds); } } |