diff options
93 files changed, 3958 insertions, 503 deletions
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 99f24e3a14..a97d335170 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -240,6 +240,7 @@ - [Florin-Popescu](https://github.com/Florin-Popescu) - [m0g3r](https://github.com/m0g3r) - [martin-77](https://github.com/martin-77) + - [Oggeb1](https://github.com/Oggeb1) # Emby Contributors diff --git a/Emby.Naming/ExternalFiles/ExternalPathParser.cs b/Emby.Naming/ExternalFiles/ExternalPathParser.cs index 8e7da5db42..1f16161282 100644 --- a/Emby.Naming/ExternalFiles/ExternalPathParser.cs +++ b/Emby.Naming/ExternalFiles/ExternalPathParser.cs @@ -44,7 +44,14 @@ namespace Emby.Naming.ExternalFiles } var extension = Path.GetExtension(path.AsSpan()); - if (!(_type == DlnaProfileType.Subtitle && _namingOptions.SubtitleFileExtensions.Contains(extension, StringComparison.OrdinalIgnoreCase)) + + // .idx carries VobSub per-track language metadata. Recognize it here rather + // than adding it to NamingOptions.SubtitleFileExtensions, which also gates + // subtitle uploads/saves. + var isVobSubIndex = _type == DlnaProfileType.Subtitle && extension.Equals(".idx", StringComparison.OrdinalIgnoreCase); + + if (!isVobSubIndex + && !(_type == DlnaProfileType.Subtitle && _namingOptions.SubtitleFileExtensions.Contains(extension, StringComparison.OrdinalIgnoreCase)) && !(_type == DlnaProfileType.Audio && _namingOptions.AudioFileExtensions.Contains(extension, StringComparison.OrdinalIgnoreCase)) && !(_type == DlnaProfileType.Lyric && _namingOptions.LyricFileExtensions.Contains(extension, StringComparison.OrdinalIgnoreCase))) { diff --git a/Emby.Naming/TV/EpisodePathParser.cs b/Emby.Naming/TV/EpisodePathParser.cs index 0c737964b4..f06aa909ba 100644 --- a/Emby.Naming/TV/EpisodePathParser.cs +++ b/Emby.Naming/TV/EpisodePathParser.cs @@ -158,7 +158,9 @@ namespace Emby.Naming.TV if (nextIndex >= name.Length || !"0123456789iIpP".Contains(name[nextIndex], StringComparison.Ordinal)) { - if (int.TryParse(endingNumberGroup.ValueSpan, NumberStyles.Integer, CultureInfo.InvariantCulture, out num)) + // A range cannot end before it starts, so a lower number belongs to the episode title rather than to a range. + if (int.TryParse(endingNumberGroup.ValueSpan, NumberStyles.Integer, CultureInfo.InvariantCulture, out num) + && num >= result.EpisodeNumber) { result.EndingEpisodeNumber = num; } @@ -226,7 +228,7 @@ namespace Emby.Naming.TV info.SeriesName = result.SeriesName; } - if (!info.EndingEpisodeNumber.HasValue && info.EpisodeNumber.HasValue) + if (!info.EndingEpisodeNumber.HasValue && result.EndingEpisodeNumber >= info.EpisodeNumber) { info.EndingEpisodeNumber = result.EndingEpisodeNumber; } diff --git a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs index 933cfc8cbe..02b104756e 100644 --- a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs +++ b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs @@ -27,6 +27,11 @@ namespace Emby.Server.Implementations.EntryPoints; /// </summary> public sealed class LibraryChangedNotifier : IHostedService, IDisposable { + // A batch holds a live reference to every item it names, so it has to stay small enough that a + // library scan - which changes items faster than any batch window closes - cannot grow it without + // bound. Reached only by a scan; interactive use closes a batch on the window long before this. + internal const int MaxBatchSize = 2000; + private readonly ILibraryManager _libraryManager; private readonly IServerConfigurationManager _configurationManager; private readonly IProviderManager _providerManager; @@ -35,11 +40,11 @@ public sealed class LibraryChangedNotifier : IHostedService, IDisposable private readonly ILogger<LibraryChangedNotifier> _logger; private readonly Lock _libraryChangedSyncLock = new(); - private readonly List<Folder> _foldersAddedTo = new(); - private readonly List<Folder> _foldersRemovedFrom = new(); - private readonly List<BaseItem> _itemsAdded = new(); - private readonly List<BaseItem> _itemsRemoved = new(); - private readonly List<BaseItem> _itemsUpdated = new(); + private readonly Dictionary<Guid, Folder> _foldersAddedTo = []; + private readonly Dictionary<Guid, Folder> _foldersRemovedFrom = []; + private readonly Dictionary<Guid, BaseItem> _itemsAdded = []; + private readonly Dictionary<Guid, BaseItem> _itemsRemoved = []; + private readonly Dictionary<Guid, BaseItem> _itemsUpdated = []; private readonly ConcurrentDictionary<Guid, DateTime> _lastProgressMessageTimes = new(); private Timer? _libraryUpdateTimer; @@ -173,7 +178,7 @@ public sealed class LibraryChangedNotifier : IHostedService, IDisposable private void OnLibraryItemRemoved(object? sender, ItemChangeEventArgs e) => OnLibraryChange(e.Item, e.Parent, _itemsRemoved, _foldersRemovedFrom); - private void OnLibraryChange(BaseItem item, BaseItem parent, List<BaseItem> itemsList, List<Folder>? foldersList) + private void OnLibraryChange(BaseItem item, BaseItem parent, Dictionary<Guid, BaseItem> itemsList, Dictionary<Guid, Folder>? foldersList) { if (!FilterItem(item)) { @@ -182,23 +187,28 @@ public sealed class LibraryChangedNotifier : IHostedService, IDisposable lock (_libraryChangedSyncLock) { - var updateDuration = TimeSpan.FromSeconds(_configurationManager.Configuration.LibraryUpdateDuration); - + // The window runs from the first change of a batch and is never extended. Extending it on + // every change would keep a library scan's batch open for the whole scan, and the batch + // holds the items it names alive, so it would grow to the size of the library. if (_libraryUpdateTimer is null) { + var updateDuration = TimeSpan.FromSeconds(_configurationManager.Configuration.LibraryUpdateDuration); _libraryUpdateTimer = new Timer(LibraryUpdateTimerCallback, null, updateDuration, Timeout.InfiniteTimeSpan); } - else - { - _libraryUpdateTimer.Change(updateDuration, Timeout.InfiniteTimeSpan); - } if (foldersList is not null && parent is Folder folder) { - foldersList.Add(folder); + foldersList[folder.Id] = folder; } - itemsList.Add(item); + itemsList[item.Id] = item; + + // A window long enough to cover a burst still has to give way once the batch is large + // enough to be worth sending on its own. + if (_itemsAdded.Count + _itemsRemoved.Count + _itemsUpdated.Count >= MaxBatchSize) + { + _libraryUpdateTimer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); + } } } @@ -211,22 +221,16 @@ public sealed class LibraryChangedNotifier : IHostedService, IDisposable List<BaseItem> itemsRemoved; lock (_libraryChangedSyncLock) { - // Remove dupes in case some were saved multiple times - foldersAddedTo = _foldersAddedTo - .DistinctBy(x => x.Id) - .ToList(); - - foldersRemovedFrom = _foldersRemovedFrom - .DistinctBy(x => x.Id) - .ToList(); + foldersAddedTo = _foldersAddedTo.Values.ToList(); + foldersRemovedFrom = _foldersRemovedFrom.Values.ToList(); itemsUpdated = _itemsUpdated - .Where(i => !_itemsAdded.Contains(i)) - .DistinctBy(x => x.Id) + .Where(e => !_itemsAdded.ContainsKey(e.Key)) + .Select(e => e.Value) .ToList(); - itemsAdded = _itemsAdded.ToList(); - itemsRemoved = _itemsRemoved.ToList(); + itemsAdded = _itemsAdded.Values.ToList(); + itemsRemoved = _itemsRemoved.Values.ToList(); if (_libraryUpdateTimer is not null) { @@ -241,6 +245,15 @@ public sealed class LibraryChangedNotifier : IHostedService, IDisposable _foldersRemovedFrom.Clear(); } + if (itemsAdded.Count == 0 + && itemsUpdated.Count == 0 + && itemsRemoved.Count == 0 + && foldersAddedTo.Count == 0 + && foldersRemovedFrom.Count == 0) + { + return; + } + await SendChangeNotifications(itemsAdded, itemsUpdated, itemsRemoved, foldersAddedTo, foldersRemovedFrom, CancellationToken.None).ConfigureAwait(false); } diff --git a/Emby.Server.Implementations/EntryPoints/UserDataChangeNotifier.cs b/Emby.Server.Implementations/EntryPoints/UserDataChangeNotifier.cs index fc174b7c14..b182e5837b 100644 --- a/Emby.Server.Implementations/EntryPoints/UserDataChangeNotifier.cs +++ b/Emby.Server.Implementations/EntryPoints/UserDataChangeNotifier.cs @@ -18,15 +18,17 @@ namespace Emby.Server.Implementations.EntryPoints public sealed class UserDataChangeNotifier : IHostedService, IDisposable { private const int UpdateDuration = 500; + internal const int MaxBatchSize = 2000; private readonly ISessionManager _sessionManager; private readonly IUserDataManager _userDataManager; private readonly IUserManager _userManager; - private readonly Dictionary<Guid, List<BaseItem>> _changedItems = new(); + private readonly Dictionary<Guid, Dictionary<Guid, BaseItem>> _changedItems = []; private readonly Lock _syncLock = new(); private Timer? _updateTimer; + private int _changedItemCount; /// <summary> /// Initializes a new instance of the <see cref="UserDataChangeNotifier"/> class. @@ -69,50 +71,64 @@ namespace Emby.Server.Implementations.EntryPoints lock (_syncLock) { - if (_updateTimer is null) + // The window runs from the first change of a batch and is never extended, so a stream + // of changes that never pauses - a library scan - still closes its batches instead of + // holding every item it touched alive until the stream stops. + _updateTimer ??= new Timer( + UpdateTimerCallback, + null, + UpdateDuration, + Timeout.Infinite); + + if (!_changedItems.TryGetValue(e.UserId, out Dictionary<Guid, BaseItem>? keys)) { - _updateTimer = new Timer( - UpdateTimerCallback, - null, - UpdateDuration, - Timeout.Infinite); - } - else - { - _updateTimer.Change(UpdateDuration, Timeout.Infinite); - } - - if (!_changedItems.TryGetValue(e.UserId, out List<BaseItem>? keys)) - { - keys = new List<BaseItem>(); + keys = []; _changedItems[e.UserId] = keys; } - keys.Add(e.Item); - var baseItem = e.Item; // Go up one level for indicators if (baseItem is not null) { + Track(keys, baseItem); + var parent = baseItem.GetOwner() ?? baseItem.GetParent(); if (parent is not null) { - keys.Add(parent); + Track(keys, parent); } } + + // A window long enough to cover a burst still has to give way once the batch is + // large enough to be worth sending on its own. + if (_changedItemCount >= MaxBatchSize) + { + _updateTimer.Change(0, Timeout.Infinite); + } + } + } + + private void Track(Dictionary<Guid, BaseItem> keys, BaseItem item) + { + var before = keys.Count; + keys[item.Id] = item; + + if (keys.Count != before) + { + _changedItemCount++; } } private async void UpdateTimerCallback(object? state) { - List<KeyValuePair<Guid, List<BaseItem>>> changes; + List<KeyValuePair<Guid, Dictionary<Guid, BaseItem>>> changes; lock (_syncLock) { - // Remove dupes in case some were saved multiple times changes = _changedItems.ToList(); _changedItems.Clear(); + _changedItemCount = 0; if (_updateTimer is not null) { @@ -121,17 +137,22 @@ namespace Emby.Server.Implementations.EntryPoints } } + if (changes.Count == 0) + { + return; + } + foreach (var (userId, changedItems) in changes) { await _sessionManager.SendMessageToUserSessions( [userId], SessionMessageType.UserDataChanged, - () => GetUserDataChangeInfo(userId, changedItems), + () => GetUserDataChangeInfo(userId, changedItems.Values), default).ConfigureAwait(false); } } - private UserDataChangeInfo GetUserDataChangeInfo(Guid userId, List<BaseItem> changedItems) + private UserDataChangeInfo GetUserDataChangeInfo(Guid userId, IEnumerable<BaseItem> changedItems) { var user = _userManager.GetUserById(userId) ?? throw new ArgumentException("Invalid user ID", nameof(userId)); @@ -140,7 +161,6 @@ namespace Emby.Server.Implementations.EntryPoints { UserId = userId, UserDataList = changedItems - .DistinctBy(x => x.Id) .Select(i => { var dto = _userDataManager.GetUserDataDto(i, user); diff --git a/Emby.Server.Implementations/IO/LibraryMonitor.cs b/Emby.Server.Implementations/IO/LibraryMonitor.cs index 1bf0f8c76c..0f92e2f03e 100644 --- a/Emby.Server.Implementations/IO/LibraryMonitor.cs +++ b/Emby.Server.Implementations/IO/LibraryMonitor.cs @@ -8,6 +8,7 @@ using Emby.Server.Implementations.Library; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers; using MediaBrowser.Model.IO; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -21,6 +22,7 @@ namespace Emby.Server.Implementations.IO private readonly ILibraryManager _libraryManager; private readonly IServerConfigurationManager _configurationManager; private readonly IFileSystem _fileSystem; + private readonly IDirectoryService _directoryService; private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; /// <summary> @@ -47,6 +49,7 @@ namespace Emby.Server.Implementations.IO /// <param name="libraryManager">The library manager.</param> /// <param name="configurationManager">The configuration manager.</param> /// <param name="fileSystem">The filesystem.</param> + /// <param name="directoryService">The directory service.</param> /// <param name="appLifetime">The <see cref="IHostApplicationLifetime"/>.</param> /// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param> public LibraryMonitor( @@ -54,6 +57,7 @@ namespace Emby.Server.Implementations.IO ILibraryManager libraryManager, IServerConfigurationManager configurationManager, IFileSystem fileSystem, + IDirectoryService directoryService, IHostApplicationLifetime appLifetime, DotIgnoreIgnoreRule dotIgnoreIgnoreRule) { @@ -61,6 +65,7 @@ namespace Emby.Server.Implementations.IO _logger = logger; _configurationManager = configurationManager; _fileSystem = fileSystem; + _directoryService = directoryService; _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; appLifetime.ApplicationStarted.Register(Start); @@ -363,6 +368,8 @@ namespace Emby.Server.Implementations.IO return; } + _directoryService.Invalidate(path); + // Ignore certain files, If the parent of an ignored path has a change event, ignore that too foreach (var i in _tempIgnoredPaths.Keys) { diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 3db8265f6e..e6fa94fbef 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -15,7 +16,6 @@ using Emby.Naming.Common; using Emby.Naming.TV; using Emby.Naming.Video; using Emby.Server.Implementations.Library.Resolvers; -using Emby.Server.Implementations.Library.Validators; using Emby.Server.Implementations.Playlists; using Emby.Server.Implementations.ScheduledTasks.Tasks; using Emby.Server.Implementations.Sorting; @@ -35,7 +35,6 @@ using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; -using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Playlists; using MediaBrowser.Controller.Providers; @@ -75,7 +74,6 @@ namespace Emby.Server.Implementations.Library private readonly Lazy<IProviderManager> _providerManagerFactory; private readonly Lazy<IUserViewManager> _userViewManagerFactory; private readonly IServerApplicationHost _appHost; - private readonly IMediaEncoder _mediaEncoder; private readonly IFileSystem _fileSystem; private readonly IItemRepository _itemRepository; private readonly IItemPersistenceService _persistenceService; @@ -88,6 +86,7 @@ namespace Emby.Server.Implementations.Library private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; private readonly ILocalizationManager _localization; + private readonly IDirectoryService _directoryService; private readonly FastConcurrentLru<Guid, BaseItem> _cache; private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; private readonly IMediaStreamRepository _mediaStreamRepository; @@ -122,7 +121,6 @@ namespace Emby.Server.Implementations.Library /// <param name="fileSystem">The file system.</param> /// <param name="providerManagerFactory">The provider manager.</param> /// <param name="userViewManagerFactory">The user view manager.</param> - /// <param name="mediaEncoder">The media encoder.</param> /// <param name="itemRepository">The item repository.</param> /// <param name="persistenceService">The item persistence service.</param> /// <param name="nextUpService">The next up service.</param> @@ -148,7 +146,6 @@ namespace Emby.Server.Implementations.Library IFileSystem fileSystem, Lazy<IProviderManager> providerManagerFactory, Lazy<IUserViewManager> userViewManagerFactory, - IMediaEncoder mediaEncoder, IItemRepository itemRepository, IItemPersistenceService persistenceService, INextUpService nextUpService, @@ -174,7 +171,6 @@ namespace Emby.Server.Implementations.Library _fileSystem = fileSystem; _providerManagerFactory = providerManagerFactory; _userViewManagerFactory = userViewManagerFactory; - _mediaEncoder = mediaEncoder; _itemRepository = itemRepository; _persistenceService = persistenceService; _nextUpService = nextUpService; @@ -189,6 +185,7 @@ namespace Emby.Server.Implementations.Library _pathManager = pathManager; _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; _localization = localization; + _directoryService = directoryService; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -1210,6 +1207,12 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc /> + public Guid GetPersonId(string name) + { + return GetItemByNameId<Person>(Person.GetPath(name)); + } + + /// <inheritdoc /> public Person? GetPerson(string name) { var path = Person.GetPath(name); @@ -1222,6 +1225,33 @@ namespace Emby.Server.Implementations.Library return null; } + /// <inheritdoc /> + public Person GetOrCreatePerson(string name) + { + var existing = GetPerson(name); + if (existing is not null) + { + return existing; + } + + var path = Person.GetPath(name); + var info = Directory.CreateDirectory(path); + var item = new Person + { + Name = name, + Id = GetItemByNameId<Person>(path), + DateCreated = info.CreationTimeUtc, + DateModified = info.LastWriteTimeUtc, + Path = path + }; + + item.PresentationUniqueKey = item.CreatePresentationUniqueKey(); + + CreateItem(item, null); + + return item; + } + /// <summary> /// Gets the studio. /// </summary> @@ -1354,15 +1384,6 @@ namespace Emby.Server.Implementations.Library return GetNewItemIdInternal(path, typeof(T), forceCaseInsensitiveId); } - /// <inheritdoc /> - public Task ValidatePeopleAsync(IProgress<double> progress, CancellationToken cancellationToken) - { - // Ensure the location is available. - Directory.CreateDirectory(_configurationManager.ApplicationPaths.PeoplePath); - - return new PeopleValidator(this, _logger, _fileSystem).ValidatePeople(cancellationToken, progress); - } - /// <summary> /// Reloads the root media folder. /// </summary> @@ -1489,6 +1510,10 @@ namespace Emby.Server.Implementations.Library var numComplete = 0; var numTasks = tasks.Count; + _logger.LogInformation("Running {TaskCount} post-scan task(s)", numTasks); + + var phaseStart = Stopwatch.GetTimestamp(); + foreach (var task in tasks) { // Prevent access to modified closure @@ -1506,20 +1531,45 @@ namespace Emby.Server.Implementations.Library progress.Report(innerPercent); }); - _logger.LogDebug("Running post-scan task {0}", task.GetType().Name); + var taskName = task.GetType().Name; + var taskStart = Stopwatch.GetTimestamp(); + + _logger.LogInformation( + "Running post-scan task {TaskNumber}/{TaskCount}: {TaskName}", + currentNumComplete + 1, + numTasks, + taskName); try { await task.Run(innerProgress, cancellationToken).ConfigureAwait(false); + + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} completed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } catch (OperationCanceledException) { - _logger.LogInformation("Post-scan task cancelled: {0}", task.GetType().Name); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} cancelled after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); throw; } catch (Exception ex) { - _logger.LogError(ex, "Error running post-scan task"); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogError( + ex, + "Post-scan task {TaskName} failed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } numComplete++; @@ -1528,6 +1578,12 @@ namespace Emby.Server.Implementations.Library progress.Report(percent * 100); } + var phaseElapsed = Stopwatch.GetElapsedTime(phaseStart); + _logger.LogInformation( + "All post-scan tasks completed after {Minutes} minute(s) and {Seconds} seconds", + Math.Truncate(phaseElapsed.TotalMinutes), + phaseElapsed.Seconds); + _persistenceService.UpdateInheritedValues(); progress.Report(100); @@ -3720,6 +3776,10 @@ namespace Emby.Server.Implementations.Library AddMediaPathInternal(name, path, false); } } + + // The libraries root was listed before this folder existed, so drop that listing: + // anything still reading it resolves the library set without the new folder. + _directoryService.Invalidate(virtualFolderPath); } finally { @@ -3746,27 +3806,14 @@ namespace Emby.Server.Implementations.Library var itemUpdateType = ItemUpdateType.MetadataDownload; var saveEntity = false; - var createEntity = false; var personEntity = GetPerson(person.Name); if (personEntity is null) { try { - var path = Person.GetPath(person.Name); - var info = Directory.CreateDirectory(path); - personEntity = new Person() - { - Name = person.Name, - Id = GetItemByNameId<Person>(path), - DateCreated = info.CreationTimeUtc, - DateModified = info.LastWriteTimeUtc, - Path = path - }; - - personEntity.PresentationUniqueKey = personEntity.CreatePresentationUniqueKey(); + personEntity = GetOrCreatePerson(person.Name); saveEntity = true; - createEntity = true; } catch (Exception ex) { @@ -3800,11 +3847,6 @@ namespace Emby.Server.Implementations.Library if (saveEntity) { - if (createEntity) - { - CreateItems([personEntity], null, CancellationToken.None); - } - await RunMetadataSavers(personEntity, itemUpdateType).ConfigureAwait(false); personEntity.DateLastSaved = DateTime.UtcNow; @@ -3920,6 +3962,7 @@ namespace Emby.Server.Implementations.Library try { Directory.Delete(path, true); + _directoryService.Invalidate(path); } finally { @@ -3989,6 +4032,7 @@ namespace Emby.Server.Implementations.Library if (!string.IsNullOrEmpty(shortcut)) { _fileSystem.DeleteFile(shortcut); + _directoryService.Invalidate(shortcut); } var libraryOptions = CollectionFolder.GetLibraryOptions(virtualFolderPath); @@ -4032,6 +4076,7 @@ namespace Emby.Server.Implementations.Library } _fileSystem.CreateShortcut(lnk, _appHost.ReverseVirtualPath(path)); + _directoryService.Invalidate(lnk); RemoveContentTypeOverrides(path); } diff --git a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs index fa7112eb90..690466be70 100644 --- a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Threading; @@ -61,6 +62,9 @@ public class ArtistsValidator var count = names.Count; var refreshed = 0; + var liveIds = new HashSet<Guid>(); + var unresolved = 0; + foreach (var name in names) { try @@ -73,13 +77,20 @@ public class ArtistsValidator // Fall back to GetArtist if not found (creates new item if needed) item ??= _libraryManager.GetArtist(name); - var isNew = !existingArtistIds.Contains(item.Id); - var neverRefreshed = item.DateLastRefreshed == default; - if (isNew || neverRefreshed) + // A name with no item is nothing to refresh, and nothing to keep alive either. + if (item is not null) { - await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); - refreshed++; + liveIds.Add(item.Id); + + var isNew = !existingArtistIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; + + if (isNew || neverRefreshed) + { + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } } catch (OperationCanceledException) @@ -88,6 +99,7 @@ public class ArtistsValidator } catch (Exception ex) { + unresolved++; _logger.LogError(ex, "Error refreshing {ArtistName}", name); } @@ -101,13 +113,26 @@ public class ArtistsValidator _logger.LogInformation("Refreshed metadata for {RefreshedCount} new artists out of {TotalCount} total", refreshed, count); + // Every name that threw is a name whose artist is missing from the live set, and deleting against + // a live set with holes in it deletes artists the library still refers to. Leave the sweep to a + // run that got a clean read of them. + if (unresolved > 0) + { + _logger.LogWarning( + "Not removing dead artists: {Count} of {TotalCount} names could not be resolved this run", + unresolved, + count); + + progress.Report(100); + return; + } + var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery { IncludeItemTypes = [BaseItemKind.MusicArtist], - IsDeadArtist = true, IsLocked = false - }).Cast<MusicArtist>() - .Where(item => item.IsAccessedByName) + }).OfType<MusicArtist>() + .Where(item => item.IsAccessedByName && !liveIds.Contains(item.Id)) .ToList(); foreach (var item in deadEntities) diff --git a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs index 078a0b921d..7d53f40ce7 100644 --- a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs @@ -1,12 +1,12 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.IO; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library.Validators; @@ -17,112 +17,143 @@ namespace Emby.Server.Implementations.Library.Validators; public class PeopleValidator { /// <summary> - /// The _library manager. + /// The library manager. /// </summary> private readonly ILibraryManager _libraryManager; /// <summary> - /// The _logger. + /// The logger. /// </summary> - private readonly ILogger _logger; - - private readonly IFileSystem _fileSystem; + private readonly ILogger<PeopleValidator> _logger; /// <summary> /// Initializes a new instance of the <see cref="PeopleValidator" /> class. /// </summary> /// <param name="libraryManager">The library manager.</param> /// <param name="logger">The logger.</param> - /// <param name="fileSystem">The file system.</param> - public PeopleValidator(ILibraryManager libraryManager, ILogger logger, IFileSystem fileSystem) + public PeopleValidator(ILibraryManager libraryManager, ILogger<PeopleValidator> logger) { _libraryManager = libraryManager; _logger = logger; - _fileSystem = fileSystem; } /// <summary> /// Validates the people. /// </summary> - /// <param name="cancellationToken">The cancellation token.</param> /// <param name="progress">The progress.</param> + /// <param name="cancellationToken">The cancellation token.</param> /// <returns>Task.</returns> - public async Task ValidatePeople(CancellationToken cancellationToken, IProgress<double> progress) + public async Task Run(IProgress<double> progress, CancellationToken cancellationToken) { // Before the refresh below walks them: a credit no item maps to any more stands for nothing, // and while it is there the person it names cannot reach the dead-person sweep either. var numOrphaned = _libraryManager.DeleteOrphanedCredits(); if (numOrphaned > 0) { - _logger.LogDebug("Deleted {Amount} credits no item maps to", numOrphaned); + _logger.LogInformation("Deleted {Amount} credits no item maps to", numOrphaned); } - var people = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); - - var numComplete = 0; - - var numPeople = people.Count; + var names = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); + var existingPersonIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + IncludeItemTypes = [BaseItemKind.Person] + }).ToHashSet(); - IProgress<double> subProgress = new Progress<double>((val) => progress.Report(val / 2)); + var (newNames, deadIds) = PartitionCreditsByPersonId(names, _libraryManager.GetPersonId, existingPersonIds); - _logger.LogDebug("Will refresh {Amount} people", numPeople); + var numComplete = 0; + var count = names.Count; + var refreshed = 0; - foreach (var person in people) + foreach (var name in names) { cancellationToken.ThrowIfCancellationRequested(); try { - var item = _libraryManager.GetPerson(person); - if (item is null) - { - _logger.LogWarning("Failed to get person: {Name}", person); - continue; - } + var item = _libraryManager.GetOrCreatePerson(name); + var isNew = !existingPersonIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; - var options = new MetadataRefreshOptions(new DirectoryService(_fileSystem)) + if (isNew || neverRefreshed) { - ImageRefreshMode = MetadataRefreshMode.ValidationOnly, - MetadataRefreshMode = MetadataRefreshMode.ValidationOnly - }; - - await item.RefreshMetadata(options, cancellationToken).ConfigureAwait(false); + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } catch (OperationCanceledException) { + // Don't clutter the log throw; } catch (Exception ex) { - _logger.LogError(ex, "Error validating IBN entry {Person}", person); + _logger.LogError(ex, "Error refreshing {PersonName}", name); } - // Update progress numComplete++; double percent = numComplete; - percent /= numPeople; + percent /= count; + percent *= 100; - subProgress.Report(100 * percent); + progress.Report(percent); } - var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery - { - IncludeItemTypes = [BaseItemKind.Person], - IsDeadPerson = true, - IsLocked = false - }); + _logger.LogInformation( + "Refreshed metadata for {RefreshedCount} people out of {TotalCount} total, {NewCount} of which had no item yet", + refreshed, + count, + newNames.Count); - subProgress = new Progress<double>((val) => progress.Report((val / 2) + 50)); + // A person somebody locked is theirs, not ours, however little the library still credits them. + var deadEntities = deadIds + .Select(_libraryManager.GetItemById) + .OfType<Person>() + .Where(item => !item.IsLocked) + .ToList(); - var i = 0; - foreach (var item in deadEntities.Chunk(500)) + foreach (var item in deadEntities) { - _libraryManager.DeleteItemsUnsafeFast(item, true); - subProgress.Report(100f / deadEntities.Count * (i++ * 100)); + _logger.LogInformation("Deleting dead {ItemType} {ItemId} {ItemName}", item.GetType().Name, item.Id.ToString("N", CultureInfo.InvariantCulture), item.Name); } + _libraryManager.DeleteItemsUnsafeFast(deadEntities, deleteSourceFiles: true); + progress.Report(100); + } + + /// <summary> + /// Splits the person items into the ones a credit still calls for and the ones nothing does. + /// </summary> + /// <param name="creditNames">Every name credited on an item, from the people table.</param> + /// <param name="getPersonId">Maps a credit name to the id its person item has.</param> + /// <param name="existingPersonIds">The ids of the person items that exist.</param> + /// <returns>The credits needing an item, and the ids of the items nothing credits.</returns> + internal static (List<string> NewNames, List<Guid> DeadIds) PartitionCreditsByPersonId( + IReadOnlyList<string> creditNames, + Func<string, Guid> getPersonId, + IReadOnlySet<Guid> existingPersonIds) + { + ArgumentNullException.ThrowIfNull(creditNames); + ArgumentNullException.ThrowIfNull(getPersonId); + ArgumentNullException.ThrowIfNull(existingPersonIds); + + var newNames = new List<string>(); + var liveIds = new HashSet<Guid>(); + + foreach (var name in creditNames) + { + var personId = getPersonId(name); + + // Distinct credit names can normalize onto one id; only the first of them needs an item. + if (liveIds.Add(personId) && !existingPersonIds.Contains(personId)) + { + newNames.Add(name); + } + } + + var deadIds = existingPersonIds.Where(id => !liveIds.Contains(id)).ToList(); - _logger.LogInformation("People validation complete, deleted {Orphaned} orphaned credits", numOrphaned); + return (newNames, deadIds); } } diff --git a/Emby.Server.Implementations/Localization/Core/be.json b/Emby.Server.Implementations/Localization/Core/be.json index 49ebc45f06..c5b1213096 100644 --- a/Emby.Server.Implementations/Localization/Core/be.json +++ b/Emby.Server.Implementations/Localization/Core/be.json @@ -112,5 +112,11 @@ "NameExtraInterview": "Інтэрв'ю", "NameExtraNumbered": "{0} {1}", "NameExtraScene": "Сцэна", - "NameExtraTrailer": "Трэйлер" + "NameExtraTrailer": "Трэйлер", + "NameExtraBehindTheScenes": "За кулісамі", + "NameExtraClip": "Кліп", + "NameExtraFeaturette": "Кароткаметражка", + "NameExtraSample": "Прыклад", + "NameExtraShort": "Кароткаметражка", + "NameExtraThemeSong": "Тэматычная песня" } diff --git a/Emby.Server.Implementations/Localization/Core/bs.json b/Emby.Server.Implementations/Localization/Core/bs.json index aa7fe4eb24..5686807d9a 100644 --- a/Emby.Server.Implementations/Localization/Core/bs.json +++ b/Emby.Server.Implementations/Localization/Core/bs.json @@ -106,5 +106,17 @@ "TaskMoveTrickplayImages": "Migracija lokacije slike Trickplay", "TaskMoveTrickplayImagesDescription": "Premješta postojeće datoteke trik-igara prema postavkama biblioteke.", "CleanupUserDataTask": "Zadatak čišćenja korisničkih podataka", - "CleanupUserDataTaskDescription": "Čisti sve korisničke podatke (stanje praćenja, status omiljenog itd.) sa medija koji više nije prisutan najmanje 90 dana." + "CleanupUserDataTaskDescription": "Čisti sve korisničke podatke (stanje praćenja, status omiljenog itd.) sa medija koji više nije prisutan najmanje 90 dana.", + "NameExtraBehindTheScenes": "Iza kulisa", + "NameExtraClip": "Isječak", + "NameExtraDeletedScene": "Izbrišana scena", + "NameExtraFeaturette": "Kratki prilog", + "NameExtraInterview": "Intervju", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Uzorak", + "NameExtraScene": "Scena", + "NameExtraShort": "Kratko", + "NameExtraThemeSong": "Tema", + "NameExtraThemeVideo": "Tematski video", + "NameExtraTrailer": "Najava" } diff --git a/Emby.Server.Implementations/Localization/Core/el.json b/Emby.Server.Implementations/Localization/Core/el.json index c0ad2c165a..610bc286b8 100644 --- a/Emby.Server.Implementations/Localization/Core/el.json +++ b/Emby.Server.Implementations/Localization/Core/el.json @@ -108,5 +108,16 @@ "CleanupUserDataTaskDescription": "Καθαρίζει όλα τα δεδομένα χρήστη (κατάσταση παρακολούθησης, κατάσταση αγαπημένων κ.λπ.) από πολυμέσα που δεν υπάρχουν πλέον για τουλάχιστον 90 ημέρες.", "CleanupUserDataTask": "Εργασία εκκαθάρισης δεδομένων χρήστη", "LyricDownloadFailureFromForItem": "Αποτυχία λήψης στίχων από {0} για {1}", - "Original": "Πρωτότυπο" + "Original": "Πρωτότυπο", + "NameExtraBehindTheScenes": "Πίσω από τις Σκηνές", + "NameExtraDeletedScene": "Διεγραμμένη Σκηνή", + "NameExtraFeaturette": "Πρόσθετα βίντεο", + "NameExtraInterview": "Συνέντευξη", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Δείγμα", + "NameExtraScene": "Σκηνή", + "NameExtraShort": "Βίντεο μικρού μήκους", + "NameExtraThemeSong": "Θεματικό Τραγούδι", + "NameExtraThemeVideo": "Θεματικό Βίντεο", + "NameExtraTrailer": "τρέιλερ ταινίας" } diff --git a/Emby.Server.Implementations/Localization/Core/fi.json b/Emby.Server.Implementations/Localization/Core/fi.json index 36a248a1d1..9a453120dd 100644 --- a/Emby.Server.Implementations/Localization/Core/fi.json +++ b/Emby.Server.Implementations/Localization/Core/fi.json @@ -113,5 +113,12 @@ "NameExtraClip": "Klippi", "NameExtraDeletedScene": "Poistettu Kohtaus", "NameExtraFeaturette": "Lyhytelokuva", - "NameExtraInterview": "Haastattelu" + "NameExtraInterview": "Haastattelu", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Näyte", + "NameExtraScene": "Kohtaus", + "NameExtraShort": "Lyhytfilmi", + "NameExtraThemeSong": "Tunnusmusiikki", + "NameExtraThemeVideo": "Tunnusvideo", + "NameExtraTrailer": "Traileri" } diff --git a/Emby.Server.Implementations/Localization/Core/fo.json b/Emby.Server.Implementations/Localization/Core/fo.json index 1a1c89da35..bd15bac865 100644 --- a/Emby.Server.Implementations/Localization/Core/fo.json +++ b/Emby.Server.Implementations/Localization/Core/fo.json @@ -8,7 +8,7 @@ "Books": "Bøkur", "ChapterNameValue": "Kapittul {0}", "Favorites": "Yndis", - "Folders": "Mappur", + "Folders": "Skjáttur", "Forced": "Kravt", "FailedLoginAttemptWithUserName": "Miseydnað innritanarroynd frá {0}", "HeaderFavoriteEpisodes": "Yndispartar", @@ -104,7 +104,7 @@ "NotificationOptionCameraImageUploaded": "Ljósmynd uppsend", "NameExtraShort": "Stuttfilmur", "NameExtraThemeSong": "Eyðkennislag", - "NameExtraTrailer": "Forfilmur", + "NameExtraTrailer": "Brellbiti", "NameExtraInterview": "Samrøða", "NameExtraBehindTheScenes": "Aftanfyri leiktjøldini", "NameExtraClip": "Klipp", @@ -119,7 +119,7 @@ "TaskMoveTrickplayImagesDescription": "Flytur verandi trickplay-fílur sambært savnsstillingunum.", "NameExtraThemeVideo": "Eyðkenniskykmynd", "NameExtraDeletedScene": "Úrtikin mynd", - "NameExtraScene": "Mynd (scena)", + "NameExtraScene": "Mynd", "NameExtraUnknown": "Eykatilfar", "Original": "Upprunalig(t/ur)" } diff --git a/Emby.Server.Implementations/Localization/Core/ga.json b/Emby.Server.Implementations/Localization/Core/ga.json index 1ee606cc64..30e11d15f0 100644 --- a/Emby.Server.Implementations/Localization/Core/ga.json +++ b/Emby.Server.Implementations/Localization/Core/ga.json @@ -108,5 +108,17 @@ "CleanupUserDataTask": "Tasc glantacháin sonraí úsáideora", "CleanupUserDataTaskDescription": "Glanann sé gach sonraí úsáideora (stádas faire, stádas is fearr leat srl.) ó mheáin nach bhfuil i láthair a thuilleadh ar feadh 90 lá ar a laghad.", "Original": "Bunaidh", - "LyricDownloadFailureFromForItem": "Theip ar liricí a íoslódáil ó {0} do {1}" + "LyricDownloadFailureFromForItem": "Theip ar liricí a íoslódáil ó {0} do {1}", + "NameExtraBehindTheScenes": "Taobh thiar de na Radhairc", + "NameExtraClip": "Gearrthóg", + "NameExtraDeletedScene": "Radharc Scriosta", + "NameExtraFeaturette": "Mionghné", + "NameExtraInterview": "Agallamh", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Sampla", + "NameExtraScene": "Radharc", + "NameExtraShort": "Gearr", + "NameExtraThemeSong": "Amhrán Téama", + "NameExtraThemeVideo": "Físeán Téama", + "NameExtraTrailer": "Leantóir" } diff --git a/Emby.Server.Implementations/Localization/Core/hr.json b/Emby.Server.Implementations/Localization/Core/hr.json index 442c26b30b..2d38c173f0 100644 --- a/Emby.Server.Implementations/Localization/Core/hr.json +++ b/Emby.Server.Implementations/Localization/Core/hr.json @@ -108,5 +108,17 @@ "CleanupUserDataTask": "Zadatak čišćenja korisničkih podataka", "CleanupUserDataTaskDescription": "Briše sve korisničke podatke (stanje gledanja, status favorita itd.) s medija koji više nisu prisutni najmanje 90 dana.", "Original": "Original", - "LyricDownloadFailureFromForItem": "Preuzimanje tekstova pjesmi od {0} za {1} nije uspjelo" + "LyricDownloadFailureFromForItem": "Preuzimanje tekstova pjesmi od {0} za {1} nije uspjelo", + "NameExtraBehindTheScenes": "Iza kulisa", + "NameExtraClip": "Klip", + "NameExtraDeletedScene": "Obrisana Scena", + "NameExtraFeaturette": "Promotivni video", + "NameExtraInterview": "Intervju", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Uzorak", + "NameExtraScene": "Scena", + "NameExtraShort": "Kratki film", + "NameExtraThemeSong": "Glavna Pjesma", + "NameExtraThemeVideo": "Tema videa", + "NameExtraTrailer": "Trailer" } diff --git a/Emby.Server.Implementations/Localization/Core/lb.json b/Emby.Server.Implementations/Localization/Core/lb.json index 917f26a49c..21d31c5fbf 100644 --- a/Emby.Server.Implementations/Localization/Core/lb.json +++ b/Emby.Server.Implementations/Localization/Core/lb.json @@ -108,5 +108,17 @@ "LyricDownloadFailureFromForItem": "Feeler beim Download vun de Songtexter vun {0} fir {1}", "Original": "Original", "CleanupUserDataTask": "Aufgab fir Berengege vu Benotzerdaten", - "CleanupUserDataTaskDescription": "Läscht all Benotzerdaten (Ofspillstatus, Favoritestatus, asw.) vu Medien, déi zënter mindestens 90 Deeg net méi besteeënd sinn." + "CleanupUserDataTaskDescription": "Läscht all Benotzerdaten (Ofspillstatus, Favoritestatus, asw.) vu Medien, déi zënter mindestens 90 Deeg net méi besteeënd sinn.", + "NameExtraBehindTheScenes": "Hannert de Kulissen", + "NameExtraClip": "Clip", + "NameExtraDeletedScene": "Geläschte Scène", + "NameExtraFeaturette": "Featurette", + "NameExtraInterview": "Interview", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Beispill", + "NameExtraScene": "Scène", + "NameExtraShort": "Kuerzfilm", + "NameExtraThemeSong": "Theme-Lidd", + "NameExtraThemeVideo": "Theme-Video", + "NameExtraTrailer": "Bande-Annonce" } diff --git a/Emby.Server.Implementations/Localization/Core/lt-LT.json b/Emby.Server.Implementations/Localization/Core/lt-LT.json index dbfeabd88e..c41cedf98a 100644 --- a/Emby.Server.Implementations/Localization/Core/lt-LT.json +++ b/Emby.Server.Implementations/Localization/Core/lt-LT.json @@ -100,14 +100,14 @@ "TaskAudioNormalization": "Garso normalizavimas", "TaskAudioNormalizationDescription": "Skenuoja failus, ieškant garso normalizavimo duomenų.", "TaskExtractMediaSegments": "Medijos segmentų nuskaitymas", - "TaskDownloadMissingLyrics": "Parsisiųsti trūkstamus dainų tekstus", + "TaskDownloadMissingLyrics": "Atsisiųsti trūkstamus dainų tekstus", "TaskExtractMediaSegmentsDescription": "Ištraukia arba gauna medijos segmentus iš MediaSegment ijungtų įskiepių.", "TaskMoveTrickplayImages": "Pakeisti Trickplay atvaizdų vietą", "TaskMoveTrickplayImagesDescription": "Perkelia egzistuojančius Trickplay failus pagal bibliotekos nustatymus.", - "TaskDownloadMissingLyricsDescription": "Parsisiųsti dainų žodžius", + "TaskDownloadMissingLyricsDescription": "Atsisiųsti dainų tekstus", "CleanupUserDataTask": "Naudotojo duomenų valymo užduotis", "CleanupUserDataTaskDescription": "Iš medijos, kurios nebėra bent 90 dienų, išvalo visus naudotojo duomenis (žiūrėjimo būseną, mėgstamą būseną ir t. t.).", - "LyricDownloadFailureFromForItem": "Nepavyko atsisiųsti dainos žodžių iš {0}, skirto {1}", + "LyricDownloadFailureFromForItem": "Nepavyko atsisiųsti dainos teksto iš {0}, skirto {1}", "NameExtraBehindTheScenes": "Užkulisiuose", "NameExtraClip": "Klipas", "NameExtraDeletedScene": "Ištrinta scena", diff --git a/Emby.Server.Implementations/Localization/Core/lv.json b/Emby.Server.Implementations/Localization/Core/lv.json index 76fa9e3cf7..52f1eecbe4 100644 --- a/Emby.Server.Implementations/Localization/Core/lv.json +++ b/Emby.Server.Implementations/Localization/Core/lv.json @@ -108,5 +108,17 @@ "CleanupUserDataTask": "Lietotāju datu tīrīšanas uzdevums", "CleanupUserDataTaskDescription": "Notīra visus lietotāja datus (skatīšanās stāvokļus, favorītu statusi utt.) no medijiem, kas vairs nav pieejami vismaz 90 dienas.", "Original": "Oriģināls", - "LyricDownloadFailureFromForItem": "Dziesmu vārdi nevarēja tikt lejupielādēti no {0} priekš {1}" + "LyricDownloadFailureFromForItem": "Dziesmu vārdi nevarēja tikt lejupielādēti no {0} priekš {1}", + "NameExtraBehindTheScenes": "Aiz kadra", + "NameExtraClip": "Klips", + "NameExtraDeletedScene": "Izdzēsta aina", + "NameExtraFeaturette": "Īsfilma", + "NameExtraInterview": "Intervija", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Paraugs", + "NameExtraScene": "Aina", + "NameExtraShort": "Īsfilma", + "NameExtraThemeSong": "Motīvu dziesma", + "NameExtraThemeVideo": "Tēmas video", + "NameExtraTrailer": "Treileris" } diff --git a/Emby.Server.Implementations/Localization/Core/nb.json b/Emby.Server.Implementations/Localization/Core/nb.json index 752b74ec1c..735bc2c793 100644 --- a/Emby.Server.Implementations/Localization/Core/nb.json +++ b/Emby.Server.Implementations/Localization/Core/nb.json @@ -106,5 +106,15 @@ "TaskMoveTrickplayImagesDescription": "Flytter eksisterende Trickplay-filer i henhold til biblioteksinstillingene.", "TaskExtractMediaSegmentsDescription": "Trekker ut eller henter mediasegmenter fra plugins som støtter MediaSegment.", "CleanupUserDataTaskDescription": "Sletter all brukerdata (avspillings-status, favoritter osv.) fra innhold som har vært utilgjengelig i minst 90 dager.", - "CleanupUserDataTask": "Oppgave for opprydding av brukerdata" + "CleanupUserDataTask": "Oppgave for opprydding av brukerdata", + "NameExtraBehindTheScenes": "Bak kulissene", + "NameExtraDeletedScene": "Slettet scene", + "NameExtraFeaturette": "Presentasjonsfilm", + "NameExtraInterview": "Intervju", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Prøve", + "NameExtraScene": "Scene", + "NameExtraThemeSong": "Tema-låt", + "NameExtraThemeVideo": "Tema-video", + "NameExtraTrailer": "Trailer" } diff --git a/Emby.Server.Implementations/Localization/Core/ro.json b/Emby.Server.Implementations/Localization/Core/ro.json index 358c19881f..dccec8067d 100644 --- a/Emby.Server.Implementations/Localization/Core/ro.json +++ b/Emby.Server.Implementations/Localization/Core/ro.json @@ -111,5 +111,14 @@ "Original": "Original", "NameExtraBehindTheScenes": "În culise", "NameExtraClip": "Clip", - "NameExtraDeletedScene": "Scenă ștearsă" + "NameExtraDeletedScene": "Scenă ștearsă", + "NameExtraFeaturette": "Material bonus", + "NameExtraInterview": "Interviu", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Monstră", + "NameExtraScene": "Scenă", + "NameExtraShort": "Scurt", + "NameExtraThemeSong": "Audio de Fundal", + "NameExtraThemeVideo": "Video de Fundal", + "NameExtraTrailer": "Trailer" } diff --git a/Emby.Server.Implementations/Localization/Core/sl-SI.json b/Emby.Server.Implementations/Localization/Core/sl-SI.json index a1b5b714af..6ea625d66c 100644 --- a/Emby.Server.Implementations/Localization/Core/sl-SI.json +++ b/Emby.Server.Implementations/Localization/Core/sl-SI.json @@ -108,5 +108,17 @@ "CleanupUserDataTask": "Čiščenje uporabniških podatkov", "CleanupUserDataTaskDescription": "Izbriše vse uporabniške podatke (stanje ogleda, priljubljene itd.) za vsebine, ki že več kot 90 dni niso na voljo.", "LyricDownloadFailureFromForItem": "Besedila ni bilo mogoče prenesti iz {0} za {1}", - "Original": "Original" + "Original": "Original", + "NameExtraBehindTheScenes": "V zakulisju", + "NameExtraClip": "Klip", + "NameExtraDeletedScene": "Izbrisan prizor", + "NameExtraFeaturette": "Kratek dokumentarec o izdelavi filma", + "NameExtraInterview": "Intervju", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "Vzorec", + "NameExtraScene": "Prizor", + "NameExtraShort": "Kratki film", + "NameExtraThemeSong": "Tematska Pesem", + "NameExtraThemeVideo": "Tematski Video", + "NameExtraTrailer": "Napovednik" } diff --git a/Emby.Server.Implementations/Localization/Core/th.json b/Emby.Server.Implementations/Localization/Core/th.json index 716e3ae55d..77e526db74 100644 --- a/Emby.Server.Implementations/Localization/Core/th.json +++ b/Emby.Server.Implementations/Localization/Core/th.json @@ -22,20 +22,20 @@ "NewVersionIsAvailable": "เวอร์ชันใหม่ของเซิร์ฟเวอร์ Jellyfin พร้อมให้ดาวน์โหลดแล้ว", "NameSeasonUnknown": "ไม่ทราบซีซัน", "NameSeasonNumber": "ซีซัน {0}", - "NameInstallFailed": "การติดตั้ง {0} ล้มเหลว", + "NameInstallFailed": "ติดตั้ง {0} ไม่สำเร็จ", "MusicVideos": "มิวสิควิดีโอ", - "Music": "ดนตรี", + "Music": "เพลง", "Movies": "ภาพยนตร์", - "MixedContent": "เนื้อหาผสม", - "Latest": "ล่าสุด", - "LabelRunningTimeValue": "ผ่านไปแล้ว: {0}", - "LabelIpAddressValue": "ที่อยู่ IP: {0}", - "Inherit": "สืบทอด", - "HomeVideos": "โฮมวิดีโอ", - "HeaderNextUp": "ถัดไป", - "HeaderLiveTV": "ทีวีสด", - "HeaderFavoriteShows": "รายการที่ชื่นชอบ", - "HeaderFavoriteEpisodes": "ตอนที่ชื่นชอบ", + "MixedContent": "เนื้อหาหลากหลายประเภท", + "Latest": "มาใหม่ล่าสุด", + "LabelRunningTimeValue": "ความยาว: {0}", + "LabelIpAddressValue": "หมายเลข IP: {0}", + "Inherit": "ใช้ค่าเริ่มต้น", + "HomeVideos": "วิดีโอส่วนตัว", + "HeaderNextUp": "รายการถัดไป", + "HeaderLiveTV": "ทีวีถ่ายทอดสด", + "HeaderFavoriteShows": "รายการที่ชอบ", + "HeaderFavoriteEpisodes": "ตอนที่ชอบ", "HeaderContinueWatching": "ดูต่อ", "Genres": "ประเภท", "Folders": "โฟลเดอร์", @@ -107,6 +107,19 @@ "TaskMoveTrickplayImages": "ย้ายตำแหน่งเก็บภาพตัวอย่าง Trickplay", "CleanupUserDataTask": "ส่วนงานล้างข้อมูลผู้ใช้", "CleanupUserDataTaskDescription": "ล้างข้อมูลผู้ใช้ทั้งหมด (สถานะการรับชม สถานะรายการโปรด ฯลฯ) จากสื่อที่ไม่ได้ใช้งานแล้วอย่างน้อย 90 วัน", - "LyricDownloadFailureFromForItem": "ไม่สามารถดาวน์โหลดเนื้อเพลงจาก {0} สำหรับ {1}", - "Original": "ต้นฉบับ" + "LyricDownloadFailureFromForItem": "ดาวน์โหลดเนื้อเพลงจาก {0} สำหรับ {1} ไม่สำเร็จ", + "Original": "ต้นฉบับ", + "NameExtraBehindTheScenes": "เบื้องหลังการถ่ายทำ", + "NameExtraClip": "คลิปวิดีโอ", + "NameExtraDeletedScene": "ฉากที่ถูกตัดออก", + "NameExtraFeaturette": "คลิปสั้นพิเศษ", + "NameExtraInterview": "บทสัมภาษณ์", + "NameExtraNumbered": "{0} {1}", + "NameExtraSample": "ตัวอย่าง", + "NameExtraScene": "ฉาก", + "NameExtraShort": "ภาพยนตร์สั้น", + "NameExtraThemeSong": "เพลงประกอบ", + "NameExtraThemeVideo": "วิดีโอธีม", + "NameExtraTrailer": "ตัวอย่างภาพยนตร์", + "NameExtraUnknown": "เนื้อหาพิเศษ" } diff --git a/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs b/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs index 8d133dc074..687947616f 100644 --- a/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs +++ b/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs @@ -11,7 +11,7 @@ using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.ScheduledTasks.Tasks; /// <summary> -/// Optimizes Jellyfin's database by issuing a VACUUM command. +/// Optimizes Jellyfin's database by issuing VACUUM and ANALYZE commands. /// </summary> public class OptimizeDatabaseTask : IScheduledTask, IConfigurableScheduledTask { @@ -82,7 +82,7 @@ public class OptimizeDatabaseTask : IScheduledTask, IConfigurableScheduledTask return; } - _logger.LogInformation("Optimizing and vacuuming jellyfin.db..."); + _logger.LogInformation("Vacuuming and analyzing jellyfin.db..."); try { diff --git a/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs b/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs index 42835d7ad0..092a621bfc 100644 --- a/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs +++ b/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Emby.Server.Implementations.Library.Validators; using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; @@ -29,6 +30,7 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask private readonly IDbContextFactory<JellyfinDbContext> _dbContextFactory; private readonly IFileSystem _fileSystem; private readonly ILogger<PeopleValidationTask> _logger; + private readonly ILogger<PeopleValidator> _validatorLogger; private readonly IItemTypeLookup _itemTypeLookup; /// <summary> @@ -39,6 +41,7 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask /// <param name="dbContextFactory">Instance of the <see cref="IDbContextFactory{TContext}"/> interface.</param> /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> /// <param name="logger">Instance of the <see cref="ILogger{PeopleValidationTask}"/> interface.</param> + /// <param name="validatorLogger">Instance of the <see cref="ILogger{PeopleValidator}"/> interface.</param> /// <param name="itemTypeLookup">Instance of the <see cref="IItemTypeLookup"/> interface.</param> public PeopleValidationTask( ILibraryManager libraryManager, @@ -46,6 +49,7 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask IDbContextFactory<JellyfinDbContext> dbContextFactory, IFileSystem fileSystem, ILogger<PeopleValidationTask> logger, + ILogger<PeopleValidator> validatorLogger, IItemTypeLookup itemTypeLookup) { _libraryManager = libraryManager; @@ -53,6 +57,7 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask _dbContextFactory = dbContextFactory; _fileSystem = fileSystem; _logger = logger; + _validatorLogger = validatorLogger; _itemTypeLookup = itemTypeLookup; } @@ -165,7 +170,9 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask // Phase 2: Validate people (33-66%). Runs after orphaned PeopleBaseItemMap entries are // cleaned up above, so dead people are removed in a single pass instead of requiring a second run. IProgress<double> validateProgress = new Progress<double>((val) => progress.Report((val / 3) + 33)); - await _libraryManager.ValidatePeopleAsync(validateProgress, cancellationToken).ConfigureAwait(false); + await new PeopleValidator(_libraryManager, _validatorLogger) + .Run(validateProgress, cancellationToken) + .ConfigureAwait(false); // Phase 3: Refresh images for people missing them (66-100%) IProgress<double> refreshProgress = new Progress<double>((val) => progress.Report((val / 3) + 66)); diff --git a/Emby.Server.Implementations/SyncPlay/Group.cs b/Emby.Server.Implementations/SyncPlay/Group.cs index 38a0018a70..923bfc67aa 100644 --- a/Emby.Server.Implementations/SyncPlay/Group.cs +++ b/Emby.Server.Implementations/SyncPlay/Group.cs @@ -91,6 +91,18 @@ namespace Emby.Server.Implementations.SyncPlay public long DefaultPing { get; } = 500; /// <summary> + /// Gets the maximum ping, in milliseconds, accepted from a session. + /// </summary> + /// <remarks> + /// Pings are reported by clients and are scaled into the delays used to schedule playback, + /// so an unbounded value lets a single session push the whole group's resume point + /// arbitrarily far out, or overflow the arithmetic entirely. Anything above this is not a + /// usable measurement for synchronisation. + /// </remarks> + /// <value>The maximum ping.</value> + public long MaxPing { get; } = 10000; + + /// <summary> /// Gets the maximum time offset error accepted for dates reported by clients, in milliseconds. /// </summary> /// <value>The maximum time offset error.</value> @@ -438,7 +450,7 @@ namespace Emby.Server.Implementations.SyncPlay { if (_participants.TryGetValue(session.Id, out GroupMember value)) { - value.Ping = ping; + value.Ping = Math.Clamp(ping, 0, MaxPing); } } @@ -451,7 +463,9 @@ namespace Emby.Server.Implementations.SyncPlay max = Math.Max(max, session.Ping); } - return max; + // A group with no participants has no ping to report. Returning long.MinValue would + // overflow the callers that scale this value into ticks, so fall back to the default. + return max == long.MinValue ? DefaultPing : max; } /// <inheritdoc /> diff --git a/Emby.Server.Implementations/SyncPlay/SyncPlayManager.cs b/Emby.Server.Implementations/SyncPlay/SyncPlayManager.cs index b45d754554..b88ee33358 100644 --- a/Emby.Server.Implementations/SyncPlay/SyncPlayManager.cs +++ b/Emby.Server.Implementations/SyncPlay/SyncPlayManager.cs @@ -181,8 +181,8 @@ namespace Emby.Server.Implementations.SyncPlay { if (existingGroup.GroupId.Equals(request.GroupId)) { - // Restore session. - UpdateSessionsCounter(session.UserId, 1); + // Restore session. The session is already in the group and has already + // been counted, so the counter must not be incremented a second time. group.SessionJoin(session, request, cancellationToken); return; } @@ -332,8 +332,11 @@ namespace Emby.Server.Implementations.SyncPlay // Group lock required as Group is not thread-safe. lock (group) { - // Make sure that session still belongs to this group. - if (_sessionToGroupMap.TryGetValue(session.Id, out var checkGroup) && !checkGroup.GroupId.Equals(group.GroupId)) + // Make sure that session still belongs to this group. The lookup can fail + // outright when the session left while this request was waiting on the group + // lock, which is exactly the case this re-check exists to catch. + if (!_sessionToGroupMap.TryGetValue(session.Id, out var checkGroup) + || !checkGroup.GroupId.Equals(group.GroupId)) { // Drop request. return; @@ -400,7 +403,7 @@ namespace Emby.Server.Implementations.SyncPlay // Update sessions counter. var newSessionsCounter = _activeUsers.AddOrUpdate( userId, - 1, + toAdd, (_, sessionsCounter) => sessionsCounter + toAdd); // Should never happen. diff --git a/Jellyfin.Api/Controllers/LibraryStructureController.cs b/Jellyfin.Api/Controllers/LibraryStructureController.cs index 5c596c21b9..65bfe25d21 100644 --- a/Jellyfin.Api/Controllers/LibraryStructureController.cs +++ b/Jellyfin.Api/Controllers/LibraryStructureController.cs @@ -16,6 +16,7 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Entities; using Microsoft.AspNetCore.Authorization; @@ -34,6 +35,7 @@ public class LibraryStructureController : BaseJellyfinApiController private readonly IServerApplicationPaths _appPaths; private readonly ILibraryManager _libraryManager; private readonly ILibraryMonitor _libraryMonitor; + private readonly IDirectoryService _directoryService; /// <summary> /// Initializes a new instance of the <see cref="LibraryStructureController"/> class. @@ -41,14 +43,17 @@ public class LibraryStructureController : BaseJellyfinApiController /// <param name="serverConfigurationManager">Instance of <see cref="IServerConfigurationManager"/> interface.</param> /// <param name="libraryManager">Instance of <see cref="ILibraryManager"/> interface.</param> /// <param name="libraryMonitor">Instance of <see cref="ILibraryMonitor"/> interface.</param> + /// <param name="directoryService">Instance of <see cref="IDirectoryService"/> interface.</param> public LibraryStructureController( IServerConfigurationManager serverConfigurationManager, ILibraryManager libraryManager, - ILibraryMonitor libraryMonitor) + ILibraryMonitor libraryMonitor, + IDirectoryService directoryService) { _appPaths = serverConfigurationManager.ApplicationPaths; _libraryManager = libraryManager; _libraryMonitor = libraryMonitor; + _directoryService = directoryService; } /// <summary> @@ -178,11 +183,11 @@ public class LibraryStructureController : BaseJellyfinApiController var tempPath = Path.Combine( rootFolderPath, Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture)); - Directory.Move(currentPath, tempPath); + _directoryService.Move(currentPath, tempPath); currentPath = tempPath; } - Directory.Move(currentPath, newPath); + _directoryService.Move(currentPath, newPath); } finally { diff --git a/Jellyfin.Api/Controllers/SubtitleController.cs b/Jellyfin.Api/Controllers/SubtitleController.cs index e5df873f5b..c4851091c1 100644 --- a/Jellyfin.Api/Controllers/SubtitleController.cs +++ b/Jellyfin.Api/Controllers/SubtitleController.cs @@ -557,7 +557,7 @@ public class SubtitleController : BaseJellyfinApiController if (!string.IsNullOrEmpty(fallbackFontPath)) { var fontFile = _fileSystem.GetFiles(fallbackFontPath) - .First(i => string.Equals(i.Name, name, StringComparison.OrdinalIgnoreCase)); + .FirstOrDefault(i => string.Equals(i.Name, name, StringComparison.OrdinalIgnoreCase)); var fileSize = fontFile?.Length; if (fontFile is not null && fileSize is not null && fileSize > 0) diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs index b09b279699..4995d37f4b 100644 --- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs +++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs @@ -458,6 +458,7 @@ public class DynamicHlsHelper { case VideoRangeType.HLG: case VideoRangeType.DOVIWithHLG: + case VideoRangeType.DOVIInvalid when string.Equals(state.VideoStream.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase): builder.Append(",VIDEO-RANGE=HLG"); break; default: diff --git a/Jellyfin.Data/Enums/VideoRangeType.cs b/Jellyfin.Data/Enums/VideoRangeType.cs index ce232d73c3..e7cc340d21 100644 --- a/Jellyfin.Data/Enums/VideoRangeType.cs +++ b/Jellyfin.Data/Enums/VideoRangeType.cs @@ -61,8 +61,9 @@ public enum VideoRangeType DOVIWithELHDR10Plus, /// <summary> - /// Dolby Vision with invalid configuration. e.g. Profile 8 compat id 6. - /// When using this range, the server would assume the video is still HDR10 after removing the Dolby Vision metadata. + /// Dolby Vision with invalid configuration, e.g. Profile 8 compat id 6 or inconsistent base-layer color metadata. + /// The base layer is classified as HDR only when its transfer characteristics signal PQ or HLG. + /// Otherwise, it is classified as SDR. /// </summary> DOVIInvalid, diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs index 2ff8131ade..70e4ca3b1d 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs @@ -121,20 +121,22 @@ public sealed partial class BaseItemRepository { using var context = _dbProvider.CreateDbContext(); - var query = context.ItemValuesMap - .AsNoTracking() - .Where(e => itemValueTypes.Any(w => w == e.ItemValue.Type)); + var maps = context.ItemValuesMap.AsNoTracking(); if (withItemTypes.Count > 0) { - query = query.Where(e => withItemTypes.Contains(e.Item.Type)); + maps = maps.Where(e => withItemTypes.Contains(e.Item.Type)); } if (excludeItemTypes.Count > 0) { - query = query.Where(e => !excludeItemTypes.Contains(e.Item.Type)); + maps = maps.Where(e => !excludeItemTypes.Contains(e.Item.Type)); } - return query.Select(e => e.ItemValue) + return context.ItemValues + .AsNoTracking() + .WhereOneOrMany(itemValueTypes, e => e.Type) + .Where(e => maps.Any(m => m.ItemValueId == e.ItemValueId)) + .Select(e => new { e.CleanValue, e.Value }) .GroupBy(e => e.CleanValue) .Select(g => g.Min(v => v.Value)!) .ToArray(); diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs index c9e08b1b5d..1ed10cce2b 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs @@ -626,18 +626,26 @@ public sealed partial class BaseItemRepository .ToArray(); var tags = context.ItemValuesMap - .Where(ivm => ivm.ItemValue.Type == ItemValueType.Tags) - .Where(ivm => matchingItemIds.Contains(ivm.ItemId)) - .Select(ivm => ivm.ItemValue) + .Join( + context.ItemValues, + ivm => ivm.ItemValueId, + iv => iv.ItemValueId, + (ivm, iv) => new { ivm.ItemId, iv.Type, iv.CleanValue, iv.Value }) + .Where(iv => iv.Type == ItemValueType.Tags) + .Where(iv => matchingItemIds.Contains(iv.ItemId)) .GroupBy(iv => iv.CleanValue) .Select(g => g.Min(iv => iv.Value)) .OrderBy(t => t) .ToArray(); var genres = context.ItemValuesMap - .Where(ivm => ivm.ItemValue.Type == ItemValueType.Genre) - .Where(ivm => matchingItemIds.Contains(ivm.ItemId)) - .Select(ivm => ivm.ItemValue) + .Join( + context.ItemValues, + ivm => ivm.ItemValueId, + iv => iv.ItemValueId, + (ivm, iv) => new { ivm.ItemId, iv.Type, iv.CleanValue, iv.Value }) + .Where(iv => iv.Type == ItemValueType.Genre) + .Where(iv => matchingItemIds.Contains(iv.ItemId)) .GroupBy(iv => iv.CleanValue) .Select(g => g.Min(iv => iv.Value)) .OrderBy(g => g) diff --git a/Jellyfin.Server.Implementations/Item/ItemPersistenceService.cs b/Jellyfin.Server.Implementations/Item/ItemPersistenceService.cs index efff3457a3..c8672e189b 100644 --- a/Jellyfin.Server.Implementations/Item/ItemPersistenceService.cs +++ b/Jellyfin.Server.Implementations/Item/ItemPersistenceService.cs @@ -176,14 +176,6 @@ public class ItemPersistenceService : IItemPersistenceService var context = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); await using (context.ConfigureAwait(false)) { - if (!await context.BaseItems - .AnyAsync(bi => bi.Id == item.Id, cancellationToken) - .ConfigureAwait(false)) - { - _logger.LogWarning("Unable to save ImageInfo for non existing BaseItem"); - return; - } - await context.BaseItemImageInfos .Where(e => e.ItemId == item.Id) .ExecuteDeleteAsync(cancellationToken) @@ -193,7 +185,26 @@ public class ItemPersistenceService : IItemPersistenceService .AddRangeAsync(images, cancellationToken) .ConfigureAwait(false); - await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + try + { + await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + catch (DbUpdateException) + { + // Checking that the item exists before writing leaves a gap a scan can delete it + // through, turning the insert into a foreign key violation that fails the whole + // refresh instead of the no-op intended here. Let the insert be the check: it is the + // only point at which the answer cannot go stale. Nothing is orphaned by the delete + // above, because deleting the item cascades to its images anyway. + if (await context.BaseItems + .AnyAsync(bi => bi.Id == item.Id, cancellationToken) + .ConfigureAwait(false)) + { + throw; + } + + _logger.LogWarning("Unable to save ImageInfo for non existing BaseItem {ItemId}", item.Id); + } } } diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationService.cs b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs index beafc3916f..a5b6bc4604 100644 --- a/Jellyfin.Server/Migrations/JellyfinMigrationService.cs +++ b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs @@ -183,7 +183,13 @@ internal class JellyfinMigrationService } } - public async Task MigrateStepAsync(JellyfinMigrationStageTypes stage, IServiceProvider? serviceProvider) + /// <summary> + /// Runs all pending migrations of the requested stage. + /// </summary> + /// <param name="stage">The stage to migrate.</param> + /// <param name="serviceProvider">The service provider handed to the migrations.</param> + /// <returns>A value indicating whether at least one migration has been applied.</returns> + public async Task<bool> MigrateStepAsync(JellyfinMigrationStageTypes stage, IServiceProvider serviceProvider) { var logger = _startupLogger.With(_loggerFactory.CreateLogger<JellyfinMigrationService>()).BeginGroup($"Migrate stage {stage}."); ICollection<CodeMigration> migrationStage = (Migrations.FirstOrDefault(e => e.Stage == stage) as ICollection<CodeMigration>) ?? []; @@ -297,6 +303,8 @@ internal class JellyfinMigrationService completedMigrations++; } + + return completedMigrations > 0; } } @@ -445,10 +453,10 @@ internal class JellyfinMigrationService private class InternalCodeMigration : IInternalMigration { private readonly CodeMigration _codeMigration; - private readonly IServiceProvider? _serviceProvider; + private readonly IServiceProvider _serviceProvider; private JellyfinDbContext _dbContext; - public InternalCodeMigration(CodeMigration codeMigration, IServiceProvider? serviceProvider, JellyfinDbContext dbContext) + public InternalCodeMigration(CodeMigration codeMigration, IServiceProvider serviceProvider, JellyfinDbContext dbContext) { _codeMigration = codeMigration; _serviceProvider = serviceProvider; diff --git a/Jellyfin.Server/Migrations/Routines/20250618010000_MigrateLibraryUserData.cs b/Jellyfin.Server/Migrations/Routines/20250618010000_MigrateLibraryUserData.cs index 8a0a1741f1..291de23b2e 100644 --- a/Jellyfin.Server/Migrations/Routines/20250618010000_MigrateLibraryUserData.cs +++ b/Jellyfin.Server/Migrations/Routines/20250618010000_MigrateLibraryUserData.cs @@ -29,7 +29,7 @@ internal class MigrateLibraryUserData : IAsyncMigrationRoutine private readonly IDbContextFactory<JellyfinDbContext> _provider; public MigrateLibraryUserData( - IStartupLogger<MigrateLibraryDb> startupLogger, + IStartupLogger<MigrateLibraryUserData> startupLogger, IDbContextFactory<JellyfinDbContext> provider, IServerApplicationPaths paths) { diff --git a/Jellyfin.Server/Migrations/Routines/20250730215000_ReseedFolderFlag.cs b/Jellyfin.Server/Migrations/Routines/20250730215000_ReseedFolderFlag.cs index 502763ac09..c8ee44a670 100644 --- a/Jellyfin.Server/Migrations/Routines/20250730215000_ReseedFolderFlag.cs +++ b/Jellyfin.Server/Migrations/Routines/20250730215000_ReseedFolderFlag.cs @@ -24,7 +24,7 @@ internal class ReseedFolderFlag : IAsyncMigrationRoutine private readonly IDbContextFactory<JellyfinDbContext> _provider; public ReseedFolderFlag( - IStartupLogger<MigrateLibraryDb> startupLogger, + IStartupLogger<ReseedFolderFlag> startupLogger, IDbContextFactory<JellyfinDbContext> provider, IServerApplicationPaths paths) { diff --git a/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs b/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs index 4a6e74c229..8f239007d8 100644 --- a/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs +++ b/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs @@ -22,6 +22,11 @@ namespace Jellyfin.Server.Migrations.Routines; [JellyfinMigrationBackup(JellyfinDb = true)] internal class MigrateLinkedChildren : IDatabaseMigrationRoutine { + private const int ParseProgressLogStep = 25_000; + private const int FileCheckProgressLogStep = 10_000; + private const int ResolveProgressLogStep = 10_000; + private const int DeleteProgressLogStep = 25; + private readonly ILogger<MigrateLinkedChildren> _logger; private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; private readonly ILibraryManager _libraryManager; @@ -85,7 +90,6 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine var droppedChildren = 0; var linkedChildrenToAdd = new List<LinkedChildEntity>(); var processedCount = 0; - const int progressLogStep = 1000; var totalItems = itemsWithData.Count; foreach (var item in itemsWithData) @@ -95,7 +99,7 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine continue; } - if (processedCount > 0 && processedCount % progressLogStep == 0) + if (processedCount > 0 && processedCount % ParseProgressLogStep == 0) { _logger.LogInformation("Processing LinkedChildren: {Processed}/{Total} items", processedCount, totalItems); } @@ -311,11 +315,7 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine _logger.LogInformation("Found {Count} wrong-type alternate version items to remove.", wrongTypeChildIds.Count); - var itemsToDelete = wrongTypeChildIds - .Select(id => _libraryManager.GetItemById(id)) - .Where(item => item is not null) - .ToList(); - var deleted = DeleteItems(itemsToDelete!); + var deleted = ResolveAndDeleteItems(wrongTypeChildIds, "wrong-type alternate version items"); _logger.LogInformation("Removed {Count} wrong-type alternate version items. They will be recreated with the correct type on next library scan.", deleted); } @@ -342,11 +342,7 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine _logger.LogInformation("Found {Count} orphaned alternate version BaseItems to remove.", orphanedVersionIds.Count); - var itemsToDelete = orphanedVersionIds - .Select(id => _libraryManager.GetItemById(id)) - .Where(item => item is not null) - .ToList(); - var deleted = DeleteItems(itemsToDelete!); + var deleted = ResolveAndDeleteItems(orphanedVersionIds, "orphaned alternate version BaseItems"); _logger.LogInformation("Removed {Count} orphaned alternate version BaseItems.", deleted); } @@ -371,11 +367,7 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine _logger.LogInformation("Found {Count} items from deleted libraries to remove.", orphanedIds.Count); - var itemsToDelete = orphanedIds - .Select(id => _libraryManager.GetItemById(id)) - .Where(item => item is not null) - .ToList(); - var deleted = DeleteItems(itemsToDelete!); + var deleted = ResolveAndDeleteItems(orphanedIds, "items from deleted libraries"); _logger.LogInformation("Removed {Count} items from deleted libraries.", deleted); } @@ -427,8 +419,23 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine var skippedUnrootedItems = 0; var staleIds = new List<Guid>(); + var checkedCount = 0; + _logger.LogInformation("Checking {Total} items for missing files.", itemsWithPaths.Count); + foreach (var item in itemsWithPaths) { + // A miss on offline storage can block for the mount timeout, so report while scanning. + if (checkedCount > 0 && checkedCount % FileCheckProgressLogStep == 0) + { + _logger.LogInformation( + "Checking for missing files: {Checked}/{Total} items, {Stale} stale so far.", + checkedCount, + itemsWithPaths.Count, + staleIds.Count); + } + + checkedCount++; + // Expand virtual path placeholders (%AppDataPath%, %MetadataPath%) to real paths var path = _appHost.ExpandVirtualPath(item.Path!); @@ -482,16 +489,47 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine _logger.LogInformation("Found {Count} stale items to remove.", staleIds.Count); - var itemsToDelete = staleIds - .Select(id => _libraryManager.GetItemById(id)) - .Where(item => item is not null) - .ToList(); - var deleted = DeleteItems(itemsToDelete!); + var deleted = ResolveAndDeleteItems(staleIds, "items with missing files"); _logger.LogInformation("Removed {Count} stale items.", deleted); } - private int DeleteItems(IReadOnlyCollection<BaseItem> items) + private int ResolveAndDeleteItems(IReadOnlyCollection<Guid> ids, string description) + { + if (ids.Count == 0) + { + return 0; + } + + return DeleteItems(ResolveItems(ids, description), description); + } + + private List<BaseItem> ResolveItems(IReadOnlyCollection<Guid> ids, string description) + { + // Each lookup is a separate repository read; cached ones are fast, so this only reports + // once a set is large enough for the reads to add up to a noticeable stretch. + var items = new List<BaseItem>(ids.Count); + var processed = 0; + foreach (var id in ids) + { + if (processed > 0 && processed % ResolveProgressLogStep == 0) + { + _logger.LogInformation("Loading {Description}: {Processed}/{Total} items", description, processed, ids.Count); + } + + processed++; + + var item = _libraryManager.GetItemById(id); + if (item is not null) + { + items.Add(item); + } + } + + return items; + } + + private int DeleteItems(IReadOnlyCollection<BaseItem> items, string description) { if (items.Count == 0) { @@ -500,8 +538,16 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine var options = new DeleteOptions { DeleteFileLocation = false, DeleteFromExternalProvider = false }; var deleted = 0; + var processed = 0; foreach (var item in items) { + if (processed > 0 && processed % DeleteProgressLogStep == 0) + { + _logger.LogInformation("Removing {Description}: {Processed}/{Total} items", description, processed, items.Count); + } + + processed++; + try { _libraryManager.DeleteItem(item, options); diff --git a/Jellyfin.Server/Migrations/Stages/CodeMigration.cs b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs index 971b47608f..71706811b8 100644 --- a/Jellyfin.Server/Migrations/Stages/CodeMigration.cs +++ b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs @@ -4,8 +4,6 @@ using System.Threading; using System.Threading.Tasks; using Jellyfin.Server.ServerSetupApp; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Stages; @@ -22,66 +20,45 @@ internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute meta return Metadata.Order.ToString("yyyyMMddHHmmsss", CultureInfo.InvariantCulture) + "_" + Metadata.Name!; } - private IServiceCollection MigrationServices(IServiceProvider serviceProvider, IStartupLogger logger) + public async Task Perform(IServiceProvider serviceProvider, IStartupLogger logger, CancellationToken cancellationToken) { - var childServiceCollection = new ServiceCollection() - .AddSingleton(serviceProvider) - .AddSingleton(logger) - .AddSingleton(typeof(IStartupLogger<>), typeof(NestedStartupLogger<>)) - .AddSingleton<StartupLogTopic>(logger.Topic!); + if (!IsMigrationRoutine(MigrationType)) + { + throw new InvalidOperationException($"The type {MigrationType} does not implement either IMigrationRoutine or IAsyncMigrationRoutine and is not a valid migration type"); + } - foreach (ServiceDescriptor service in serviceProvider.GetRequiredService<IServiceCollection>()) + // The routine runs against a scope of the applications own container. Copying the application service + // descriptors into a child container instead would make that child container the owner of every singleton it + // forwards, so disposing it after the migration would also dispose the applications own instance of services + // like the ProviderManager and leave the server broken until the next restart. + var scope = serviceProvider.CreateAsyncScope(); + await using (scope.ConfigureAwait(false)) { - if (service.Lifetime == ServiceLifetime.Singleton && !service.ServiceType.IsGenericTypeDefinition) + // Nests everything the routine logs through an injected IStartupLogger under the migrations own topic. + using (StartupLogger.BeginAmbientTopic(logger.Topic)) { - childServiceCollection.AddSingleton(service.ServiceType, _ => serviceProvider.GetService(service.ServiceType)!); - continue; + await RunAsync(ActivatorUtilities.CreateInstance(scope.ServiceProvider, MigrationType), cancellationToken).ConfigureAwait(false); } - - childServiceCollection.Add(service); } - - return childServiceCollection; } - public async Task Perform(IServiceProvider? serviceProvider, IStartupLogger logger, CancellationToken cancellationToken) - { + // The obsolete IMigrationRoutine is still implemented by every routine that predates the async interface, so + // the members that have to touch it are grouped here behind a single suppression. #pragma warning disable CS0618 // Type or member is obsolete - if (typeof(IMigrationRoutine).IsAssignableFrom(MigrationType)) - { - if (serviceProvider is null) - { - ((IMigrationRoutine)Activator.CreateInstance(MigrationType)!).Perform(); - } - else - { - using var migrationServices = MigrationServices(serviceProvider, logger).BuildServiceProvider(); - ((IMigrationRoutine)ActivatorUtilities.CreateInstance(migrationServices, MigrationType)).Perform(); -#pragma warning restore CS0618 // Type or member is obsolete - } - } - else if (typeof(IAsyncMigrationRoutine).IsAssignableFrom(MigrationType)) - { - if (serviceProvider is null) - { - await ((IAsyncMigrationRoutine)Activator.CreateInstance(MigrationType)!).PerformAsync(cancellationToken).ConfigureAwait(false); - } - else - { - using var migrationServices = MigrationServices(serviceProvider, logger).BuildServiceProvider(); - await ((IAsyncMigrationRoutine)ActivatorUtilities.CreateInstance(migrationServices, MigrationType)).PerformAsync(cancellationToken).ConfigureAwait(false); - } - } - else - { - throw new InvalidOperationException($"The type {MigrationType} does not implement either IMigrationRoutine or IAsyncMigrationRoutine and is not a valid migration type"); - } + private static bool IsMigrationRoutine(Type migrationType) + { + return typeof(IMigrationRoutine).IsAssignableFrom(migrationType) || typeof(IAsyncMigrationRoutine).IsAssignableFrom(migrationType); } - private class NestedStartupLogger<TCategory> : StartupLogger<TCategory> + private static async Task RunAsync(object routine, CancellationToken cancellationToken) { - public NestedStartupLogger(ILogger logger, StartupLogTopic topic) : base(logger, topic) + if (routine is IMigrationRoutine migrationRoutine) { + migrationRoutine.Perform(); + return; } + + await ((IAsyncMigrationRoutine)routine).PerformAsync(cancellationToken).ConfigureAwait(false); } +#pragma warning restore CS0618 // Type or member is obsolete } diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs index 12f92efb35..35eaff6532 100644 --- a/Jellyfin.Server/Program.cs +++ b/Jellyfin.Server/Program.cs @@ -61,6 +61,7 @@ namespace Jellyfin.Server private static ILogger _logger = NullLogger.Instance; private static bool _restartOnShutdown; private static IStartupLogger<JellyfinMigrationService>? _migrationLogger; + private static bool _optimizeDatabaseAfterMigration; private static string? _restoreFromBackup; /// <summary> @@ -180,9 +181,7 @@ namespace Jellyfin.Server }) .ConfigureAppConfiguration(config => config.ConfigureAppConfiguration(options, appPaths, startupConfig)) .UseSerilog() - .ConfigureServices(e => e - .RegisterStartupLogger() - .AddSingleton<IServiceCollection>(e)) + .ConfigureServices(e => e.RegisterStartupLogger()) .Build(); /* @@ -209,14 +208,15 @@ namespace Jellyfin.Server await jellyfinMigrationService.PrepareSystemForMigration(_logger).ConfigureAwait(false); // "Preparing migrations" carries through the DB read; per-migration progress is reported // as "Running migration X of Y" from inside the step once the pending set is known. - await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.CoreInitialisation, appHost.ServiceProvider).ConfigureAwait(false); + _optimizeDatabaseAfterMigration |= await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.CoreInitialisation, appHost.ServiceProvider).ConfigureAwait(false); SetupServer.ReportActivity(StartupActivity.InitializingServices); await appHost.InitializeServices(startupConfig).ConfigureAwait(false); _appHost = appHost; - await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.AppInitialisation, appHost.ServiceProvider).ConfigureAwait(false); + _optimizeDatabaseAfterMigration |= await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.AppInitialisation, appHost.ServiceProvider).ConfigureAwait(false); await jellyfinMigrationService.CleanupSystemAfterMigration(_logger).ConfigureAwait(false); + await OptimizeDatabaseAfterMigrationAsync(appHost.ServiceProvider).ConfigureAwait(false); try { configurationCompleted = true; @@ -271,12 +271,11 @@ namespace Jellyfin.Server // Don't throw additional exception if startup failed. if (appHost.ServiceProvider is not null) { - _logger.LogInformation("Running query planner optimizations in the database... This might take a while"); + _logger.LogInformation("Optimizing the database... This might take a while"); + // Deliberately untimed: a truncated optimization leaves the statistics incomplete. var databaseProvider = appHost.ServiceProvider.GetRequiredService<IJellyfinDatabaseProvider>(); - using var shutdownSource = new CancellationTokenSource(); - shutdownSource.CancelAfter((int)TimeSpan.FromSeconds(60).TotalMicroseconds); - await databaseProvider.RunShutdownTask(shutdownSource.Token).ConfigureAwait(false); + await databaseProvider.RunShutdownTask(CancellationToken.None).ConfigureAwait(false); } _appHost = null; @@ -307,14 +306,13 @@ namespace Jellyfin.Server .AddSingleton<ServerApplicationPaths>(appPaths) .RegisterStartupLogger(); - migrationStartupServiceProvider.AddSingleton(migrationStartupServiceProvider); var startupService = migrationStartupServiceProvider.BuildServiceProvider(); PrepareDatabaseProvider(startupService); var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(startupService); await jellyfinMigrationService.CheckFirstTimeRunOrMigration(appPaths, startupOptions).ConfigureAwait(false); - await jellyfinMigrationService.MigrateStepAsync(Migrations.Stages.JellyfinMigrationStageTypes.PreInitialisation, startupService).ConfigureAwait(false); + _optimizeDatabaseAfterMigration |= await jellyfinMigrationService.MigrateStepAsync(Migrations.Stages.JellyfinMigrationStageTypes.PreInitialisation, startupService).ConfigureAwait(false); } /// <summary> @@ -329,7 +327,32 @@ namespace Jellyfin.Server public static async Task ApplyCoreMigrationsAsync(IServiceProvider serviceProvider, Migrations.Stages.JellyfinMigrationStageTypes jellyfinMigrationStage) { var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(serviceProvider, _migrationLogger!); - await jellyfinMigrationService.MigrateStepAsync(jellyfinMigrationStage, serviceProvider).ConfigureAwait(false); + _optimizeDatabaseAfterMigration |= await jellyfinMigrationService.MigrateStepAsync(jellyfinMigrationStage, serviceProvider).ConfigureAwait(false); + } + + private static async Task OptimizeDatabaseAfterMigrationAsync(IServiceProvider serviceProvider) + { + if (!_optimizeDatabaseAfterMigration) + { + return; + } + + // Reset first: a restart runs no migrations and must not optimize again. + _optimizeDatabaseAfterMigration = false; + SetupServer.ReportActivity(StartupActivity.OptimizingDatabase); + _logger.LogInformation("Migrations have been applied, optimizing the database... This might take a while"); + + try + { + // Deliberately untimed: incomplete statistics are worse than a slow start. + var databaseProvider = serviceProvider.GetRequiredService<IJellyfinDatabaseProvider>(); + await databaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + // A missed optimization only costs performance, so never fail startup over this. + _logger.LogError(ex, "Error while optimizing the database after migration"); + } } /// <summary> diff --git a/Jellyfin.Server/ServerSetupApp/StartupActivity.cs b/Jellyfin.Server/ServerSetupApp/StartupActivity.cs index 888cc617d4..abfc5bc8ba 100644 --- a/Jellyfin.Server/ServerSetupApp/StartupActivity.cs +++ b/Jellyfin.Server/ServerSetupApp/StartupActivity.cs @@ -27,6 +27,9 @@ public static class StartupActivity /// <summary>Bringing up core services and plugins.</summary> public const string InitializingServices = "Initializing services"; + /// <summary>Refreshing the database statistics after migrations have run.</summary> + public const string OptimizingDatabase = "Optimizing database"; + /// <summary>Running the final startup tasks.</summary> public const string FinishingStartup = "Finishing startup"; diff --git a/Jellyfin.Server/ServerSetupApp/StartupLogger.cs b/Jellyfin.Server/ServerSetupApp/StartupLogger.cs index 0121854ce3..b72b0c0eab 100644 --- a/Jellyfin.Server/ServerSetupApp/StartupLogger.cs +++ b/Jellyfin.Server/ServerSetupApp/StartupLogger.cs @@ -1,5 +1,6 @@ using System; using System.Globalization; +using System.Threading; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -8,6 +9,8 @@ namespace Jellyfin.Server.ServerSetupApp; /// <inheritdoc/> public class StartupLogger : IStartupLogger { + private static readonly AsyncLocal<StartupLogTopic?> _ambientTopic = new(); + private readonly StartupLogTopic? _topic; /// <summary> @@ -17,6 +20,7 @@ public class StartupLogger : IStartupLogger public StartupLogger(ILogger logger) { BaseLogger = logger; + _topic = _ambientTopic.Value; } /// <summary> @@ -39,6 +43,18 @@ public class StartupLogger : IStartupLogger /// </summary> protected ILogger BaseLogger { get; set; } + /// <summary> + /// Makes <paramref name="topic"/> the topic that loggers created on this execution context attach to. + /// </summary> + /// <param name="topic">The topic to nest newly created loggers under.</param> + /// <returns>A scope that restores the previously ambient topic when disposed.</returns> + internal static IDisposable BeginAmbientTopic(StartupLogTopic? topic) + { + var scope = new AmbientTopicScope(_ambientTopic.Value); + _ambientTopic.Value = topic; + return scope; + } + /// <inheritdoc/> public IStartupLogger BeginGroup(FormattableString logEntry) { @@ -121,4 +137,19 @@ public class StartupLogger : IStartupLogger Topic.Children.Add(startupEntry); } } + + private sealed class AmbientTopicScope : IDisposable + { + private readonly StartupLogTopic? _previous; + + public AmbientTopicScope(StartupLogTopic? previous) + { + _previous = previous; + } + + public void Dispose() + { + _ambientTopic.Value = _previous; + } + } } diff --git a/MediaBrowser.Controller/Entities/InternalItemsQuery.cs b/MediaBrowser.Controller/Entities/InternalItemsQuery.cs index 0e5a5047cd..eb2a3676ac 100644 --- a/MediaBrowser.Controller/Entities/InternalItemsQuery.cs +++ b/MediaBrowser.Controller/Entities/InternalItemsQuery.cs @@ -432,12 +432,18 @@ namespace MediaBrowser.Controller.Entities public string? HasNoSubtitleTrackWithLanguage { get; set; } + /// <summary> + /// Gets or sets a value indicating whether to return only items nothing names any more. + /// </summary> public bool? IsDeadArtist { get; set; } public bool? IsDeadStudio { get; set; } public bool? IsDeadGenre { get; set; } + /// <summary> + /// Gets or sets a value indicating whether to return only items nothing names any more. + /// </summary> public bool? IsDeadPerson { get; set; } /// <summary> diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index c8cca1fa93..9028b0d6b8 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -107,6 +107,13 @@ namespace MediaBrowser.Controller.Library Person? GetPerson(string name); /// <summary> + /// Gets a Person, creating and persisting it if no item exists for the name yet. + /// </summary> + /// <param name="name">The name of the person.</param> + /// <returns>The person.</returns> + Person GetOrCreatePerson(string name); + + /// <summary> /// Finds the by path. /// </summary> /// <param name="path">The path.</param> @@ -153,15 +160,6 @@ namespace MediaBrowser.Controller.Library Year GetYear(int value); /// <summary> - /// Validate and refresh the People sub-set of the IBN. - /// The items are stored in the db but not loaded into memory until actually requested by an operation. - /// </summary> - /// <param name="progress">The progress.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>Task.</returns> - Task ValidatePeopleAsync(IProgress<double> progress, CancellationToken cancellationToken); - - /// <summary> /// Reloads the root media folder. /// </summary> /// <param name="progress">The progress.</param> @@ -709,6 +707,14 @@ namespace MediaBrowser.Controller.Library /// <returns><c>true</c> if ignored, <c>false</c> otherwise.</returns> bool IgnoreFile(FileSystemMetadata file, BaseItem parent); + /// <summary> + /// Gets the id a <see cref="Person"/> item for the name would have, without looking it up + /// or creating it. + /// </summary> + /// <param name="name">The name of the person.</param> + /// <returns>The item id for the name.</returns> + Guid GetPersonId(string name); + Guid GetStudioId(string name); Guid GetGenreId(string name); diff --git a/MediaBrowser.Controller/LibraryTaskScheduler/LimitedConcurrencyLibraryScheduler.cs b/MediaBrowser.Controller/LibraryTaskScheduler/LimitedConcurrencyLibraryScheduler.cs index 6da398129a..be75117b6f 100644 --- a/MediaBrowser.Controller/LibraryTaskScheduler/LimitedConcurrencyLibraryScheduler.cs +++ b/MediaBrowser.Controller/LibraryTaskScheduler/LimitedConcurrencyLibraryScheduler.cs @@ -17,7 +17,8 @@ namespace MediaBrowser.Controller.LibraryTaskScheduler; /// </summary> public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibraryScheduler, IAsyncDisposable { - private const int CleanupGracePeriod = 60; + private static readonly TimeSpan _cleanupGracePeriod = TimeSpan.FromSeconds(60); + private readonly IHostApplicationLifetime _hostApplicationLifetime; private readonly ILogger<LimitedConcurrencyLibraryScheduler> _logger; private readonly IServerConfigurationManager _serverConfigurationManager; @@ -31,6 +32,8 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr private readonly Lock _taskLock = new(); private readonly Channel<TaskQueueItem> _tasks = Channel.CreateUnbounded<TaskQueueItem>(); + private readonly CancellationTokenSource _disposeTokenSource = new(); + private readonly TimeSpan _gracePeriod; private volatile int _workCounter; private Task? _cleanupTask; @@ -46,10 +49,34 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr IHostApplicationLifetime hostApplicationLifetime, ILogger<LimitedConcurrencyLibraryScheduler> logger, IServerConfigurationManager serverConfigurationManager) + : this(hostApplicationLifetime, logger, serverConfigurationManager, _cleanupGracePeriod) + { + } + + internal LimitedConcurrencyLibraryScheduler( + IHostApplicationLifetime hostApplicationLifetime, + ILogger<LimitedConcurrencyLibraryScheduler> logger, + IServerConfigurationManager serverConfigurationManager, + TimeSpan gracePeriod) { _hostApplicationLifetime = hostApplicationLifetime; _logger = logger; _serverConfigurationManager = serverConfigurationManager; + _gracePeriod = gracePeriod; + } + + /// <summary> + /// Gets the number of runners the scheduler currently keeps alive. + /// </summary> + internal int ActiveRunnerCount + { + get + { + lock (_taskLock) + { + return _taskRunners.Count; + } + } } private void ScheduleTaskCleanup() @@ -68,31 +95,65 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr async Task RunCleanupTask() { - _logger.LogDebug("Schedule cleanup task in {CleanupGracePerioid} sec.", CleanupGracePeriod); - await Task.Delay(TimeSpan.FromSeconds(CleanupGracePeriod)).ConfigureAwait(false); - if (_disposed) + while (true) { - _logger.LogDebug("Abort cleaning up, already disposed."); - return; - } + _logger.LogDebug("Schedule cleanup task in {CleanupGracePeriod}.", _gracePeriod); + try + { + await Task.Delay(_gracePeriod, _disposeTokenSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _logger.LogDebug("Abort cleaning up, already disposed."); + return; + } - lock (_taskLock) - { - if (_tasks.Reader.Count > 0 || _workCounter > 0) + if (_disposed) { - _logger.LogDebug("Delay cleanup task, operations still running."); - // tasks are still there so its still in use. Reschedule cleanup task. - // we cannot just exit here and rely on the other invoker because there is a considerable timeframe where it could have already ended. - _cleanupTask = RunCleanupTask(); + _logger.LogDebug("Abort cleaning up, already disposed."); return; } + + CancellationTokenSource[] runners; + lock (_taskLock) + { + if (_tasks.Reader.Count > 0 || _workCounter > 0) + { + _logger.LogDebug("Delay cleanup task, operations still running."); + // tasks are still there so its still in use. Wait another grace period. + // we cannot just exit here and rely on the other invoker because there is a considerable timeframe where it could have already ended. + continue; + } + + runners = [.. _taskRunners.Keys]; + + // Retire the runners before they are told to stop: an operation starting while + // they wind down must spawn its own instead of counting these towards the fanout. + _taskRunners.Clear(); + + // Hand the next operation the ability to schedule a cleanup again. Without this + // the very first cleanup would be the only one that ever runs. + _cleanupTask = null; + } + + _logger.LogDebug("Cleanup runners."); + await StopRunners(runners).ConfigureAwait(false); + return; } + } + } - _logger.LogDebug("Cleanup runners."); - foreach (var item in _taskRunners.ToArray()) + private static async Task StopRunners(CancellationTokenSource[] runners) + { + foreach (var runner in runners) + { + try { - await item.Key.CancelAsync().ConfigureAwait(false); - _taskRunners.Remove(item.Key); + await runner.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + // The runner already stopped on its own and disposed its stop source. } } } @@ -127,12 +188,17 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr { var stopToken = new CancellationTokenSource(); var combinedSource = CancellationTokenSource.CreateLinkedTokenSource(stopToken.Token, _hostApplicationLifetime.ApplicationStopping); + + // Keyed on its own stop source, because cancelling that is what reaches the linked + // source the runner waits on. Cancellation does not travel the other way. + // Started without the runner's own token: a task cancelled before it is scheduled + // never runs its body, so it would never take itself out of _taskRunners again. _taskRunners.Add( - combinedSource, + stopToken, Task.Factory.StartNew( ItemWorker, - (combinedSource, stopToken), - combinedSource.Token, + (stopToken, combinedSource), + CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default)); } @@ -145,7 +211,7 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr _deadlockDetector.Value = stopToken.TaskStop; try { - while (!stopToken.GlobalStop.Token.IsCancellationRequested) + while (!stopToken.GlobalStop.IsCancellationRequested) { var item = await _tasks.Reader.ReadAsync(stopToken.GlobalStop.Token).ConfigureAwait(false); try @@ -162,15 +228,24 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr } } } - catch (OperationCanceledException) when (stopToken.TaskStop.IsCancellationRequested) + catch (OperationCanceledException) when (stopToken.GlobalStop.IsCancellationRequested) { // thats how you do it, interupt the waiter thread. There is nothing to do here when it was on purpose. } + catch (ChannelClosedException) + { + // the scheduler was disposed and will not hand out any more work. + } finally { _logger.LogDebug("Cleanup Runner'."); _deadlockDetector.Value = default!; - _taskRunners.Remove(stopToken.TaskStop); + + lock (_taskLock) + { + _taskRunners.Remove(stopToken.TaskStop); + } + stopToken.GlobalStop.Dispose(); stopToken.TaskStop.Dispose(); } @@ -195,7 +270,7 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr finally { item.Progress.Report(100); - item.Done.SetResult(); + item.Done.TrySetResult(); } } @@ -285,16 +360,33 @@ public sealed class LimitedConcurrencyLibraryScheduler : ILimitedConcurrencyLibr _disposed = true; _tasks.Writer.Complete(); - foreach (var item in _taskRunners) + + // Nobody is left to run these, so release whoever is waiting on them. + while (_tasks.Reader.TryRead(out var item)) { - await item.Key.CancelAsync().ConfigureAwait(false); + item.Done.TrySetResult(); } - if (_cleanupTask is not null) + CancellationTokenSource[] runners; + Task? cleanupTask; + lock (_taskLock) { - await _cleanupTask.ConfigureAwait(false); - _cleanupTask?.Dispose(); + runners = [.. _taskRunners.Keys]; + _taskRunners.Clear(); + cleanupTask = _cleanupTask; } + + await StopRunners(runners).ConfigureAwait(false); + + // Cuts the grace period short instead of holding up shutdown for the rest of it. + await _disposeTokenSource.CancelAsync().ConfigureAwait(false); + + if (cleanupTask is not null) + { + await cleanupTask.ConfigureAwait(false); + } + + _disposeTokenSource.Dispose(); } private class TaskQueueItem diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 06188ad511..73cdf18e91 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -18,7 +18,6 @@ </PropertyGroup> <ItemGroup> - <PackageReference Include="BitFaster.Caching" /> <PackageReference Include="Microsoft.Extensions.Configuration.Binder" /> </ItemGroup> diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 10c21ee03c..a0f4087395 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -442,7 +442,8 @@ namespace MediaBrowser.Controller.MediaEncoding && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 || IsHdr10Plus(state.VideoStream) || IsDoviWithHdr10Bl(state.VideoStream) - || state.VideoStream.VideoRangeType == VideoRangeType.HLG); + || state.VideoStream.VideoRangeType == VideoRangeType.HLG + || state.VideoStream.VideoRangeType == VideoRangeType.DOVIInvalid); } private static bool IsDeinterlaceAvailable(EncodingJobInfo state) @@ -695,7 +696,11 @@ namespace MediaBrowser.Controller.MediaEncoding "ogg" or "oga" or "ogv" or "webm" or "webma" => "opus", "m4a" or "m4b" or "mp4" or "mov" or "mkv" or "mka" => "aac", "ts" or "avi" or "flv" or "f4v" or "swf" => "mp3", - _ => inferredCodec + // Containers that share their name with the codec they carry. + "aac" or "ac3" or "alac" or "dts" or "eac3" or "flac" or "mp2" or "mp3" or "opus" or "truehd" or "vorbis" => inferredCodec, + // Anything else - manifests such as m3u8/mpd in particular - names a container that + // is not an audio codec. Never hand that name to ffmpeg as an encoder. + _ => "aac" }; } @@ -1386,7 +1391,8 @@ namespace MediaBrowser.Controller.MediaEncoding or VideoRangeType.DOVIWithEL or VideoRangeType.DOVIWithHDR10Plus or VideoRangeType.DOVIWithELHDR10Plus - or VideoRangeType.DOVIInvalid; + || (rangeType == VideoRangeType.DOVIInvalid + && string.Equals(stream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase)); // invalid may be hlg now } public static bool IsDovi(MediaStream stream) @@ -1396,7 +1402,8 @@ namespace MediaBrowser.Controller.MediaEncoding return IsDoviWithHdr10Bl(stream) || (rangeType is VideoRangeType.DOVI or VideoRangeType.DOVIWithHLG - or VideoRangeType.DOVIWithSDR); + or VideoRangeType.DOVIWithSDR + or VideoRangeType.DOVIInvalid); } public static bool IsHdr10Plus(MediaStream stream) @@ -1416,7 +1423,8 @@ namespace MediaBrowser.Controller.MediaEncoding private static DynamicHdrMetadataRemovalPlan ShouldRemoveDynamicHdrMetadata(EncodingJobInfo state) { var videoStream = state.VideoStream; - if (videoStream.VideoRange is not VideoRange.HDR) + if (videoStream.VideoRange is not VideoRange.HDR + && videoStream.VideoRangeType != VideoRangeType.DOVIInvalid) { return DynamicHdrMetadataRemovalPlan.None; } @@ -6311,7 +6319,7 @@ namespace MediaBrowser.Controller.MediaEncoding string.Join(',', overlayFilters)); var mapPrefix = Convert.ToInt32(state.SubtitleStream.IsExternal); - var subtitleStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.SubtitleStream); + var subtitleStreamIndex = GetSubtitleStreamIndexForFfmpeg(state.MediaSource, state.SubtitleStream); var videoStreamIndex = FindIndex(state.MediaSource.MediaStreams, state.VideoStream); if (hasSubs) @@ -7943,6 +7951,24 @@ namespace MediaBrowser.Controller.MediaEncoding return -1; } + public static int GetSubtitleStreamIndexForFfmpeg(MediaSourceInfo mediaSource, MediaStream subtitleStream) + { + var index = FindIndex(mediaSource.MediaStreams, subtitleStream); + if (index == -1 || subtitleStream.IsExternal || mediaSource.VideoType != VideoType.BluRay) + { + return index; + } + + var hiddenStreamsBefore = mediaSource.MediaStreams.Count(s => + s.Type == MediaStreamType.Audio + && !s.IsExternal + && (string.Equals(s.Codec, "truehd", StringComparison.OrdinalIgnoreCase) + || string.Equals(s.Codec, "atmos", StringComparison.OrdinalIgnoreCase)) + && s.Index < subtitleStream.Index); + + return index + hiddenStreamsBefore; + } + public static bool IsCopyCodec(string codec) { return string.Equals(codec, "copy", StringComparison.OrdinalIgnoreCase); diff --git a/MediaBrowser.Controller/Providers/DirectoryService.cs b/MediaBrowser.Controller/Providers/DirectoryService.cs index 6060d051a5..f8e0bf4ed9 100644 --- a/MediaBrowser.Controller/Providers/DirectoryService.cs +++ b/MediaBrowser.Controller/Providers/DirectoryService.cs @@ -5,13 +5,19 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using MediaBrowser.Model.IO; namespace MediaBrowser.Controller.Providers { public class DirectoryService : IDirectoryService { - // TODO make static and switch to FastConcurrentLru. + // TODO replace with one shared bounded cache. + private const int MaxCachedRecords = 100_000; + private const int AccessIntervalMs = 1_000; + // Timeout cache if no access for 5 minutes. + private const int IdleTimeoutMs = 5 * 60 * 1_000; + private readonly ConcurrentDictionary<string, FileSystemMetadata[]> _cache = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<string, FileSystemMetadata> _fileCache = new(StringComparer.Ordinal); @@ -20,6 +26,12 @@ namespace MediaBrowser.Controller.Providers private readonly IFileSystem _fileSystem; + // ConcurrentDictionary.Count locks the dictionary, so keep an estimated counter. + // Concurrent factory runs can overcount and a clear racing an add can undercount, + // it only has to be roughly right. + private int _recordCount; + private long _lastAccess = Environment.TickCount64; + public DirectoryService(IFileSystem fileSystem) { _fileSystem = fileSystem; @@ -27,20 +39,26 @@ namespace MediaBrowser.Controller.Providers public FileSystemMetadata[] GetFileSystemEntries(string path) { + DropCacheIfIdleOrFull(); + return _cache.GetOrAdd( path, - static (p, fileSystem) => + static (p, state) => { + FileSystemMetadata[] entries; try { - return fileSystem.GetFileSystemEntries(p).ToArray(); + entries = state.FileSystem.GetFileSystemEntries(p).ToArray(); } catch (DirectoryNotFoundException) { - return []; + entries = []; } + + Interlocked.Add(ref state.Service._recordCount, entries.Length + 1); + return entries; }, - _fileSystem); + (FileSystem: _fileSystem, Service: this)); } public List<FileSystemMetadata> GetDirectories(string path) @@ -89,13 +107,18 @@ namespace MediaBrowser.Controller.Providers public FileSystemMetadata? GetFileSystemEntry(string path) { + DropCacheIfIdleOrFull(); + if (!_fileCache.TryGetValue(path, out var result)) { var file = _fileSystem.GetFileSystemInfo(path); if (file?.Exists ?? false) { result = file; - _fileCache.TryAdd(path, result); + if (_fileCache.TryAdd(path, result)) + { + Interlocked.Increment(ref _recordCount); + } } } @@ -107,32 +130,96 @@ namespace MediaBrowser.Controller.Providers public IReadOnlyList<string> GetFilePaths(string path, bool clearCache) { - if (clearCache) + if (clearCache && _filePathCache.TryRemove(path, out var cached)) { - _filePathCache.TryRemove(path, out _); + Interlocked.Add(ref _recordCount, -(cached.Count + 1)); } + DropCacheIfIdleOrFull(); + var filePaths = _filePathCache.GetOrAdd( path, - static (p, fileSystem) => + static (p, state) => { + List<string> filePaths; try { - return fileSystem.GetFilePaths(p).OrderBy(x => x).ToList(); + filePaths = state.FileSystem.GetFilePaths(p).OrderBy(x => x).ToList(); } catch (DirectoryNotFoundException) { - return []; + filePaths = []; } + + Interlocked.Add(ref state.Service._recordCount, filePaths.Count + 1); + return filePaths; }, - _fileSystem); + (FileSystem: _fileSystem, Service: this)); return filePaths; } + public void Invalidate(string path) + { + Forget(path); + + var parent = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(parent)) + { + Forget(parent); + } + } + + public void Move(string source, string destination) + { + Directory.Move(source, destination); + + Invalidate(source); + Invalidate(destination); + } + public bool IsAccessible(string path) { return _fileSystem.GetFileSystemEntryPaths(path).Any(); } + + private void DropCacheIfIdleOrFull() + { + var nowMs = Environment.TickCount64; + var idleMs = nowMs - _lastAccess; + + if (idleMs >= IdleTimeoutMs || _recordCount >= MaxCachedRecords) + { + _cache.Clear(); + _fileCache.Clear(); + _filePathCache.Clear(); + _recordCount = 0; + _lastAccess = nowMs; + return; + } + + if (idleMs >= AccessIntervalMs) + { + _lastAccess = nowMs; + } + } + + private void Forget(string path) + { + if (_cache.TryRemove(path, out var entries)) + { + Interlocked.Add(ref _recordCount, -(entries.Length + 1)); + } + + if (_fileCache.TryRemove(path, out _)) + { + Interlocked.Decrement(ref _recordCount); + } + + if (_filePathCache.TryRemove(path, out var filePaths)) + { + Interlocked.Add(ref _recordCount, -(filePaths.Count + 1)); + } + } } } diff --git a/MediaBrowser.Controller/Providers/IDirectoryService.cs b/MediaBrowser.Controller/Providers/IDirectoryService.cs index 8a3fa33da3..3a943d5f0c 100644 --- a/MediaBrowser.Controller/Providers/IDirectoryService.cs +++ b/MediaBrowser.Controller/Providers/IDirectoryService.cs @@ -23,6 +23,19 @@ namespace MediaBrowser.Controller.Providers IReadOnlyList<string> GetFilePaths(string path, bool clearCache); + /// <summary> + /// Forgets what is cached about a path and about the directory containing it. + /// </summary> + /// <param name="path">The file or directory path that changed.</param> + void Invalidate(string path); + + /// <summary> + /// Moves a directory and forgets what is cached about both paths. + /// </summary> + /// <param name="source">The directory to move.</param> + /// <param name="destination">The path to move the directory to.</param> + void Move(string source, string destination); + bool IsAccessible(string path); } } diff --git a/MediaBrowser.Controller/SyncPlay/GroupStates/WaitingGroupState.cs b/MediaBrowser.Controller/SyncPlay/GroupStates/WaitingGroupState.cs index eb38eeb503..f4fab29800 100644 --- a/MediaBrowser.Controller/SyncPlay/GroupStates/WaitingGroupState.cs +++ b/MediaBrowser.Controller/SyncPlay/GroupStates/WaitingGroupState.cs @@ -501,7 +501,7 @@ namespace MediaBrowser.Controller.SyncPlay.GroupStates { // Client, that was buffering, resumed playback but did not update others in time. delayTicks = context.GetHighestPing() * 2 * TimeSpan.TicksPerMillisecond; - delayTicks = Math.Max(delayTicks, context.DefaultPing); + delayTicks = Math.Max(delayTicks, TimeSpan.FromMilliseconds(context.DefaultPing).Ticks); context.LastActivity = currentTime.AddTicks(delayTicks); diff --git a/MediaBrowser.Controller/SyncPlay/Queue/PlayQueueManager.cs b/MediaBrowser.Controller/SyncPlay/Queue/PlayQueueManager.cs index 9326864d78..258b92e4d9 100644 --- a/MediaBrowser.Controller/SyncPlay/Queue/PlayQueueManager.cs +++ b/MediaBrowser.Controller/SyncPlay/Queue/PlayQueueManager.cs @@ -157,7 +157,10 @@ namespace MediaBrowser.Controller.SyncPlay.Queue /// </summary> public void RestoreSortedPlaylist() { - if (PlayingItemIndex != NoPlayingItemIndex) + // The shuffled playlist is only populated while the shuffle mode is active, so there is + // nothing to map back when the playlist is already sorted. Guarding on its contents keeps + // a redundant request for the sorted mode from indexing an empty list. + if (PlayingItemIndex != NoPlayingItemIndex && _shuffledPlaylist.Count > 0) { var playingItem = _shuffledPlaylist[PlayingItemIndex]; PlayingItemIndex = _sortedPlaylist.IndexOf(playingItem); diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index 0ddd378352..f64fd73763 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -1152,6 +1152,11 @@ namespace MediaBrowser.MediaEncoding.Encoder { process.Process.PriorityClass = ProcessPriorityClass.BelowNormal; } + catch (InvalidOperationException) + { + // The process finished before its priority could be lowered. That says nothing + // about whether the platform allows it, so keep the capability for the next one. + } catch (Exception ex) { _canSetProcessPriority = false; @@ -1361,12 +1366,20 @@ namespace MediaBrowser.MediaEncoding.Encoder return _configurationManager.GetEncodingOptions().EnableSubtitleExtraction; } - private sealed class ProcessWrapper : IDisposable + internal sealed class ProcessWrapper : IDisposable { private readonly MediaEncoder _mediaEncoder; + // The exit event is raised on the thread pool, so it writes the state below while the + // caller that started the process is reading it. + private readonly Lock _exitLock = new(); + private bool _disposed = false; + private bool _hasExited; + + private int? _exitCode; + public ProcessWrapper(Process process, MediaEncoder mediaEncoder) { Process = process; @@ -1376,49 +1389,84 @@ namespace MediaBrowser.MediaEncoding.Encoder public Process Process { get; } - public bool HasExited { get; private set; } + // The exit event can lag behind the wait that returned, so ask the process rather than + // report one that has exited as still running. + public bool HasExited => ReadExitState().HasExited; + + // As above: rather than report no exit code for a process that has one. + public int? ExitCode => ReadExitState().ExitCode; - public int? ExitCode { get; private set; } + private (bool HasExited, int? ExitCode) ReadExitState() + { + lock (_exitLock) + { + if (!_hasExited && !_disposed) + { + try + { + if (Process.HasExited) + { + _hasExited = true; + _exitCode = Process.ExitCode; + } + } + catch (InvalidOperationException) + { + // No process is associated with this object, or it was disposed from + // under us - ObjectDisposedException derives from this one. + } + } + + return (_hasExited, _exitCode); + } + } private void OnProcessExited(object sender, EventArgs e) { var process = (Process)sender; - HasExited = true; - - try - { - ExitCode = process.ExitCode; - } - catch + lock (_exitLock) { + _hasExited = true; + + try + { + _exitCode = process.ExitCode; + } + catch + { + } } - DisposeProcess(process); + // Only stop tracking it. The caller that started the process still holds it to read + // its output and its exit code, so disposing it here handed whoever was quickest to + // exit - an ffprobe on a file it rejects outright - an ObjectDisposedException. + Untrack(); } - private void DisposeProcess(Process process) + private void Untrack() { lock (_mediaEncoder._runningProcessesLock) { _mediaEncoder._runningProcesses.Remove(this); } - - process.Dispose(); } public void Dispose() { - if (!_disposed) + lock (_exitLock) { - if (Process is not null) + if (_disposed) { - Process.Exited -= OnProcessExited; - DisposeProcess(Process); + return; } + + _disposed = true; } - _disposed = true; + Process.Exited -= OnProcessExited; + Untrack(); + Process.Dispose(); } } } diff --git a/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs b/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs index e8c636e7fb..fba644b74a 100644 --- a/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs +++ b/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs @@ -649,7 +649,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles List<MediaStream> subtitleStreams, CancellationToken cancellationToken) { - var inputPath = _mediaEncoder.GetInputArgument(mediaSource.Path, mediaSource); + var inputPath = _mediaEncoder.GetInputPathArgument(mediaSource.Path, mediaSource); var outputPaths = new List<string>(); var args = string.Format( CultureInfo.InvariantCulture, @@ -673,7 +673,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles var outputCodec = IsCodecCopyable(subtitleStream.Codec) ? "copy" : "srt"; // FFmpeg does not provide an .idx/.sub muxer, so VobSub streams must be written as MKS files. var outputFormatOption = MediaStream.IsVobSubFormat(subtitleStream.Codec) ? " -f matroska" : string.Empty; - var streamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream); + var streamIndex = EncodingHelper.GetSubtitleStreamIndexForFfmpeg(mediaSource, subtitleStream); if (streamIndex == -1) { diff --git a/MediaBrowser.Model/Dlna/StreamBuilder.cs b/MediaBrowser.Model/Dlna/StreamBuilder.cs index ab8d5dd5b2..4799ed5410 100644 --- a/MediaBrowser.Model/Dlna/StreamBuilder.cs +++ b/MediaBrowser.Model/Dlna/StreamBuilder.cs @@ -26,6 +26,8 @@ namespace MediaBrowser.Model.Dlna internal const TranscodeReason VideoReasons = TranscodeReason.VideoCodecNotSupported | VideoCodecReasons; internal const TranscodeReason DirectStreamReasons = AudioReasons | TranscodeReason.ContainerNotSupported | TranscodeReason.VideoCodecTagNotSupported; + private const string ManifestContainers = "hls,applehttp,dash"; + private readonly ILogger _logger; private readonly ITranscoderSupport _transcoderSupport; private static readonly string[] _supportedHlsVideoCodecs = ["h264", "hevc", "vp9", "av1"]; @@ -718,6 +720,14 @@ namespace MediaBrowser.Model.Dlna isEligibleForDirectPlay = false; } + // A manifest is not a byte stream, so it cannot be handed to the client as one. The variant + // and segment URIs inside it are relative to the origin and do not resolve against the + // Jellyfin url the client would fetch it from. + if (ContainerHelper.ContainsContainer(ManifestContainers, item.Container)) + { + isEligibleForDirectPlay = false; + } + if (bitrateLimitExceeded) { transcodeReasons = TranscodeReason.ContainerBitrateExceedsLimit; diff --git a/MediaBrowser.Model/Entities/MediaStream.cs b/MediaBrowser.Model/Entities/MediaStream.cs index f057714bea..67af843626 100644 --- a/MediaBrowser.Model/Entities/MediaStream.cs +++ b/MediaBrowser.Model/Entities/MediaStream.cs @@ -810,6 +810,11 @@ namespace MediaBrowser.Model.Entities return (VideoRange.Unknown, VideoRangeType.Unknown); } + var isPq = string.Equals(ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase); + var isHlg = string.Equals(ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase); + // Invalid DV only retains HDR when the base layer explicitly signals PQ or HLG. + var baseVideoRange = isPq || isHlg ? VideoRange.HDR : VideoRange.SDR; + var codecTag = CodecTag; var dvProfile = DvProfile; var rpuPresentFlag = RpuPresentFlag == 1; @@ -834,7 +839,7 @@ namespace MediaBrowser.Model.Entities 4 => (VideoRange.HDR, VideoRangeType.DOVIWithHLG), 2 => (VideoRange.SDR, VideoRangeType.DOVIWithSDR), // Out of Dolby Spec files should be marked as invalid - _ => (VideoRange.HDR, VideoRangeType.DOVIInvalid) + _ => (baseVideoRange, VideoRangeType.DOVIInvalid) }, 7 => (VideoRange.HDR, VideoRangeType.DOVIWithEL), 10 => dvBlCompatId switch @@ -844,11 +849,26 @@ namespace MediaBrowser.Model.Entities 2 => (VideoRange.SDR, VideoRangeType.DOVIWithSDR), 4 => (VideoRange.HDR, VideoRangeType.DOVIWithHLG), // Out of Dolby Spec files should be marked as invalid - _ => (VideoRange.HDR, VideoRangeType.DOVIInvalid) + _ => (baseVideoRange, VideoRangeType.DOVIInvalid) }, _ => (VideoRange.SDR, VideoRangeType.SDR) }; + var expectedTransfer = dvRangeSet.Item2 switch + { + VideoRangeType.DOVIWithHDR10 or VideoRangeType.DOVIWithEL => "smpte2084", + VideoRangeType.DOVIWithHLG => "arib-std-b67", + _ => null + }; + + if (expectedTransfer is not null + && (!string.Equals(ColorSpace, "bt2020nc", StringComparison.OrdinalIgnoreCase) + || !string.Equals(ColorTransfer, expectedTransfer, StringComparison.OrdinalIgnoreCase) + || !string.Equals(ColorPrimaries, "bt2020", StringComparison.OrdinalIgnoreCase))) + { + return (baseVideoRange, VideoRangeType.DOVIInvalid); + } + if (Hdr10PlusPresentFlag == true) { return dvRangeSet.Item2 switch @@ -862,13 +882,11 @@ namespace MediaBrowser.Model.Entities return dvRangeSet; } - var colorTransfer = ColorTransfer; - - if (string.Equals(colorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase)) + if (isPq) { return Hdr10PlusPresentFlag == true ? (VideoRange.HDR, VideoRangeType.HDR10Plus) : (VideoRange.HDR, VideoRangeType.HDR10); } - else if (string.Equals(colorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)) + else if (isHlg) { return (VideoRange.HDR, VideoRangeType.HLG); } diff --git a/MediaBrowser.Providers/Lyric/LyricManager.cs b/MediaBrowser.Providers/Lyric/LyricManager.cs index af31e373ef..a19262c3a7 100644 --- a/MediaBrowser.Providers/Lyric/LyricManager.cs +++ b/MediaBrowser.Providers/Lyric/LyricManager.cs @@ -32,6 +32,7 @@ public class LyricManager : ILyricManager private readonly IFileSystem _fileSystem; private readonly ILibraryMonitor _libraryMonitor; private readonly IMediaSourceManager _mediaSourceManager; + private readonly IDirectoryService _directoryService; private readonly ILyricProvider[] _lyricProviders; private readonly ILyricParser[] _lyricParsers; @@ -43,6 +44,7 @@ public class LyricManager : ILyricManager /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> /// <param name="libraryMonitor">Instance of the <see cref="ILibraryMonitor"/> interface.</param> /// <param name="mediaSourceManager">Instance of the <see cref="IMediaSourceManager"/> interface.</param> + /// <param name="directoryService">Instance of the <see cref="IDirectoryService"/> interface.</param> /// <param name="lyricProviders">The list of <see cref="ILyricProvider"/>.</param> /// <param name="lyricParsers">The list of <see cref="ILyricParser"/>.</param> public LyricManager( @@ -50,6 +52,7 @@ public class LyricManager : ILyricManager IFileSystem fileSystem, ILibraryMonitor libraryMonitor, IMediaSourceManager mediaSourceManager, + IDirectoryService directoryService, IEnumerable<ILyricProvider> lyricProviders, IEnumerable<ILyricParser> lyricParsers) { @@ -57,6 +60,7 @@ public class LyricManager : ILyricManager _fileSystem = fileSystem; _libraryMonitor = libraryMonitor; _mediaSourceManager = mediaSourceManager; + _directoryService = directoryService; _lyricProviders = lyricProviders .OrderBy(i => i is IHasOrder hasOrder ? hasOrder.Order : 0) .ToArray(); @@ -250,6 +254,8 @@ public class LyricManager : ILyricManager { _libraryMonitor.ReportFileSystemChangeComplete(path, false); } + + _directoryService.Invalidate(path); } return audio.RefreshMetadata(CancellationToken.None); @@ -446,6 +452,8 @@ public class LyricManager : ILyricManager await stream.CopyToAsync(fs).ConfigureAwait(false); } + _directoryService.Invalidate(savePath); + return; } catch (Exception ex) diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index fbd9e5435e..e7b15305b3 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -1143,16 +1143,21 @@ namespace MediaBrowser.Providers.Manager return; } - _refreshQueue.Enqueue((itemId, options), priority); - + // PriorityQueue is not thread safe and the processor dequeues concurrently, so every + // touch of the queue takes the lock. lock (_refreshQueueLock) { - if (!_isProcessingRefreshQueue) + _refreshQueue.Enqueue((itemId, options), priority); + + if (_isProcessingRefreshQueue) { - _isProcessingRefreshQueue = true; - Task.Run(StartProcessingRefreshQueue); + return; } + + _isProcessingRefreshQueue = true; } + + Task.Run(StartProcessingRefreshQueue); } private async Task StartProcessingRefreshQueue() @@ -1161,17 +1166,33 @@ namespace MediaBrowser.Providers.Manager if (_disposed) { + lock (_refreshQueueLock) + { + _isProcessingRefreshQueue = false; + } + return; } var cancellationToken = _disposeCancellationTokenSource.Token; libraryManager.ClearIgnoreRuleCache(); - while (_refreshQueue.TryDequeue(out var refreshItem, out _)) + + while (true) { - if (_disposed) + (Guid ItemId, MetadataRefreshOptions RefreshOptions) refreshItem; + + // Dequeueing and standing down happen under one lock, otherwise a refresh queued + // just after the queue ran dry would see a processor that has already stopped. + lock (_refreshQueueLock) { - return; + if (_disposed + || cancellationToken.IsCancellationRequested + || !_refreshQueue.TryDequeue(out refreshItem, out _)) + { + _isProcessingRefreshQueue = false; + break; + } } try @@ -1188,19 +1209,21 @@ namespace MediaBrowser.Providers.Manager await task.ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - break; + // Shutting down: the next pass sees the token and stands the processor down. + continue; } catch (Exception ex) { + // Includes a provider that cancelled for its own reasons, such as an HTTP + // timeout, which must not stop the queue draining. _logger.LogError(ex, "Error refreshing item"); } } - lock (_refreshQueueLock) + if (!_disposed) { - _isProcessingRefreshQueue = false; libraryManager.ClearIgnoreRuleCache(); } } diff --git a/MediaBrowser.Providers/MediaInfo/MediaInfoResolver.cs b/MediaBrowser.Providers/MediaInfo/MediaInfoResolver.cs index 6f9d5f19da..ecb6e5d990 100644 --- a/MediaBrowser.Providers/MediaInfo/MediaInfoResolver.cs +++ b/MediaBrowser.Providers/MediaInfo/MediaInfoResolver.cs @@ -231,10 +231,28 @@ namespace MediaBrowser.Providers.MediaInfo return Array.Empty<ExternalPathParserResult>(); } + // VobSub .sub payloads only carry per-track language metadata when read via + // their paired .idx file, so probe the .idx instead and skip the .sub. Pairing + // requires the same directory (ffprobe can't resolve a split pair) and an + // ordinal comparison (ffprobe matches the .sub by exact case on case-sensitive + // filesystems, so a looser match could suppress a .sub with no working .idx). + // An .idx file with no paired .sub cannot be probed at all, so it is left out + // entirely rather than surfaced (which would otherwise fail every probe and, + // since the .idx would keep "existing" from Jellyfin's point of view, prevent + // stale subtitle stream metadata from ever being cleared once the .sub is gone). + HashSet<string>? pairedVobSubKeys = _type == DlnaProfileType.Subtitle + ? GetPairedVobSubKeys(files) + : null; + var externalPathInfos = new List<ExternalPathParserResult>(); ReadOnlySpan<char> prefix = video.FileNameWithoutExtension; foreach (var file in files) { + if (IsSuppressedVobSubFile(file, pairedVobSubKeys)) + { + continue; + } + var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(file.AsSpan()); if (fileNameWithoutExtension.Length >= prefix.Length && prefix.Equals(fileNameWithoutExtension[..prefix.Length], StringComparison.OrdinalIgnoreCase) @@ -305,6 +323,77 @@ namespace MediaBrowser.Providers.MediaInfo } /// <summary> + /// Determines whether a candidate file is part of a VobSub .idx/.sub pair that + /// should be resolved to only its .idx file, or an .idx file with no paired .sub + /// that cannot be probed at all. + /// </summary> + /// <param name="file">The full path to the candidate file.</param> + /// <param name="pairedVobSubKeys">The set of pairing keys with both an .idx and .sub present, or null if not applicable.</param> + /// <returns><c>true</c> if the file should be suppressed; otherwise, <c>false</c>.</returns> + private static bool IsSuppressedVobSubFile(string file, HashSet<string>? pairedVobSubKeys) + { + if (pairedVobSubKeys is null) + { + return false; + } + + var extension = Path.GetExtension(file.AsSpan()); + if (extension.Equals(".sub", StringComparison.OrdinalIgnoreCase)) + { + // A paired .idx exists; probe it instead of the .sub payload. + return pairedVobSubKeys.Contains(GetVobSubPairingKey(file)); + } + + if (extension.Equals(".idx", StringComparison.OrdinalIgnoreCase)) + { + // Without its .sub payload, the .idx cannot be probed for any data. + return !pairedVobSubKeys.Contains(GetVobSubPairingKey(file)); + } + + return false; + } + + /// <summary> + /// Builds the set of directory+basename keys that have both an .idx and a .sub + /// file present, in a single pass over the candidate files. + /// </summary> + /// <param name="files">The candidate files to search.</param> + /// <returns>The set of pairing keys with both an .idx and .sub present.</returns> + private static HashSet<string> GetPairedVobSubKeys(IEnumerable<string> files) + { + var idxKeys = new HashSet<string>(StringComparer.Ordinal); + var subKeys = new HashSet<string>(StringComparer.Ordinal); + foreach (var file in files) + { + var extension = Path.GetExtension(file.AsSpan()); + if (extension.Equals(".idx", StringComparison.OrdinalIgnoreCase)) + { + idxKeys.Add(GetVobSubPairingKey(file)); + } + else if (extension.Equals(".sub", StringComparison.OrdinalIgnoreCase)) + { + subKeys.Add(GetVobSubPairingKey(file)); + } + } + + idxKeys.IntersectWith(subKeys); + return idxKeys; + } + + /// <summary> + /// Builds a directory+basename key used to pair a VobSub .idx file with its .sub + /// payload only when both live in the same directory. + /// </summary> + /// <param name="file">The full path to the file.</param> + /// <returns>A key combining the containing directory and file name without extension.</returns> + private static string GetVobSubPairingKey(string file) + { + var directory = Path.GetDirectoryName(file) ?? string.Empty; + var baseName = Path.GetFileNameWithoutExtension(file); + return Path.Combine(directory, baseName); + } + + /// <summary> /// Returns the media info of the given file. /// </summary> /// <param name="path">The path to the file.</param> diff --git a/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieSimilarProvider.cs b/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieSimilarProvider.cs index 5206de78ce..6a3c72d2fa 100644 --- a/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieSimilarProvider.cs +++ b/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieSimilarProvider.cs @@ -58,7 +58,7 @@ public class TmdbMovieSimilarProvider : IRemoteSimilarItemsProvider<Movie> } var providerName = MetadataProvider.Tmdb.ToString(); - var page = 0; + var page = 1; var totalPages = 1; while (page <= totalPages && !cancellationToken.IsCancellationRequested) @@ -67,12 +67,12 @@ public class TmdbMovieSimilarProvider : IRemoteSimilarItemsProvider<Movie> try { (pageResults, totalPages) = await _tmdbClientManager - .GetMovieSimilarPageAsync(tmdbId, page, TmdbUtils.GetImageLanguagesParam(string.Empty), cancellationToken) + .GetMovieRecommendationsPageAsync(tmdbId, page, TmdbUtils.GetImageLanguagesParam(string.Empty), cancellationToken) .ConfigureAwait(false); } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to get similar movies from TMDb for {TmdbId} page {Page}", tmdbId, page); + _logger.LogWarning(ex, "Failed to get recommended movies from TMDb for {TmdbId} page {Page}", tmdbId, page); yield break; } @@ -81,12 +81,12 @@ public class TmdbMovieSimilarProvider : IRemoteSimilarItemsProvider<Movie> yield break; } - foreach (var similar in pageResults) + foreach (var recommendation in pageResults) { yield return new SimilarItemReference { ProviderName = providerName, - ProviderId = similar.Id.ToString(CultureInfo.InvariantCulture) + ProviderId = recommendation.Id.ToString(CultureInfo.InvariantCulture) }; } diff --git a/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesSimilarProvider.cs b/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesSimilarProvider.cs index c85718b993..40c7de05ad 100644 --- a/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesSimilarProvider.cs +++ b/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesSimilarProvider.cs @@ -67,12 +67,12 @@ public class TmdbSeriesSimilarProvider : IRemoteSimilarItemsProvider<Series> try { (pageResults, totalPages) = await _tmdbClientManager - .GetSeriesSimilarPageAsync(tmdbId, page, TmdbUtils.GetImageLanguagesParam(string.Empty), cancellationToken) + .GetSeriesRecommendationsPageAsync(tmdbId, page, TmdbUtils.GetImageLanguagesParam(string.Empty), cancellationToken) .ConfigureAwait(false); } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to get similar TV shows from TMDb for {TmdbId} page {Page}", tmdbId, page); + _logger.LogWarning(ex, "Failed to get recommended TV shows from TMDb for {TmdbId} page {Page}", tmdbId, page); yield break; } @@ -81,12 +81,12 @@ public class TmdbSeriesSimilarProvider : IRemoteSimilarItemsProvider<Series> yield break; } - foreach (var similar in pageResults) + foreach (var recommendation in pageResults) { yield return new SimilarItemReference { ProviderName = providerName, - ProviderId = similar.Id.ToString(CultureInfo.InvariantCulture) + ProviderId = recommendation.Id.ToString(CultureInfo.InvariantCulture) }; } diff --git a/MediaBrowser.Providers/Plugins/Tmdb/TmdbClientManager.cs b/MediaBrowser.Providers/Plugins/Tmdb/TmdbClientManager.cs index 5379796465..fb3e5ee92b 100644 --- a/MediaBrowser.Providers/Plugins/Tmdb/TmdbClientManager.cs +++ b/MediaBrowser.Providers/Plugins/Tmdb/TmdbClientManager.cs @@ -25,16 +25,19 @@ namespace MediaBrowser.Providers.Plugins.Tmdb { private const int CacheDurationInHours = 1; - private readonly IMemoryCache _memoryCache; + // Sized in TMDb records - see EstimateSize - rather than in responses, because the responses + // differ in weight by orders of magnitude. + private const int CacheSizeLimit = 100_000; + + private readonly MemoryCache _memoryCache; private readonly TMDbClient _tmDbClient; /// <summary> /// Initializes a new instance of the <see cref="TmdbClientManager"/> class. /// </summary> - /// <param name="memoryCache">An instance of <see cref="IMemoryCache"/>.</param> - public TmdbClientManager(IMemoryCache memoryCache) + public TmdbClientManager() { - _memoryCache = memoryCache; + _memoryCache = new MemoryCache(new MemoryCacheOptions { SizeLimit = CacheSizeLimit }); var apiKey = Plugin.Instance.Configuration.TmdbApiKey; apiKey = string.IsNullOrEmpty(apiKey) ? TmdbUtils.ApiKey : apiKey; @@ -78,7 +81,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (movie is not null) { - _memoryCache.Set(key, movie, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, movie); } return movie; @@ -112,7 +115,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (collection is not null) { - _memoryCache.Set(key, collection, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, collection); } return collection; @@ -152,7 +155,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (series is not null) { - _memoryCache.Set(key, series, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, series); } return series; @@ -208,7 +211,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (group is not null) { - _memoryCache.Set(key, group, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, group); } return group; @@ -244,7 +247,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (season is not null) { - _memoryCache.Set(key, season, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, season); } return season; @@ -296,7 +299,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (episode is not null) { - _memoryCache.Set(key, episode, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, episode); } return episode; @@ -328,7 +331,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (person is not null) { - _memoryCache.Set(key, person, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, person); } return person; @@ -366,7 +369,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (result is not null) { - _memoryCache.Set(key, result, TimeSpan.FromHours(CacheDurationInHours)); + Cache(key, result); } return result; @@ -397,7 +400,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (searchResults?.Results?.Count > 0) { - _memoryCache.Set(key, searchResults, TimeSpan.FromHours(CacheDurationInHours)); + CacheSearch(key, searchResults); } return searchResults?.Results; @@ -425,7 +428,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (searchResults?.Results?.Count > 0) { - _memoryCache.Set(key, searchResults, TimeSpan.FromHours(CacheDurationInHours)); + CacheSearch(key, searchResults); } return searchResults?.Results; @@ -468,7 +471,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (searchResults?.Results?.Count > 0) { - _memoryCache.Set(key, searchResults, TimeSpan.FromHours(CacheDurationInHours)); + CacheSearch(key, searchResults); } return searchResults?.Results; @@ -498,26 +501,26 @@ namespace MediaBrowser.Providers.Plugins.Tmdb if (searchResults?.Results?.Count > 0) { - _memoryCache.Set(key, searchResults, TimeSpan.FromHours(CacheDurationInHours)); + CacheSearch(key, searchResults); } return searchResults?.Results; } /// <summary> - /// Gets a single page of similar movies for a movie from the TMDb API. + /// Gets a single page of recommended movies for a movie from the TMDb API. /// </summary> /// <param name="tmdbId">The TMDb id of the movie.</param> /// <param name="page">The page number to fetch (1-based).</param> /// <param name="language">The language for results.</param> /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>A tuple containing the list of similar movies and the total number of pages available.</returns> - public async Task<(IReadOnlyList<SearchMovie> Results, int TotalPages)> GetMovieSimilarPageAsync(int tmdbId, int page, string? language, CancellationToken cancellationToken) + /// <returns>A tuple containing the list of recommended movies and the total number of pages available.</returns> + public async Task<(IReadOnlyList<SearchMovie> Results, int TotalPages)> GetMovieRecommendationsPageAsync(int tmdbId, int page, string? language, CancellationToken cancellationToken) { await EnsureClientConfigAsync().ConfigureAwait(false); var searchResults = await _tmDbClient - .GetMovieSimilarAsync(tmdbId, language, page, cancellationToken) + .GetMovieRecommendationsAsync(tmdbId, language, page, cancellationToken) .ConfigureAwait(false); if (searchResults?.Results is null || searchResults.Results.Count == 0) @@ -529,19 +532,19 @@ namespace MediaBrowser.Providers.Plugins.Tmdb } /// <summary> - /// Gets a single page of similar TV shows for a series from the TMDb API. + /// Gets a single page of recommended TV shows for a series from the TMDb API. /// </summary> /// <param name="tmdbId">The TMDb id of the TV show.</param> /// <param name="page">The page number to fetch (1-based).</param> /// <param name="language">The language for results.</param> /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>A tuple containing the list of similar TV shows and the total number of pages available.</returns> - public async Task<(IReadOnlyList<SearchTv> Results, int TotalPages)> GetSeriesSimilarPageAsync(int tmdbId, int page, string? language, CancellationToken cancellationToken) + /// <returns>A tuple containing the list of recommended TV shows and the total number of pages available.</returns> + public async Task<(IReadOnlyList<SearchTv> Results, int TotalPages)> GetSeriesRecommendationsPageAsync(int tmdbId, int page, string? language, CancellationToken cancellationToken) { await EnsureClientConfigAsync().ConfigureAwait(false); var searchResults = await _tmDbClient - .GetTvShowSimilarAsync(tmdbId, language, page, cancellationToken) + .GetTvShowRecommendationsAsync(tmdbId, language, page, cancellationToken) .ConfigureAwait(false); if (searchResults?.Results is null || searchResults.Results.Count == 0) @@ -753,6 +756,84 @@ namespace MediaBrowser.Providers.Plugins.Tmdb return _tmDbClient.Config; } + /// <summary> + /// Stores a response under the shared expiry, weighed by what it costs to keep. + /// </summary> + private void Cache<T>(string key, T value) + where T : class + => _memoryCache.Set( + key, + value, + new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(CacheDurationInHours), + Size = EstimateSize(value) + }); + + /// <summary> + /// Stores a page of search results, whose weight is simply how many there are. + /// </summary> + private void CacheSearch<T>(string key, SearchContainer<T> results) + => _memoryCache.Set( + key, + results, + new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(CacheDurationInHours), + Size = 1 + Count(results.Results) + }); + + private static long Count<T>(IReadOnlyCollection<T>? items) => items?.Count ?? 0; + + /// <summary> + /// Estimates what keeping a response costs, counting the sub-records that dominate it. + /// </summary> + private static long EstimateSize(object? value) => value switch + { + TvShow series => 1 + + Count(series.Credits?.Cast) + Count(series.Credits?.Crew) + + EstimateAggregateSize(series.AggregateCredits) + + Count(series.Seasons), + TvSeason season => 1 + + Count(season.Credits?.Cast) + Count(season.Credits?.Crew) + + Count(season.Episodes), + TvEpisode episode => 1 + + Count(episode.Credits?.Cast) + Count(episode.Credits?.Crew) + + Count(episode.Credits?.GuestStars), + Movie movie => 1 + Count(movie.Credits?.Cast) + Count(movie.Credits?.Crew), + Collection collection => 1 + Count(collection.Parts), + TvGroupCollection groups => 1 + Count(groups.Groups), + FindContainer found => 1 + + Count(found.MovieResults) + Count(found.TvResults) + + Count(found.PersonResults) + Count(found.TvEpisode) + Count(found.TvSeason), + _ => 1 + }; + + /// <summary> + /// Weighs aggregate credits, where each person carries one record per episode they worked on. + /// </summary> + private static long EstimateAggregateSize(CreditsAggregate? credits) + { + if (credits is null) + { + return 0; + } + + var size = Count(credits.Cast) + Count(credits.Crew); + + foreach (var cast in credits.Cast ?? []) + { + size += Count(cast.Roles); + } + + foreach (var crew in credits.Crew ?? []) + { + size += Count(crew.Jobs); + } + + return size; + } + /// <inheritdoc /> public void Dispose() { @@ -768,7 +849,7 @@ namespace MediaBrowser.Providers.Plugins.Tmdb { if (disposing) { - _memoryCache?.Dispose(); + _memoryCache.Dispose(); _tmDbClient?.Dispose(); } } diff --git a/MediaBrowser.Providers/Subtitles/SubtitleManager.cs b/MediaBrowser.Providers/Subtitles/SubtitleManager.cs index c3458d4b2a..cd9dda21a0 100644 --- a/MediaBrowser.Providers/Subtitles/SubtitleManager.cs +++ b/MediaBrowser.Providers/Subtitles/SubtitleManager.cs @@ -33,6 +33,7 @@ namespace MediaBrowser.Providers.Subtitles private readonly ILibraryMonitor _monitor; private readonly IMediaSourceManager _mediaSourceManager; private readonly ILocalizationManager _localization; + private readonly IDirectoryService _directoryService; private readonly HashSet<string> _allowedSubtitleFormats; private readonly ISubtitleProvider[] _subtitleProviders; @@ -43,6 +44,7 @@ namespace MediaBrowser.Providers.Subtitles ILibraryMonitor monitor, IMediaSourceManager mediaSourceManager, ILocalizationManager localizationManager, + IDirectoryService directoryService, IEnumerable<ISubtitleProvider> subtitleProviders, NamingOptions namingOptions) { @@ -51,6 +53,7 @@ namespace MediaBrowser.Providers.Subtitles _monitor = monitor; _mediaSourceManager = mediaSourceManager; _localization = localizationManager; + _directoryService = directoryService; _subtitleProviders = subtitleProviders .OrderBy(i => i is IHasOrder hasOrder ? hasOrder.Order : 0) .ToArray(); @@ -281,6 +284,8 @@ namespace MediaBrowser.Providers.Subtitles await stream.CopyToAsync(fs).ConfigureAwait(false); } + _directoryService.Invalidate(path); + return; } else @@ -395,6 +400,8 @@ namespace MediaBrowser.Providers.Subtitles _monitor.ReportFileSystemChangeComplete(path, false); } + _directoryService.Invalidate(path); + return item.RefreshMetadata(CancellationToken.None); } diff --git a/MediaBrowser.Providers/TV/EpisodeMetadataService.cs b/MediaBrowser.Providers/TV/EpisodeMetadataService.cs index 596ca8d201..f662ac2367 100644 --- a/MediaBrowser.Providers/TV/EpisodeMetadataService.cs +++ b/MediaBrowser.Providers/TV/EpisodeMetadataService.cs @@ -44,6 +44,31 @@ public class EpisodeMetadataService : MetadataService<Episode, EpisodeInfo> { var updatedType = base.BeforeSaveInternal(item, isFullRefresh, updateType); + // An episode cannot end before it starts. + if (item.IndexNumberEnd < item.IndexNumber) + { + Logger.LogWarning( + "Discarding episode range end {IndexNumberEnd} preceding episode number {IndexNumber} for {Path}", + item.IndexNumberEnd, + item.IndexNumber, + item.Path); + + item.IndexNumberEnd = null; + updatedType |= ItemUpdateType.MetadataImport; + } + else if (item.IndexNumberEnd.HasValue && !item.IndexNumber.HasValue) + { + // Without a first episode the end does not describe a range. Promoting it to the episode number + // would invent an identity the metadata never supplied, so drop the orphaned value instead. + Logger.LogWarning( + "Discarding episode range end {IndexNumberEnd} without an episode number for {Path}", + item.IndexNumberEnd, + item.Path); + + item.IndexNumberEnd = null; + updatedType |= ItemUpdateType.MetadataImport; + } + var seriesName = item.FindSeriesName(); if (!string.Equals(item.SeriesName, seriesName, StringComparison.Ordinal)) { diff --git a/MediaBrowser.XbmcMetadata/Parsers/EpisodeNfoParser.cs b/MediaBrowser.XbmcMetadata/Parsers/EpisodeNfoParser.cs index 19b1bbe7b6..c3e5c791c0 100644 --- a/MediaBrowser.XbmcMetadata/Parsers/EpisodeNfoParser.cs +++ b/MediaBrowser.XbmcMetadata/Parsers/EpisodeNfoParser.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Threading; using System.Xml; @@ -44,41 +46,60 @@ namespace MediaBrowser.XbmcMetadata.Parsers var xmlFile = File.ReadAllText(metadataFile); - var srch = "</episodedetails>"; - var index = xmlFile.IndexOf(srch, StringComparison.OrdinalIgnoreCase); - - var xml = xmlFile; + // Split the nfo into its episodedetails blocks. + // This is needed because XBMC metadata uses multiple episodedetails blocks instead of an episodenumberend tag. + const string Srch = "</episodedetails>"; + var blocks = new List<string>(); + int index; + while ((index = xmlFile.IndexOf(Srch, StringComparison.OrdinalIgnoreCase)) != -1) + { + blocks.Add(xmlFile.Substring(0, index + Srch.Length)); + xmlFile = xmlFile.Substring(index + Srch.Length); + } - if (index != -1) + if (blocks.Count == 0) { - xml = xmlFile.Substring(0, index + srch.Length); - xmlFile = xmlFile.Substring(index + srch.Length); + // No closing tag, let the xml reader deal with whatever is in the file + blocks.Add(xmlFile); } // These are not going to be valid xml so no sense in causing the provider to fail and spamming the log with exceptions try { - // Extract episode details from the first episodedetails block - ReadEpisodeDetailsFromXml(item, xml, settings, cancellationToken); + if (blocks.Count == 1) + { + ReadEpisodeDetailsFromXml(item, blocks[0], settings, cancellationToken); + return; + } + + // The blocks are not guaranteed to be written in ascending episode order, so parse them all + // and sort them before merging. + var episodes = blocks + .Select(block => + { + var episode = new MetadataResult<Episode>() + { + Item = new Episode() + }; - // Extract the last episode number from nfo - // Retrieves all additional episodedetails blocks from the rest of the nfo and concatenates the name, originalTitle and overview tags with the first episode - // This is needed because XBMC metadata uses multiple episodedetails blocks instead of episodenumberend tag + ReadEpisodeDetailsFromXml(episode, block, settings, cancellationToken); + + return (Xml: block, Result: episode); + }) + .OrderBy(episode => episode.Result.Item.IndexNumber ?? int.MaxValue) + .ToList(); + + // Extract the details of the lowest numbered episode into the item that is returned to the caller + ReadEpisodeDetailsFromXml(item, episodes[0].Xml, settings, cancellationToken); + + // Concatenate the name, originalTitle and overview tags of the remaining episodes with the first one + // and take the highest episode number as the last episode of the file var name = new StringBuilder(item.Item.Name); var originalTitle = new StringBuilder(item.Item.OriginalTitle); var overview = new StringBuilder(item.Item.Overview); - while ((index = xmlFile.IndexOf(srch, StringComparison.OrdinalIgnoreCase)) != -1) + for (var i = 1; i < episodes.Count; i++) { - xml = xmlFile.Substring(0, index + srch.Length); - xmlFile = xmlFile.Substring(index + srch.Length); - - var additionalEpisode = new MetadataResult<Episode>() - { - Item = new Episode() - }; - - // Extract episode details from additional episodedetails block - ReadEpisodeDetailsFromXml(additionalEpisode, xml, settings, cancellationToken); + var additionalEpisode = episodes[i].Result; if (!string.IsNullOrEmpty(additionalEpisode.Item.Name)) { diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs index 27dbeaba6a..77abb45f2a 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs @@ -37,14 +37,16 @@ public interface IJellyfinDatabaseProvider void ConfigureConventions(ModelConfigurationBuilder configurationBuilder); /// <summary> - /// If supported this should run any periodic maintaince tasks. + /// If supported this should run any periodic maintaince tasks, reclaiming unused space and refreshing the query + /// planner statistics. Also used after migrations have modified the database. /// </summary> /// <param name="cancellationToken">The token to abort the operation.</param> /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> Task RunScheduledOptimisation(CancellationToken cancellationToken); /// <summary> - /// If supported this should perform any actions that are required on stopping the jellyfin server. + /// If supported this should perform any actions that are required on stopping the jellyfin server, including the + /// same maintenance as <see cref="RunScheduledOptimisation(CancellationToken)"/>. /// </summary> /// <param name="cancellationToken">The token that will be used to abort the operation.</param> /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs index 8020fe1f93..f11cde7e48 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs @@ -103,17 +103,9 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider } /// <inheritdoc/> - public async Task RunScheduledOptimisation(CancellationToken cancellationToken) + public Task RunScheduledOptimisation(CancellationToken cancellationToken) { - var context = await DbContextFactory!.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); - await using (context.ConfigureAwait(false)) - { - await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("PRAGMA optimize", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("VACUUM", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); - _logger.LogInformation("jellyfin.db optimized successfully!"); - } + return OptimizeAsync(cancellationToken); } /// <inheritdoc/> @@ -125,19 +117,37 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider /// <inheritdoc/> public async Task RunShutdownTask(CancellationToken cancellationToken) { + // Run before disposing the application + try + { + await OptimizeAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + // A missed optimization only costs performance, so never fail the shutdown over this. + _logger.LogError(ex, "Error while optimizing jellyfin.db"); + } + + SqliteConnection.ClearAllPools(); + } + + private async Task OptimizeAsync(CancellationToken cancellationToken) + { if (DbContextFactory is null) { return; } - // Run before disposing the application var context = await DbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); await using (context.ConfigureAwait(false)) { - await context.Database.ExecuteSqlRawAsync("PRAGMA optimize", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("VACUUM", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("PRAGMA analysis_limit=0", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("ANALYZE", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + _logger.LogInformation("jellyfin.db optimized successfully!"); } - - SqliteConnection.ClearAllPools(); } /// <inheritdoc/> diff --git a/src/Jellyfin.LiveTv/TunerHosts/M3UTunerHost.cs b/src/Jellyfin.LiveTv/TunerHosts/M3UTunerHost.cs index fb606be0ef..902ca76af8 100644 --- a/src/Jellyfin.LiveTv/TunerHosts/M3UTunerHost.cs +++ b/src/Jellyfin.LiveTv/TunerHosts/M3UTunerHost.cs @@ -32,6 +32,7 @@ namespace Jellyfin.LiveTv.TunerHosts { private static readonly string[] _mimeTypesCanShareHttpStream = ["video/MP2T"]; private static readonly string[] _extensionsCanShareHttpStream = [".ts", ".tsv", ".m2t"]; + private static readonly string[] _manifestExtensions = [".m3u8", ".m3u", ".mpd"]; private readonly IHttpClientFactory _httpClientFactory; private readonly IServerApplicationHost _appHost; @@ -151,11 +152,20 @@ namespace Jellyfin.LiveTv.TunerHosts var protocol = _mediaSourceManager.GetPathProtocol(path); var isRemote = true; - if (Uri.TryCreate(path, UriKind.Absolute, out var uri)) + Uri.TryCreate(path, UriKind.Absolute, out var uri); + if (uri is not null) { isRemote = !_networkManager.IsInLocalNetwork(uri.Host); } + // A manifest is not a byte stream. Serving one directly hands the client a playlist whose + // variant and segment URIs are relative to the origin, and those do not resolve against the + // Jellyfin url the client fetched it from. Remux or transcode these instead. + if (IsManifest(path, uri)) + { + supportsDirectPlay = false; + } + var httpHeaders = new Dictionary<string, string>(); if (protocol == MediaProtocol.Http) @@ -210,6 +220,20 @@ namespace Jellyfin.LiveTv.TunerHosts return mediaSource; } + /// <summary> + /// Determines whether a channel path points at an HLS or DASH manifest rather than at a byte stream. + /// </summary> + /// <param name="path">The channel path.</param> + /// <param name="uri">The channel path parsed as an absolute uri, or <c>null</c> if it is not one.</param> + /// <returns><c>true</c> if the path names a streaming manifest.</returns> + private static bool IsManifest(string path, Uri uri) + { + // Use the uri path when there is one so that a query string does not hide the extension. + var extension = Path.GetExtension(uri is null ? path : uri.AbsolutePath); + + return _manifestExtensions.Contains(extension, StringComparison.OrdinalIgnoreCase); + } + public Task<List<TunerHostInfo>> DiscoverDevices(int discoveryDurationMs, CancellationToken cancellationToken) { return Task.FromResult(new List<TunerHostInfo>()); diff --git a/tests/Jellyfin.Controller.Tests/DirectoryServiceTests.cs b/tests/Jellyfin.Controller.Tests/DirectoryServiceTests.cs index 1f59908a86..e57fbfe473 100644 --- a/tests/Jellyfin.Controller.Tests/DirectoryServiceTests.cs +++ b/tests/Jellyfin.Controller.Tests/DirectoryServiceTests.cs @@ -1,3 +1,5 @@ +using System.Globalization; +using System.IO; using System.Linq; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.IO; @@ -8,29 +10,31 @@ namespace Jellyfin.Controller.Tests { public class DirectoryServiceTests { - private const string LowerCasePath = "/music/someartist"; - private const string UpperCasePath = "/music/SOMEARTIST"; + // Path.GetDirectoryName, which Invalidate uses to find the parent, normalizes the + // separators, so cache keys only match the parent it returns when they use the platform's. + private static readonly string _lowerCasePath = LocalPath("/music/someartist"); + private static readonly string _upperCasePath = LocalPath("/music/SOMEARTIST"); private static readonly FileSystemMetadata[] _lowerCaseFileSystemMetadata = { new() { - FullName = LowerCasePath + "/Artwork", + FullName = Path.Combine(_lowerCasePath, "Artwork"), IsDirectory = true }, new() { - FullName = LowerCasePath + "/Some Other Folder", + FullName = Path.Combine(_lowerCasePath, "Some Other Folder"), IsDirectory = true }, new() { - FullName = LowerCasePath + "/Song 2.mp3", + FullName = Path.Combine(_lowerCasePath, "Song 2.mp3"), IsDirectory = false }, new() { - FullName = LowerCasePath + "/Song 3.mp3", + FullName = Path.Combine(_lowerCasePath, "Song 3.mp3"), IsDirectory = false } }; @@ -39,12 +43,12 @@ namespace Jellyfin.Controller.Tests { new() { - FullName = UpperCasePath + "/Lyrics", + FullName = Path.Combine(_upperCasePath, "Lyrics"), IsDirectory = true }, new() { - FullName = UpperCasePath + "/Song 1.mp3", + FullName = Path.Combine(_upperCasePath, "Song 1.mp3"), IsDirectory = false } }; @@ -53,12 +57,12 @@ namespace Jellyfin.Controller.Tests public void GetFileSystemEntries_GivenPathsWithDifferentCasing_CachesAll() { var fileSystemMock = new Mock<IFileSystem>(); - fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == UpperCasePath), false)).Returns(_upperCaseFileSystemMetadata); - fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == LowerCasePath), false)).Returns(_lowerCaseFileSystemMetadata); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == _upperCasePath), false)).Returns(_upperCaseFileSystemMetadata); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == _lowerCasePath), false)).Returns(_lowerCaseFileSystemMetadata); var directoryService = new DirectoryService(fileSystemMock.Object); - var upperCaseResult = directoryService.GetFileSystemEntries(UpperCasePath); - var lowerCaseResult = directoryService.GetFileSystemEntries(LowerCasePath); + var upperCaseResult = directoryService.GetFileSystemEntries(_upperCasePath); + var lowerCaseResult = directoryService.GetFileSystemEntries(_lowerCasePath); Assert.Equal(_upperCaseFileSystemMetadata, upperCaseResult); Assert.Equal(_lowerCaseFileSystemMetadata, lowerCaseResult); @@ -68,12 +72,12 @@ namespace Jellyfin.Controller.Tests public void GetFiles_GivenPathsWithDifferentCasing_ReturnsCorrectFiles() { var fileSystemMock = new Mock<IFileSystem>(); - fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == UpperCasePath), false)).Returns(_upperCaseFileSystemMetadata); - fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == LowerCasePath), false)).Returns(_lowerCaseFileSystemMetadata); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == _upperCasePath), false)).Returns(_upperCaseFileSystemMetadata); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == _lowerCasePath), false)).Returns(_lowerCaseFileSystemMetadata); var directoryService = new DirectoryService(fileSystemMock.Object); - var upperCaseResult = directoryService.GetFiles(UpperCasePath); - var lowerCaseResult = directoryService.GetFiles(LowerCasePath); + var upperCaseResult = directoryService.GetFiles(_upperCasePath); + var lowerCaseResult = directoryService.GetFiles(_lowerCasePath); Assert.Equal(_upperCaseFileSystemMetadata.Where(f => !f.IsDirectory), upperCaseResult); Assert.Equal(_lowerCaseFileSystemMetadata.Where(f => !f.IsDirectory), lowerCaseResult); @@ -83,12 +87,12 @@ namespace Jellyfin.Controller.Tests public void GetDirectories_GivenPathsWithDifferentCasing_ReturnsCorrectDirectories() { var fileSystemMock = new Mock<IFileSystem>(); - fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == UpperCasePath), false)).Returns(_upperCaseFileSystemMetadata); - fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == LowerCasePath), false)).Returns(_lowerCaseFileSystemMetadata); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == _upperCasePath), false)).Returns(_upperCaseFileSystemMetadata); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.Is<string>(x => x == _lowerCasePath), false)).Returns(_lowerCaseFileSystemMetadata); var directoryService = new DirectoryService(fileSystemMock.Object); - var upperCaseResult = directoryService.GetDirectories(UpperCasePath); - var lowerCaseResult = directoryService.GetDirectories(LowerCasePath); + var upperCaseResult = directoryService.GetDirectories(_upperCasePath); + var lowerCaseResult = directoryService.GetDirectories(_lowerCasePath); Assert.Equal(_upperCaseFileSystemMetadata.Where(f => f.IsDirectory), upperCaseResult); Assert.Equal(_lowerCaseFileSystemMetadata.Where(f => f.IsDirectory), lowerCaseResult); @@ -248,5 +252,171 @@ namespace Jellyfin.Controller.Tests Assert.Equal(cachedPaths, result); Assert.Equal(newPaths, secondResult); } + + [Fact] + public void GetFileSystemEntries_RepeatedPath_ReadsTheFileSystemOnce() + { + var fileSystemMock = new Mock<IFileSystem>(MockBehavior.Strict); + fileSystemMock.Setup(f => f.GetFileSystemEntries(_lowerCasePath)) + .Returns(_lowerCaseFileSystemMetadata); + + var directoryService = new DirectoryService(fileSystemMock.Object); + + directoryService.GetFileSystemEntries(_lowerCasePath); + directoryService.GetFileSystemEntries(_lowerCasePath); + + fileSystemMock.Verify(f => f.GetFileSystemEntries(_lowerCasePath), Times.Once); + } + + [Fact] + public void Invalidate_GivenADirectory_DropsBothTheListingAndTheFilePaths() + { + var fileSystemMock = new Mock<IFileSystem>(); + fileSystemMock.SetupSequence(f => f.GetFileSystemEntries(_lowerCasePath)) + .Returns(_lowerCaseFileSystemMetadata) + .Returns(_upperCaseFileSystemMetadata); + fileSystemMock.SetupSequence(f => f.GetFilePaths(_lowerCasePath, false)) + .Returns(new[] { Path.Combine(_lowerCasePath, "Song 2.mp3") }) + .Returns(new[] { Path.Combine(_lowerCasePath, "Song 2.mp3"), Path.Combine(_lowerCasePath, "Song 2.srt") }); + + var directoryService = new DirectoryService(fileSystemMock.Object); + directoryService.GetFileSystemEntries(_lowerCasePath); + directoryService.GetFilePaths(_lowerCasePath); + + directoryService.Invalidate(_lowerCasePath); + + Assert.Equal(_upperCaseFileSystemMetadata, directoryService.GetFileSystemEntries(_lowerCasePath)); + Assert.Equal(2, directoryService.GetFilePaths(_lowerCasePath).Count); + } + + [Fact] + public void Invalidate_GivenAFile_DropsTheListingOfTheDirectoryHoldingIt() + { + var newFile = Path.Combine(_lowerCasePath, "Song 2.srt"); + + var fileSystemMock = new Mock<IFileSystem>(); + fileSystemMock.SetupSequence(f => f.GetFileSystemEntries(_lowerCasePath)) + .Returns(_lowerCaseFileSystemMetadata) + .Returns(_upperCaseFileSystemMetadata); + + var directoryService = new DirectoryService(fileSystemMock.Object); + directoryService.GetFileSystemEntries(_lowerCasePath); + + directoryService.Invalidate(newFile); + + Assert.Equal(_upperCaseFileSystemMetadata, directoryService.GetFileSystemEntries(_lowerCasePath)); + } + + [Fact] + public void GetFilePaths_ClearingTheCache_KeepsTheParentDirectory() + { + var parentPath = LocalPath("/music"); + + var fileSystemMock = new Mock<IFileSystem>(); + fileSystemMock.Setup(f => f.GetFilePaths(_lowerCasePath)) + .Returns(new[] { Path.Combine(_lowerCasePath, "Song 2.mp3") }); + fileSystemMock.Setup(f => f.GetFileSystemEntries(parentPath)) + .Returns(_lowerCaseFileSystemMetadata); + + var directoryService = new DirectoryService(fileSystemMock.Object); + directoryService.GetFileSystemEntries(parentPath); + + directoryService.GetFilePaths(_lowerCasePath, true); + + directoryService.GetFileSystemEntries(parentPath); + fileSystemMock.Verify(f => f.GetFileSystemEntries(parentPath), Times.Once); + } + + [Fact] + public void GetFileSystemEntries_MoreRecordsThanTheCeiling_DropsCache() + { + // Charged by the files in a listing, not the number of listings, so a few big folders + // reach the limit where a lot of small ones would not. + const int FolderCount = 60; + var bigListing = new FileSystemMetadata[5000]; + for (var i = 0; i < bigListing.Length; i++) + { + bigListing[i] = new FileSystemMetadata + { + FullName = "/music/track" + i.ToString(CultureInfo.InvariantCulture), + IsDirectory = false + }; + } + + var fileSystemMock = new Mock<IFileSystem>(); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.IsAny<string>())) + .Returns(bigListing); + + var directoryService = new DirectoryService(fileSystemMock.Object); + + const string FirstPath = "/music/artist0"; + directoryService.GetFileSystemEntries(FirstPath); + + for (var i = 1; i < FolderCount; i++) + { + directoryService.GetFileSystemEntries("/music/artist" + i.ToString(CultureInfo.InvariantCulture)); + } + + directoryService.GetFileSystemEntries(FirstPath); + + fileSystemMock.Verify(f => f.GetFileSystemEntries(FirstPath), Times.Exactly(2)); + } + + [Fact] + public void GetFileSystemEntries_RepeatedlyInvalidatedFolder_KeepsUnrelatedEntriesCached() + { + // Invalidating gives the records back, so churning one folder must not add up to the + // ceiling and drop everything else with it. + const int ChurnCount = 50; + var bigListing = new FileSystemMetadata[5000]; + for (var i = 0; i < bigListing.Length; i++) + { + bigListing[i] = new FileSystemMetadata + { + FullName = "/music/track" + i.ToString(CultureInfo.InvariantCulture), + IsDirectory = false + }; + } + + var fileSystemMock = new Mock<IFileSystem>(); + fileSystemMock.Setup(f => f.GetFileSystemEntries(It.IsAny<string>())) + .Returns(bigListing); + + var directoryService = new DirectoryService(fileSystemMock.Object); + + const string ChurnedPath = "/music/watched"; + const string StablePath = "/music/untouched"; + directoryService.GetFileSystemEntries(StablePath); + + for (var i = 0; i < ChurnCount; i++) + { + directoryService.GetFileSystemEntries(ChurnedPath); + directoryService.Invalidate(ChurnedPath); + } + + directoryService.GetFileSystemEntries(StablePath); + + fileSystemMock.Verify(f => f.GetFileSystemEntries(StablePath), Times.Once); + } + + [Fact] + public void GetFileSystemEntry_MissingPath_IsNotRemembered() + { + const string MissingPath = "/music/not-here"; + + var fileSystemMock = new Mock<IFileSystem>(); + fileSystemMock.SetupSequence(f => f.GetFileSystemInfo(MissingPath)) + .Returns(new FileSystemMetadata { FullName = MissingPath, Exists = false }) + .Returns(new FileSystemMetadata { FullName = MissingPath, Exists = true }); + + var directoryService = new DirectoryService(fileSystemMock.Object); + + Assert.Null(directoryService.GetFileSystemEntry(MissingPath)); + + Assert.NotNull(directoryService.GetFileSystemEntry(MissingPath)); + } + + private static string LocalPath(string path) + => path.Replace('/', Path.DirectorySeparatorChar); } } diff --git a/tests/Jellyfin.Controller.Tests/LibraryTaskScheduler/LimitedConcurrencyLibrarySchedulerTests.cs b/tests/Jellyfin.Controller.Tests/LibraryTaskScheduler/LimitedConcurrencyLibrarySchedulerTests.cs new file mode 100644 index 0000000000..686d839f4f --- /dev/null +++ b/tests/Jellyfin.Controller.Tests/LibraryTaskScheduler/LimitedConcurrencyLibrarySchedulerTests.cs @@ -0,0 +1,216 @@ +using System; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.LibraryTaskScheduler; +using MediaBrowser.Model.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace Jellyfin.Controller.Tests.LibraryTaskScheduler +{ + public class LimitedConcurrencyLibrarySchedulerTests + { + private static readonly TimeSpan _shortGracePeriod = TimeSpan.FromMilliseconds(50); + + // Generous, because these only ever wait for something that should already have happened. + private static readonly TimeSpan _timeout = TimeSpan.FromSeconds(10); + + [Fact] + public async Task Enqueue_ProcessesEveryItem() + { + using var appStopping = new CancellationTokenSource(); + var scheduler = CreateScheduler(appStopping); + await using (scheduler) + { + var data = Enumerable.Range(0, 100).ToArray(); + var processed = new ConcurrentBag<int>(); + + await scheduler.Enqueue( + data, + (item, _) => + { + processed.Add(item); + return Task.CompletedTask; + }, + new Progress<double>(), + CancellationToken.None); + + Assert.Equal(data, processed.Order()); + } + } + + [Fact] + public async Task Enqueue_WithFailingWorker_StillCompletes() + { + using var appStopping = new CancellationTokenSource(); + var scheduler = CreateScheduler(appStopping); + await using (scheduler) + { + await scheduler.Enqueue( + Enumerable.Range(0, 20).ToArray(), + (item, _) => item % 2 == 0 ? throw new InvalidOperationException("boom") : Task.CompletedTask, + new Progress<double>(), + CancellationToken.None); + } + } + + /// <summary> + /// The runners wait on a source linked to <see cref="IHostApplicationLifetime.ApplicationStopping"/>, + /// so a shutdown has to reach them. It does not travel from the linked source back to the one + /// the cleanup cancels, which is what made them immortal. + /// </summary> + /// <returns>A <see cref="Task"/> representing the asynchronous unit test.</returns> + [Fact] + public async Task ApplicationStopping_RetiresRunners() + { + using var appStopping = new CancellationTokenSource(); + + // Long enough that the cleanup cannot be what retires them. + var scheduler = CreateScheduler(appStopping, gracePeriod: TimeSpan.FromMinutes(5)); + await using (scheduler) + { + await RunOneOperation(scheduler); + Assert.True(scheduler.ActiveRunnerCount > 0); + + await appStopping.CancelAsync(); + + await WaitForAsync(() => scheduler.ActiveRunnerCount == 0); + } + } + + /// <summary> + /// The cleanup used to be a one shot: it never released the scheduling slot it took, so + /// every runner spawned after the first pass stayed around for the lifetime of the server. + /// </summary> + /// <returns>A <see cref="Task"/> representing the asynchronous unit test.</returns> + [Fact] + public async Task Enqueue_RetiresIdleRunnersAfterEveryOperation() + { + using var appStopping = new CancellationTokenSource(); + var scheduler = CreateScheduler(appStopping); + await using (scheduler) + { + for (var round = 0; round < 3; round++) + { + await RunOneOperation(scheduler); + Assert.True(scheduler.ActiveRunnerCount > 0, $"no runner spawned in round {round}"); + + await WaitForAsync(() => scheduler.ActiveRunnerCount == 0); + } + } + } + + /// <summary> + /// Disposing used to sit out the rest of the cleanup grace period, holding up shutdown for + /// up to a minute. + /// </summary> + /// <returns>A <see cref="Task"/> representing the asynchronous unit test.</returns> + [Fact] + public async Task DisposeAsync_DoesNotWaitOutTheGracePeriod() + { + using var appStopping = new CancellationTokenSource(); + var scheduler = CreateScheduler(appStopping, gracePeriod: TimeSpan.FromMinutes(5)); + + await RunOneOperation(scheduler); + + var stopwatch = Stopwatch.StartNew(); + await scheduler.DisposeAsync(); + + Assert.True(stopwatch.Elapsed < _timeout, $"disposing took {stopwatch.Elapsed}"); + } + + [Fact] + public async Task Enqueue_AfterDispose_DoesNothing() + { + using var appStopping = new CancellationTokenSource(); + var scheduler = CreateScheduler(appStopping); + await scheduler.DisposeAsync(); + + var processed = 0; + await scheduler.Enqueue( + Enumerable.Range(0, 10).ToArray(), + (_, _) => + { + Interlocked.Increment(ref processed); + return Task.CompletedTask; + }, + new Progress<double>(), + CancellationToken.None); + + Assert.Equal(0, processed); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public async Task Enqueue_FromWithinAWorker_DoesNotDeadlock(int fanout) + { + using var appStopping = new CancellationTokenSource(); + var scheduler = CreateScheduler(appStopping, fanout: fanout); + await using (scheduler) + { + var inner = 0; + + var outer = scheduler.Enqueue( + Enumerable.Range(0, 8).ToArray(), + (_, _) => scheduler.Enqueue( + Enumerable.Range(0, 4).ToArray(), + (_, _) => + { + Interlocked.Increment(ref inner); + return Task.CompletedTask; + }, + new Progress<double>(), + CancellationToken.None), + new Progress<double>(), + CancellationToken.None); + + await outer.WaitAsync(_timeout, TestContext.Current.CancellationToken); + + Assert.Equal(32, inner); + } + } + + private static LimitedConcurrencyLibraryScheduler CreateScheduler( + CancellationTokenSource appStopping, + int fanout = 4, + TimeSpan? gracePeriod = null) + { + var lifetime = new Mock<IHostApplicationLifetime>(); + lifetime.SetupGet(x => x.ApplicationStopping).Returns(() => appStopping.Token); + + var configurationManager = new Mock<IServerConfigurationManager>(); + configurationManager.SetupGet(x => x.Configuration) + .Returns(new ServerConfiguration { LibraryScanFanoutConcurrency = fanout }); + + return new LimitedConcurrencyLibraryScheduler( + lifetime.Object, + NullLogger<LimitedConcurrencyLibraryScheduler>.Instance, + configurationManager.Object, + gracePeriod ?? _shortGracePeriod); + } + + private static Task RunOneOperation(LimitedConcurrencyLibraryScheduler scheduler) + => scheduler.Enqueue( + Enumerable.Range(0, 8).ToArray(), + (_, _) => Task.CompletedTask, + new Progress<double>(), + CancellationToken.None); + + private static async Task WaitForAsync(Func<bool> condition) + { + var stopwatch = Stopwatch.StartNew(); + while (!condition()) + { + Assert.True(stopwatch.Elapsed < _timeout, "timed out waiting for the scheduler to settle"); + await Task.Delay(20, TestContext.Current.CancellationToken); + } + } + } +} diff --git a/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperDoviTests.cs b/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperDoviTests.cs new file mode 100644 index 0000000000..557035e2d1 --- /dev/null +++ b/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperDoviTests.cs @@ -0,0 +1,162 @@ +using System; +using Jellyfin.Data.Enums; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.MediaEncoding; +using MediaBrowser.Controller.Streaming; +using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.Dto; +using MediaBrowser.Model.Entities; +using Moq; +using Xunit; + +using IConfiguration = Microsoft.Extensions.Configuration.IConfiguration; + +namespace Jellyfin.Controller.Tests.MediaEncoding; + +public class EncodingHelperDoviTests +{ + [Theory] + [InlineData(null, false)] + [InlineData("bt709", false)] + [InlineData("unknown", false)] + [InlineData("bt2020-10", false)] + [InlineData("smpte2084", true)] + [InlineData("arib-std-b67", true)] + public void GetSwVidFilterChain_InvalidDovi_OnlyTonemapsHdrBaseLayer(string? transfer, bool tonemap) + { + var state = CreateState("hevc", transfer); + var helper = CreateHelper(true); + + var (filters, _, _) = helper.GetSwVidFilterChain(state, new EncodingOptions(), "libx264"); + var args = string.Join(',', filters); + + Assert.Equal(VideoRangeType.DOVIInvalid, state.VideoStream.VideoRangeType); + Assert.Equal(tonemap, args.Contains("tonemapx=", StringComparison.Ordinal)); + Assert.Contains(tonemap ? "color_trc=" + transfer : "color_trc=bt709", args, StringComparison.Ordinal); + } + + [Theory] + [InlineData(null, false)] + [InlineData("bt709", false)] + [InlineData("arib-std-b67", false)] + [InlineData("smpte2084", true)] + [InlineData("SMPTE2084", true)] + public void IsDoviWithHdr10Bl_InvalidDovi_RequiresPq(string? transfer, bool expected) + { + var stream = CreateState("hevc", transfer).VideoStream; + + Assert.True(EncodingHelper.IsDovi(stream)); + Assert.Equal(expected, EncodingHelper.IsDoviWithHdr10Bl(stream)); + } + + [Theory] + [InlineData("hevc", null, "hevc_metadata=remove_dovi=1")] + [InlineData("hevc", "bt709", "hevc_metadata=remove_dovi=1")] + [InlineData("hevc", "smpte2084", "hevc_metadata=remove_dovi=1")] + [InlineData("hevc", "arib-std-b67", "hevc_metadata=remove_dovi=1")] + [InlineData("av1", null, "av1_metadata=remove_dovi=1")] + [InlineData("av1", "bt709", "av1_metadata=remove_dovi=1")] + [InlineData("av1", "smpte2084", "av1_metadata=remove_dovi=1")] + [InlineData("av1", "arib-std-b67", "av1_metadata=remove_dovi=1")] + public void GetBitStreamArgs_InvalidDovi_PreservesClientDependentRemoval(string codec, string? transfer, string expected) + { + var state = CreateState(codec, transfer); + var helper = CreateHelper(true); + + foreach (var (requestedRanges, removeDovi) in new[] { (null, false), ("SDR", false), ("HDR10", false), ("DOVIWithEL", false), ("DOVI", true), ("SDR,DOVI", true) }) + { + state.BaseRequest.VideoRangeType = requestedRanges; + + Assert.Equal(removeDovi, helper.IsDoviRemoved(state)); + if (removeDovi) + { + Assert.Contains(expected, helper.GetBitStreamArgs(state, MediaStreamType.Video), StringComparison.Ordinal); + } + else + { + Assert.Equal(codec == "hevc" ? "-bsf:v hevc_mp4toannexb" : null, helper.GetBitStreamArgs(state, MediaStreamType.Video)); + } + + Assert.False(CreateHelper(false).IsDoviRemoved(state)); + } + } + + [Theory] + [InlineData(null, true)] + [InlineData("HDR10", true)] + [InlineData("DOVI", false)] + [InlineData("SDR,DOVI", false)] + public void CanStreamCopyVideo_InvalidDovi_RequiresRemovalSupportOnlyForDoviClients(string? requestedRanges, bool copyWithoutRemovalSupport) + { + foreach (var codec in new[] { "hevc", "av1" }) + { + foreach (var transfer in new[] { "bt709", "smpte2084" }) + { + var state = CreateState(codec, transfer); + state.BaseRequest.VideoRangeType = requestedRanges; + + Assert.True(CreateHelper(true).CanStreamCopyVideo(state, state.VideoStream)); + Assert.Equal(copyWithoutRemovalSupport, CreateHelper(false).CanStreamCopyVideo(state, state.VideoStream)); + } + } + } + + [Fact] + public void GetBitStreamArgs_ValidDovi_PreservesMetadata() + { + var state = CreateState("hevc", "smpte2084"); + state.VideoStream.ColorSpace = "bt2020nc"; + state.VideoStream.ColorPrimaries = "bt2020"; + state.BaseRequest.VideoRangeType = "DOVIWithEL"; + var helper = CreateHelper(true); + + Assert.False(helper.IsDoviRemoved(state)); + Assert.Equal("-bsf:v hevc_mp4toannexb", helper.GetBitStreamArgs(state, MediaStreamType.Video)); + } + + private static EncodingJobInfo CreateState(string codec, string? transfer) + { + var stream = new MediaStream + { + Type = MediaStreamType.Video, + Codec = codec, + Width = 1920, + Height = 1080, + BitDepth = 10, + DvProfile = codec == "hevc" ? 7 : 10, + DvBlSignalCompatibilityId = codec == "hevc" ? 6 : 1, + RpuPresentFlag = 1, + BlPresentFlag = 1, + ColorSpace = "bt709", + ColorPrimaries = "bt709", + ColorTransfer = transfer + }; + + return new EncodingJobInfo(TranscodingJobType.Hls) + { + VideoStream = stream, + MediaSource = new MediaSourceInfo { Container = "mkv", MediaStreams = [stream] }, + BaseRequest = new VideoRequestDto(), + OutputVideoCodec = "copy", + IsVideoRequest = true, + IsInputVideo = true + }; + } + + private static EncodingHelper CreateHelper(bool supportsRemoval) + { + var encoder = new Mock<IMediaEncoder>(); + encoder.Setup(x => x.SupportsBitStreamFilterWithOption(It.IsAny<BitStreamFilterOptionType>())).Returns(supportsRemoval); + encoder.Setup(x => x.SupportsFilter("tonemapx")).Returns(true); + encoder.SetupGet(x => x.EncoderVersion).Returns(new Version(8, 1)); + + return new EncodingHelper( + Mock.Of<IApplicationPaths>(), + encoder.Object, + Mock.Of<ISubtitleEncoder>(), + Mock.Of<IConfiguration>(), + Mock.Of<IConfigurationManager>(), + Mock.Of<IPathManager>()); + } +} diff --git a/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperInferAudioCodecTests.cs b/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperInferAudioCodecTests.cs new file mode 100644 index 0000000000..586db2dd50 --- /dev/null +++ b/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperInferAudioCodecTests.cs @@ -0,0 +1,40 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.MediaEncoding; +using Moq; +using Xunit; +using IConfiguration = Microsoft.Extensions.Configuration.IConfiguration; + +namespace Jellyfin.Controller.Tests.MediaEncoding; + +public class EncodingHelperInferAudioCodecTests +{ + [Theory] + // Manifests and other containers that carry no inferable audio codec. + [InlineData("m3u8", "aac")] + [InlineData("mpd", "aac")] + [InlineData("wtv", "aac")] + [InlineData("", "aac")] + // Containers with a well known audio codec. + [InlineData("mp4", "aac")] + [InlineData("mkv", "aac")] + [InlineData("webm", "opus")] + [InlineData("ts", "mp3")] + // Containers named after the codec they carry. + [InlineData("flac", "flac")] + [InlineData("opus", "opus")] + [InlineData("ac3", "ac3")] + public void InferAudioCodec_ReturnsAnAudioCodec(string container, string expected) + { + Assert.Equal(expected, Create().InferAudioCodec(container)); + } + + private static EncodingHelper Create() + => new( + Mock.Of<IApplicationPaths>(), + Mock.Of<IMediaEncoder>(), + Mock.Of<ISubtitleEncoder>(), + Mock.Of<IConfiguration>(), + Mock.Of<IConfigurationManager>(), + Mock.Of<IPathManager>()); +} diff --git a/tests/Jellyfin.LiveTv.Tests/M3UTunerHostTests.cs b/tests/Jellyfin.LiveTv.Tests/M3UTunerHostTests.cs new file mode 100644 index 0000000000..4487a5ff2b --- /dev/null +++ b/tests/Jellyfin.LiveTv.Tests/M3UTunerHostTests.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.LiveTv.TunerHosts; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.LiveTv; +using MediaBrowser.Model.Dto; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.LiveTv; +using MediaBrowser.Model.MediaInfo; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Jellyfin.LiveTv.Tests +{ + public class M3UTunerHostTests + { + [Theory] + // A manifest is not a byte stream, so it must never be offered for direct play. + [InlineData("http://example.com/live/1234.m3u8", false)] + [InlineData("http://example.com/live/1234.m3u8?token=abc", false)] + [InlineData("http://example.com/live/1234.mpd", false)] + // Byte streams are unaffected. + [InlineData("http://example.com/live/1234.ts", true)] + [InlineData("http://example.com/live/1234", true)] + public async Task GetChannelStreamMediaSources_ManifestPath_DisablesDirectPlay(string path, bool expectDirectPlay) + { + var mediaSourceManager = new Mock<IMediaSourceManager>(); + mediaSourceManager.Setup(x => x.GetPathProtocol(It.IsAny<string>())).Returns(MediaProtocol.Http); + + var host = new TestableM3UTunerHost( + Mock.Of<IServerConfigurationManager>(), + mediaSourceManager.Object, + Mock.Of<ILogger<M3UTunerHost>>(), + Mock.Of<IFileSystem>(), + Mock.Of<IHttpClientFactory>(), + Mock.Of<IServerApplicationHost>(), + Mock.Of<INetworkManager>(), + Mock.Of<IStreamHelper>()); + + var sources = await host.GetMediaSources( + new TunerHostInfo { TunerCount = 0, EnableStreamLooping = false }, + new ChannelInfo { Path = path }); + + Assert.Equal(expectDirectPlay, sources[0].SupportsDirectPlay); + } + + private sealed class TestableM3UTunerHost : M3UTunerHost + { + public TestableM3UTunerHost( + IServerConfigurationManager config, + IMediaSourceManager mediaSourceManager, + ILogger<M3UTunerHost> logger, + IFileSystem fileSystem, + IHttpClientFactory httpClientFactory, + IServerApplicationHost appHost, + INetworkManager networkManager, + IStreamHelper streamHelper) + : base(config, mediaSourceManager, logger, fileSystem, httpClientFactory, appHost, networkManager, streamHelper) + { + } + + public Task<List<MediaSourceInfo>> GetMediaSources(TunerHostInfo tuner, ChannelInfo channel) + => GetChannelStreamMediaSources(tuner, channel, CancellationToken.None); + } + } +} diff --git a/tests/Jellyfin.MediaEncoding.Tests/Encoder/ProcessWrapperTests.cs b/tests/Jellyfin.MediaEncoding.Tests/Encoder/ProcessWrapperTests.cs new file mode 100644 index 0000000000..141164815c --- /dev/null +++ b/tests/Jellyfin.MediaEncoding.Tests/Encoder/ProcessWrapperTests.cs @@ -0,0 +1,103 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.MediaEncoding.Encoder; +using MediaBrowser.Model.Globalization; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.MediaInfo; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Jellyfin.MediaEncoding.Tests.Encoder; + +public class ProcessWrapperTests +{ + [Fact] + public async Task ExitedProcess_StaysUsableForTheCallerThatStartedIt() + { + using var process = CreateProcess(); + using var exitHandled = new ManualResetEventSlim(false); + + using (var wrapper = new MediaEncoder.ProcessWrapper(process, CreateEncoder())) + { + // Subscribed after the wrapper, so by the time this is set the wrapper's own handler has + // already run: whatever it does to the process has happened. + process.Exited += (_, _) => exitHandled.Set(); + + process.Start(); + await process.WaitForExitAsync(TestContext.Current.CancellationToken).ConfigureAwait(true); + + Assert.True(exitHandled.Wait(TimeSpan.FromSeconds(15), TestContext.Current.CancellationToken), "The process never raised Exited."); + + // The caller still owns the process here. Disposing it from the exit handler handed + // whoever exited quickest an ObjectDisposedException out of these three lines. + var output = await process.StandardOutput.ReadToEndAsync(TestContext.Current.CancellationToken).ConfigureAwait(true); + Assert.Equal("jellyfin", output.Trim()); + + Assert.True(wrapper.HasExited); + Assert.Equal(3, wrapper.ExitCode); + } + } + + [Fact] + public async Task ExitState_IsReadableBeforeTheExitEventArrives() + { + using var process = CreateProcess(); + + using (var wrapper = new MediaEncoder.ProcessWrapper(process, CreateEncoder())) + { + process.Start(); + await process.WaitForExitAsync(TestContext.Current.CancellationToken).ConfigureAwait(true); + + // The exit event is raised on the thread pool and can lag behind the wait that just + // returned, so neither of these may depend on it having arrived. + Assert.True(wrapper.HasExited); + Assert.Equal(3, wrapper.ExitCode); + } + } + + [Fact] + public async Task ExitCode_SurvivesDisposal() + { + using var process = CreateProcess(); + var wrapper = new MediaEncoder.ProcessWrapper(process, CreateEncoder()); + + process.Start(); + await process.WaitForExitAsync(TestContext.Current.CancellationToken).ConfigureAwait(true); + + var exitCode = wrapper.ExitCode; + wrapper.Dispose(); + + Assert.Equal(exitCode, wrapper.ExitCode); + Assert.True(wrapper.HasExited); + } + + private static MediaEncoder CreateEncoder() + => new( + Mock.Of<ILogger<MediaEncoder>>(), + Mock.Of<IServerConfigurationManager>(), + Mock.Of<IFileSystem>(), + Mock.Of<IBlurayExaminer>(), + Mock.Of<ILocalizationManager>(), + new ConfigurationBuilder().Build(), + Mock.Of<IServerConfigurationManager>()); + + // Writes to stdout and exits immediately with a non-zero code, standing in for the ffprobe that + // rejects a file outright - the process that used to win the race against its own caller. + private static Process CreateProcess() + { + var startInfo = OperatingSystem.IsWindows() + ? new ProcessStartInfo("cmd.exe", "/c echo jellyfin & exit 3") + : new ProcessStartInfo("/bin/sh", "-c \"printf 'jellyfin\\n'; exit 3\""); + + startInfo.CreateNoWindow = true; + startInfo.UseShellExecute = false; + startInfo.RedirectStandardOutput = true; + + return new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + } +} diff --git a/tests/Jellyfin.Model.Tests/Dlna/StreamBuilderManifestContainerTests.cs b/tests/Jellyfin.Model.Tests/Dlna/StreamBuilderManifestContainerTests.cs new file mode 100644 index 0000000000..dfd1eb2e85 --- /dev/null +++ b/tests/Jellyfin.Model.Tests/Dlna/StreamBuilderManifestContainerTests.cs @@ -0,0 +1,104 @@ +using System; +using Jellyfin.Data.Enums; +using MediaBrowser.Model.Dlna; +using MediaBrowser.Model.Dto; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.MediaInfo; +using MediaBrowser.Model.Session; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace Jellyfin.Model.Tests.Dlna; + +public class StreamBuilderManifestContainerTests +{ + [Theory] + // A manifest describes a stream instead of carrying one, so it can never be direct played, + // even when the client claims to support the container. + [InlineData("hls")] + [InlineData("hls,applehttp")] + [InlineData("applehttp")] + [InlineData("dash")] + public void GetOptimalVideoStream_ManifestContainer_DoesNotDirectPlay(string container) + { + var streamInfo = BuildFor(container); + + Assert.NotNull(streamInfo); + Assert.Equal(PlayMethod.Transcode, streamInfo.PlayMethod); + } + + [Fact] + public void GetOptimalVideoStream_ByteStreamContainer_StillDirectPlays() + { + var streamInfo = BuildFor("mp4"); + + Assert.NotNull(streamInfo); + Assert.Equal(PlayMethod.DirectPlay, streamInfo.PlayMethod); + } + + private static StreamInfo? BuildFor(string container) + { + var mediaSource = new MediaSourceInfo + { + Id = "test-source", + Path = "http://example.com/live/channel", + Protocol = MediaProtocol.Http, + Container = container, + SupportsDirectPlay = true, + SupportsDirectStream = true, + SupportsTranscoding = true, + IsInfiniteStream = true, + IsRemote = true, + MediaStreams = + [ + new MediaStream { Type = MediaStreamType.Video, Index = 0, Codec = "h264" }, + new MediaStream { Type = MediaStreamType.Audio, Index = 1, Codec = "aac" } + ] + }; + + var profile = new DeviceProfile + { + Name = "Manifest aware client", + DirectPlayProfiles = + [ + new DirectPlayProfile + { + Type = DlnaProfileType.Video, + Container = "mp4,hls,applehttp,dash", + VideoCodec = "h264", + AudioCodec = "aac" + } + ], + TranscodingProfiles = + [ + new TranscodingProfile + { + Type = DlnaProfileType.Video, + Context = EncodingContext.Streaming, + Protocol = MediaStreamProtocol.hls, + Container = "ts", + VideoCodec = "h264", + AudioCodec = "aac" + } + ] + }; + + var options = new MediaOptions + { + ItemId = new Guid("11D229B7-2D48-4B95-9F9B-49F6AB75E613"), + MediaSourceId = mediaSource.Id, + MediaSources = [mediaSource], + DeviceId = "test-deviceId", + Profile = profile, + AllowAudioStreamCopy = true, + AllowVideoStreamCopy = true, + EnableDirectStream = false // This is disabled in server + }; + + var transcodeSupport = new Mock<ITranscoderSupport>(); + + return new StreamBuilder(transcodeSupport.Object, new NullLogger<StreamBuilderManifestContainerTests>()) + .GetOptimalVideoStream(options); + } +} diff --git a/tests/Jellyfin.Model.Tests/Entities/MediaStreamVideoRangeTests.cs b/tests/Jellyfin.Model.Tests/Entities/MediaStreamVideoRangeTests.cs new file mode 100644 index 0000000000..f264e5a019 --- /dev/null +++ b/tests/Jellyfin.Model.Tests/Entities/MediaStreamVideoRangeTests.cs @@ -0,0 +1,129 @@ +using Jellyfin.Data.Enums; +using MediaBrowser.Model.Entities; +using Xunit; + +namespace Jellyfin.Model.Tests.Entities; + +public class MediaStreamVideoRangeTests +{ + [Theory] + [InlineData(7, 6, "smpte2084", false, VideoRangeType.DOVIWithEL)] + [InlineData(7, 6, "smpte2084", true, VideoRangeType.DOVIWithELHDR10Plus)] + [InlineData(8, 1, "smpte2084", false, VideoRangeType.DOVIWithHDR10)] + [InlineData(8, 1, "smpte2084", true, VideoRangeType.DOVIWithHDR10Plus)] + [InlineData(8, 4, "arib-std-b67", false, VideoRangeType.DOVIWithHLG)] + [InlineData(10, 1, "smpte2084", false, VideoRangeType.DOVIWithHDR10)] + [InlineData(10, 1, "smpte2084", true, VideoRangeType.DOVIWithHDR10Plus)] + [InlineData(10, 4, "arib-std-b67", false, VideoRangeType.DOVIWithHLG)] + [InlineData(8, 1, "SMPTE2084", false, VideoRangeType.DOVIWithHDR10)] + [InlineData(8, 4, "ARIB-STD-B67", false, VideoRangeType.DOVIWithHLG)] + public void GetVideoColorRange_ValidDovi_PreservesRangeType( + int profile, int compatibilityId, string transfer, bool hdr10Plus, VideoRangeType expected) + { + var stream = CreateDovi(profile, compatibilityId, "BT2020NC", transfer, "BT2020", hdr10Plus); + + Assert.Equal((VideoRange.HDR, expected), stream.GetVideoColorRange()); + } + + [Theory] + [InlineData("bt709", "bt709", "bt709", VideoRange.SDR)] + [InlineData("bt2020nc", "bt709", "bt2020", VideoRange.SDR)] + [InlineData("bt2020nc", null, "bt2020", VideoRange.SDR)] + [InlineData("bt2020nc", "", "bt2020", VideoRange.SDR)] + [InlineData("bt2020nc", "unknown", "bt2020", VideoRange.SDR)] + [InlineData("bt2020nc", "bt2020-10", "bt2020", VideoRange.SDR)] + [InlineData(null, null, null, VideoRange.SDR)] + [InlineData("bt709", "smpte2084", "bt2020", VideoRange.HDR)] + [InlineData("bt2020nc", "smpte2084", "bt709", VideoRange.HDR)] + [InlineData(null, "smpte2084", "bt2020", VideoRange.HDR)] + [InlineData("bt2020nc", "smpte2084", null, VideoRange.HDR)] + [InlineData("bt709", "arib-std-b67", "bt2020", VideoRange.HDR)] + [InlineData("bt2020nc", "arib-std-b67", "bt709", VideoRange.HDR)] + [InlineData(null, "arib-std-b67", "bt2020", VideoRange.HDR)] + [InlineData("bt2020nc", "arib-std-b67", null, VideoRange.HDR)] + public void GetVideoColorRange_InvalidDoviColors_UsesBaseLayerRange( + string? space, string? transfer, string? primaries, VideoRange expected) + { + // Cover every HDR-compatible DV profile, including the HDR10+ variants. + foreach (var (profile, compatibilityId) in new[] { (7, 6), (8, 1), (8, 4), (10, 1), (10, 4) }) + { + foreach (var hdr10Plus in new[] { false, true }) + { + var stream = CreateDovi(profile, compatibilityId, space, transfer, primaries, hdr10Plus); + + Assert.Equal(expected, stream.VideoRange); + Assert.Equal(VideoRangeType.DOVIInvalid, stream.VideoRangeType); + } + } + } + + [Theory] + [InlineData(7, 6, "arib-std-b67")] + [InlineData(8, 1, "arib-std-b67")] + [InlineData(8, 4, "smpte2084")] + [InlineData(10, 1, "arib-std-b67")] + [InlineData(10, 4, "smpte2084")] + public void GetVideoColorRange_WrongHdrTransfer_InvalidButStillHdr(int profile, int compatibilityId, string transfer) + { + var stream = CreateDovi(profile, compatibilityId, "bt2020nc", transfer, "bt2020", true); + + Assert.Equal((VideoRange.HDR, VideoRangeType.DOVIInvalid), stream.GetVideoColorRange()); + } + + [Theory] + [InlineData(5, 0, null, VideoRange.HDR, VideoRangeType.DOVI)] + [InlineData(10, 0, null, VideoRange.HDR, VideoRangeType.DOVI)] + [InlineData(8, 2, "bt709", VideoRange.SDR, VideoRangeType.DOVIWithSDR)] + [InlineData(10, 2, "bt709", VideoRange.SDR, VideoRangeType.DOVIWithSDR)] + public void GetVideoColorRange_OtherDoviProfiles_PreservesClassification( + int profile, int compatibilityId, string? transfer, VideoRange range, VideoRangeType rangeType) + { + var stream = CreateDovi(profile, compatibilityId, "bt709", transfer, "bt709", false); + + Assert.Equal((range, rangeType), stream.GetVideoColorRange()); + } + + [Theory] + [InlineData(8, null, VideoRange.SDR)] + [InlineData(8, "bt709", VideoRange.SDR)] + [InlineData(8, "smpte2084", VideoRange.HDR)] + [InlineData(10, null, VideoRange.SDR)] + [InlineData(10, "arib-std-b67", VideoRange.HDR)] + public void GetVideoColorRange_InvalidCompatibilityId_UsesBaseLayerRange(int profile, string? transfer, VideoRange expected) + { + var stream = CreateDovi(profile, 6, "bt2020nc", transfer, "bt2020", false); + + Assert.Equal((expected, VideoRangeType.DOVIInvalid), stream.GetVideoColorRange()); + } + + [Theory] + [InlineData("bt709", false, VideoRange.SDR, VideoRangeType.SDR)] + [InlineData(null, false, VideoRange.SDR, VideoRangeType.SDR)] + [InlineData("smpte2084", false, VideoRange.HDR, VideoRangeType.HDR10)] + [InlineData("smpte2084", true, VideoRange.HDR, VideoRangeType.HDR10Plus)] + [InlineData("arib-std-b67", false, VideoRange.HDR, VideoRangeType.HLG)] + public void GetVideoColorRange_WithoutDovi_PreservesClassification( + string? transfer, bool hdr10Plus, VideoRange range, VideoRangeType rangeType) + { + var stream = new MediaStream { Type = MediaStreamType.Video, ColorTransfer = transfer, Hdr10PlusPresentFlag = hdr10Plus }; + + Assert.Equal((range, rangeType), stream.GetVideoColorRange()); + stream.Type = MediaStreamType.Audio; + Assert.Equal((VideoRange.Unknown, VideoRangeType.Unknown), stream.GetVideoColorRange()); + } + + private static MediaStream CreateDovi(int profile, int compatibilityId, string? space, string? transfer, string? primaries, bool hdr10Plus) + => new() + { + Type = MediaStreamType.Video, + DvProfile = profile, + DvBlSignalCompatibilityId = compatibilityId, + RpuPresentFlag = 1, + BlPresentFlag = 1, + ElPresentFlag = profile == 7 ? 1 : 0, + ColorSpace = space, + ColorTransfer = transfer, + ColorPrimaries = primaries, + Hdr10PlusPresentFlag = hdr10Plus + }; +} diff --git a/tests/Jellyfin.Naming.Tests/TV/MultiEpisodeTests.cs b/tests/Jellyfin.Naming.Tests/TV/MultiEpisodeTests.cs index 7e708c681d..4236749423 100644 --- a/tests/Jellyfin.Naming.Tests/TV/MultiEpisodeTests.cs +++ b/tests/Jellyfin.Naming.Tests/TV/MultiEpisodeTests.cs @@ -74,6 +74,9 @@ namespace Jellyfin.Naming.Tests.TV [InlineData("Season 5/S05E23 11-59 [HDTV-1080p][x265 AC3].mkv", null)] [InlineData("Season 5/S05E23 11-59 [HDTV-1080p][HEVC AC3].mkv", null)] [InlineData("Season 1/S01E01 1-23-45 [Bluray-1080p][AV1 Opus].mkv", null)] + // Episode markers in the episode title must not be read as an episode range + [InlineData("Season 03/Star Trek Enterprise (2001) - S03E21 - E2 (1080p BluRay x265).mkv", null)] + [InlineData("Season 02/Series Name (2001) - S02E10 - E5 [WEBRip-1080p].mkv", null)] public void TestGetEndingEpisodeNumberFromFile(string filename, int? endingEpisodeNumber) { var result = _episodePathParser.Parse(filename, false); diff --git a/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs b/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs index 5749944fcd..248b236df8 100644 --- a/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs +++ b/tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net.Http; @@ -377,6 +378,116 @@ namespace Jellyfin.Providers.Tests.Manager GetMetadataProviders_CanRefreshMetadata_Tester(providerType, expected, ownedItem: true); } + [Fact] + public async Task QueueRefresh_ManyItemsQueuedFromManyThreads_ProcessesEveryOne() + { + const int ItemCount = 2000; + + var queued = Enumerable.Range(0, ItemCount).Select(_ => Guid.NewGuid()).ToArray(); + var processed = new ConcurrentBag<Guid>(); + var allProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var libraryManager = new Mock<ILibraryManager>(); + libraryManager.Setup(i => i.GetItemById(It.IsAny<Guid>())) + .Returns((Guid id) => + { + // Returning null drains the entry without the whole refresh machinery. + processed.Add(id); + if (processed.Count == ItemCount) + { + allProcessed.TrySetResult(); + } + + return null; + }); + + using var providerManager = GetProviderManager(libraryManager: libraryManager.Object); + + await Parallel.ForEachAsync( + queued, + TestContext.Current.CancellationToken, + (id, _) => + { + providerManager.QueueRefresh(id, new MetadataRefreshOptions(Mock.Of<IDirectoryService>()), RefreshPriority.Normal); + return ValueTask.CompletedTask; + }); + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + + try + { + await allProcessed.Task.WaitAsync(timeout.Token); + } + catch (OperationCanceledException) + { + // Fall through so the assertions report what was lost. + } + + Assert.Empty(providerManager.GetRefreshQueue()); + Assert.Equal(queued.Order().ToArray(), processed.Order().ToArray()); + } + + [Fact] + public async Task QueueRefresh_RefreshCancelsForItsOwnReasons_KeepsDrainingTheQueue() + { + // A provider timeout arrives as an OperationCanceledException, indistinguishable from + // a shutdown; treating it as one would strand the rest of the queue. + const int ItemCount = 200; + + var queued = Enumerable.Range(0, ItemCount).Select(_ => Guid.NewGuid()).ToArray(); + var processed = new ConcurrentBag<Guid>(); + var allProcessed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var allQueued = new ManualResetEventSlim(false); + var cancelledOnce = false; + + var libraryManager = new Mock<ILibraryManager>(); + libraryManager.Setup(i => i.GetItemById(It.IsAny<Guid>())) + .Returns((Guid id) => + { + if (!cancelledOnce) + { + cancelledOnce = true; + + // Hold the first entry until the whole batch is queued. + allQueued.Wait(TimeSpan.FromSeconds(30)); + throw new OperationCanceledException("provider timed out"); + } + + processed.Add(id); + if (processed.Count == ItemCount - 1) + { + allProcessed.TrySetResult(); + } + + return null; + }); + + using var providerManager = GetProviderManager(libraryManager: libraryManager.Object); + + foreach (var id in queued) + { + providerManager.QueueRefresh(id, new MetadataRefreshOptions(Mock.Of<IDirectoryService>()), RefreshPriority.Normal); + } + + allQueued.Set(); + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + + try + { + await allProcessed.Task.WaitAsync(timeout.Token); + } + catch (OperationCanceledException) + { + // Fall through so the assertions report what was stranded. + } + + Assert.Empty(providerManager.GetRefreshQueue()); + Assert.Equal(ItemCount - 1, processed.Count); + } + private static void GetMetadataProviders_CanRefreshMetadata_Tester( string providerType, bool expected, @@ -554,15 +665,20 @@ namespace Jellyfin.Providers.Tests.Manager private static ProviderManager GetProviderManager( ServerConfiguration? serverConfiguration = null, LibraryOptions? libraryOptions = null, - IBaseItemManager? baseItemManager = null) + IBaseItemManager? baseItemManager = null, + ILibraryManager? libraryManager = null) { var serverConfigurationManager = new Mock<IServerConfigurationManager>(MockBehavior.Strict); serverConfigurationManager.Setup(i => i.Configuration) .Returns(serverConfiguration ?? new ServerConfiguration()); - var libraryManager = new Mock<ILibraryManager>(MockBehavior.Strict); - libraryManager.Setup(i => i.GetLibraryOptions(It.IsAny<BaseItem>())) - .Returns(libraryOptions ?? new LibraryOptions()); + if (libraryManager is null) + { + var libraryManagerMock = new Mock<ILibraryManager>(MockBehavior.Strict); + libraryManagerMock.Setup(i => i.GetLibraryOptions(It.IsAny<BaseItem>())) + .Returns(libraryOptions ?? new LibraryOptions()); + libraryManager = libraryManagerMock.Object; + } var providerManager = new ProviderManager( Mock.Of<IHttpClientFactory>(), @@ -572,7 +688,7 @@ namespace Jellyfin.Providers.Tests.Manager _logger, Mock.Of<IFileSystem>(), Mock.Of<IServerApplicationPaths>(), - libraryManager.Object, + libraryManager, baseItemManager!, Mock.Of<ILyricManager>(), Mock.Of<IMemoryCache>(), diff --git a/tests/Jellyfin.Providers.Tests/MediaInfo/MediaInfoResolverTests.cs b/tests/Jellyfin.Providers.Tests/MediaInfo/MediaInfoResolverTests.cs index 876f18741f..ce451861ef 100644 --- a/tests/Jellyfin.Providers.Tests/MediaInfo/MediaInfoResolverTests.cs +++ b/tests/Jellyfin.Providers.Tests/MediaInfo/MediaInfoResolverTests.cs @@ -179,6 +179,146 @@ public class MediaInfoResolverTests Assert.Empty(streams); } + [Fact] + public void GetExternalFiles_VobSubIdxAndSubPair_OnlyReturnsIdxFile() + { + // VobSub (.sub) payloads only carry per-track language metadata when read + // alongside their paired .idx index file. When both are present, only the + // .idx file should be returned so it (not the raw .sub) gets probed. + BaseItem.MediaSourceManager = Mock.Of<IMediaSourceManager>(); + + var video = new Movie + { + Path = VideoDirectoryPath + "/My.Video.mkv" + }; + + var directoryService = new Mock<IDirectoryService>(MockBehavior.Strict); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(VideoDirectoryRegex), It.IsAny<bool>())) + .Returns(new[] { VideoDirectoryPath + "/My.Video.idx", VideoDirectoryPath + "/My.Video.sub" }); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(MetadataDirectoryRegex), It.IsAny<bool>())) + .Returns(Array.Empty<string>()); + + var streams = _subtitleResolver.GetExternalFiles(video, directoryService.Object, false).ToList(); + + var stream = Assert.Single(streams); + Assert.EndsWith(".idx", stream.Path, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void GetExternalFiles_VobSubIdxWithoutMatchingSub_DoesNotReturnIdxFile() + { + // An .idx file with no paired .sub cannot be probed for anything, so it must be + // left out entirely rather than surfaced as a doomed-to-fail probe candidate. + // Surfacing it anyway would also make it "exist" from Jellyfin's perspective + // even after the real .sub is deleted, preventing stale subtitle stream data + // from ever being cleared on a rescan. + BaseItem.MediaSourceManager = Mock.Of<IMediaSourceManager>(); + + var video = new Movie + { + Path = VideoDirectoryPath + "/My.Video.mkv" + }; + + var directoryService = GetDirectoryServiceForExternalFile("My.Video.idx"); + var streams = _subtitleResolver.GetExternalFiles(video, directoryService, false).ToList(); + + Assert.Empty(streams); + } + + [Fact] + public void GetExternalFiles_StandaloneSubWithoutIdx_StillReturnsSubFile() + { + // Guards against the .idx/.sub pairing suppression firing when there is no + // .idx sidecar at all - a lone .sub file must still be returned. + BaseItem.MediaSourceManager = Mock.Of<IMediaSourceManager>(); + + var video = new Movie + { + Path = VideoDirectoryPath + "/My.Video.mkv" + }; + + var directoryService = GetDirectoryServiceForExternalFile("My.Video.sub"); + var streams = _subtitleResolver.GetExternalFiles(video, directoryService, false).ToList(); + + var stream = Assert.Single(streams); + Assert.EndsWith(".sub", stream.Path, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void GetExternalFiles_VobSubIdxAndSubInDifferentDirectories_DoesNotPair() + { + // A same-named .idx and .sub split across the video folder and the internal + // metadata folder cannot be paired by ffprobe (it only looks next to the .idx), + // so the .sub must still be returned, but the orphaned .idx (no sibling .sub in + // its own directory) must be left out since it cannot be probed. + BaseItem.MediaSourceManager = Mock.Of<IMediaSourceManager>(); + + var video = new Movie + { + Path = VideoDirectoryPath + "/My.Video.mkv" + }; + + var directoryService = new Mock<IDirectoryService>(MockBehavior.Strict); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(VideoDirectoryRegex), It.IsAny<bool>())) + .Returns(new[] { VideoDirectoryPath + "/My.Video.sub" }); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(MetadataDirectoryRegex), It.IsAny<bool>())) + .Returns(new[] { MetadataDirectoryPath + "/My.Video.idx" }); + + var streams = _subtitleResolver.GetExternalFiles(video, directoryService.Object, false).ToList(); + + var stream = Assert.Single(streams); + Assert.EndsWith(".sub", stream.Path, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void GetExternalFiles_VobSubIdxAndSubWithMatchingLanguageFlag_SuppressesSub() + { + // A .idx/.sub pair sharing the same filename flags (e.g. a language token) should + // still pair and suppress the .sub, just like an unflagged pair. + BaseItem.MediaSourceManager = Mock.Of<IMediaSourceManager>(); + + var video = new Movie + { + Path = VideoDirectoryPath + "/My.Video.mkv" + }; + + var directoryService = new Mock<IDirectoryService>(MockBehavior.Strict); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(VideoDirectoryRegex), It.IsAny<bool>())) + .Returns(new[] { VideoDirectoryPath + "/My.Video.en.idx", VideoDirectoryPath + "/My.Video.en.sub" }); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(MetadataDirectoryRegex), It.IsAny<bool>())) + .Returns(Array.Empty<string>()); + + var streams = _subtitleResolver.GetExternalFiles(video, directoryService.Object, false).ToList(); + + var stream = Assert.Single(streams); + Assert.EndsWith(".idx", stream.Path, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void GetExternalFiles_VobSubIdxAndSubWithMismatchedNames_DoesNotPair() + { + // An .idx and .sub with different basenames (e.g. differing filename flags) are not + // a pair ffprobe would resolve. The .sub must still be returned, but the orphaned + // .idx (no same-named sibling .sub) must be left out since it cannot be probed. + BaseItem.MediaSourceManager = Mock.Of<IMediaSourceManager>(); + + var video = new Movie + { + Path = VideoDirectoryPath + "/My.Video.mkv" + }; + + var directoryService = new Mock<IDirectoryService>(MockBehavior.Strict); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(VideoDirectoryRegex), It.IsAny<bool>())) + .Returns(new[] { VideoDirectoryPath + "/My.Video.idx", VideoDirectoryPath + "/My.Video.en.sub" }); + directoryService.Setup(ds => ds.GetFilePaths(It.IsRegex(MetadataDirectoryRegex), It.IsAny<bool>())) + .Returns(Array.Empty<string>()); + + var streams = _subtitleResolver.GetExternalFiles(video, directoryService.Object, false).ToList(); + + var stream = Assert.Single(streams); + Assert.EndsWith(".sub", stream.Path, StringComparison.OrdinalIgnoreCase); + } + [Theory] [InlineData("https://url.com/My.Video.mkv")] [InlineData(VideoDirectoryPath)] // valid but no files found for this test diff --git a/tests/Jellyfin.Providers.Tests/TV/EpisodeMetadataServiceTests.cs b/tests/Jellyfin.Providers.Tests/TV/EpisodeMetadataServiceTests.cs index 8f5b1b3c48..ea762256db 100644 --- a/tests/Jellyfin.Providers.Tests/TV/EpisodeMetadataServiceTests.cs +++ b/tests/Jellyfin.Providers.Tests/TV/EpisodeMetadataServiceTests.cs @@ -1,5 +1,6 @@ using System; using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; @@ -15,9 +16,23 @@ using Xunit; namespace Jellyfin.Providers.Tests.TV; -public class EpisodeMetadataServiceTests +// put tests that mock the static LibraryManager in the same collection to avoid test interference +[Collection("LibraryManagerTests")] +public sealed class EpisodeMetadataServiceTests : IDisposable { private readonly TestEpisodeMetadataService _service = new(); + private readonly ILibraryManager? _previousLibraryManager; + + public EpisodeMetadataServiceTests() + { + _previousLibraryManager = BaseItem.LibraryManager; + BaseItem.LibraryManager = Mock.Of<ILibraryManager>(); + } + + public void Dispose() + { + BaseItem.LibraryManager = _previousLibraryManager; + } [Fact] public void MergeData_ProviderSeasonOverridesPathDerivedSeason() @@ -88,6 +103,59 @@ public class EpisodeMetadataServiceTests Assert.Equal(1, target.Item.ParentIndexNumber); } + [Theory] + [InlineData(2, 1)] + [InlineData(22, 21)] + [InlineData(21, 2)] // e.g. "Series - S03E21 - E2 (1080p BluRay x265).mkv", where "E2" is the episode title + public void BeforeSave_ReversedEpisodeRange_ClearsIndexNumberEnd(int indexNumber, int indexNumberEnd) + { + var item = new Episode + { + IndexNumber = indexNumber, + IndexNumberEnd = indexNumberEnd + }; + + var updateType = _service.BeforeSave(item); + + // The episode number identifies the item, so it is kept and the impossible range is dropped + Assert.Equal(indexNumber, item.IndexNumber); + Assert.Null(item.IndexNumberEnd); + Assert.True(updateType.HasFlag(ItemUpdateType.MetadataImport)); + } + + [Fact] + public void BeforeSave_EpisodeRangeWithoutStart_ClearsIndexNumberEnd() + { + var item = new Episode + { + IndexNumber = null, + IndexNumberEnd = 2 + }; + + var updateType = _service.BeforeSave(item); + + Assert.Null(item.IndexNumberEnd); + Assert.Null(item.IndexNumber); + Assert.True(updateType.HasFlag(ItemUpdateType.MetadataImport)); + } + + [Theory] + [InlineData(1, 2)] // Regular multi episode file + [InlineData(1, 1)] // Degenerate but not contradictory + public void BeforeSave_ValidEpisodeRange_KeepsIndexNumberEnd(int indexNumber, int indexNumberEnd) + { + var item = new Episode + { + IndexNumber = indexNumber, + IndexNumberEnd = indexNumberEnd + }; + + _service.BeforeSave(item); + + Assert.Equal(indexNumber, item.IndexNumber); + Assert.Equal(indexNumberEnd, item.IndexNumberEnd); + } + private sealed class TestEpisodeMetadataService : EpisodeMetadataService { public TestEpisodeMetadataService() @@ -106,5 +174,10 @@ public class EpisodeMetadataServiceTests { MergeData(source, target, Array.Empty<MetadataField>(), replaceData, mergeMetadataSettings); } + + public ItemUpdateType BeforeSave(Episode item) + { + return BeforeSaveInternal(item, false, ItemUpdateType.None); + } } } diff --git a/tests/Jellyfin.Server.Implementations.Tests/EntryPoints/LibraryChangedNotifierTests.cs b/tests/Jellyfin.Server.Implementations.Tests/EntryPoints/LibraryChangedNotifierTests.cs new file mode 100644 index 0000000000..cdb261de8d --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/EntryPoints/LibraryChangedNotifierTests.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations.EntryPoints; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Controller.Session; +using MediaBrowser.Model.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.EntryPoints; + +public class LibraryChangedNotifierTests +{ + // How long a test waits for the notifier's timer callback to run. Generous: the assertions are + // about a batch being sent at all, not about how promptly. + private static readonly TimeSpan _flushTimeout = TimeSpan.FromSeconds(15); + + private readonly Mock<ILibraryManager> _libraryManager = new(); + private readonly Mock<IServerConfigurationManager> _configurationManager = new(); + private readonly Mock<ISessionManager> _sessionManager = new(); + private readonly Mock<IUserManager> _userManager = new(); + private readonly Mock<IProviderManager> _providerManager = new(); + private readonly ServerConfiguration _configuration = new(); + + private int _flushCount; + + public LibraryChangedNotifierTests() + { + _configurationManager.SetupGet(e => e.Configuration).Returns(_configuration); + + // Reading the session list is the first thing a flush does, so it stands in for "a batch was + // sent" without having to mock a whole user library behind it. + _sessionManager.SetupGet(e => e.Sessions) + .Returns(() => + { + Interlocked.Increment(ref _flushCount); + return []; + }); + } + + [Fact] + public async Task OnLibraryItemUpdated_BatchSizeCapReached_SendsWithoutWaitingForWindow() + { + // Long enough that only the size cap can close the batch. + _configuration.LibraryUpdateDuration = 3600; + + var notifier = CreateNotifier(); + await notifier.StartAsync(TestContext.Current.CancellationToken); + + for (var i = 0; i < LibraryChangedNotifier.MaxBatchSize; i++) + { + RaiseItemUpdated(); + } + + Assert.True(await WaitForFlushAsync(1), "The batch was not sent once it hit the size cap."); + + await notifier.StopAsync(TestContext.Current.CancellationToken); + notifier.Dispose(); + } + + [Fact] + public async Task OnLibraryItemUpdated_ChangesNeverPause_StillSendsOnTheWindow() + { + // A scan changes items continuously. The window must run from the first change of a batch, or + // the batch never closes and holds every item it named alive for the length of the scan. + _configuration.LibraryUpdateDuration = 1; + + var notifier = CreateNotifier(); + await notifier.StartAsync(TestContext.Current.CancellationToken); + + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < _flushTimeout && Volatile.Read(ref _flushCount) == 0) + { + // Well below the window, and well below the size cap over the whole loop. + RaiseItemUpdated(); + await Task.Delay(25, TestContext.Current.CancellationToken); + } + + Assert.True(Volatile.Read(ref _flushCount) > 0, "The batch was never sent while changes kept arriving."); + + await notifier.StopAsync(TestContext.Current.CancellationToken); + notifier.Dispose(); + } + + private LibraryChangedNotifier CreateNotifier() + => new( + _libraryManager.Object, + _configurationManager.Object, + _sessionManager.Object, + _userManager.Object, + NullLogger<LibraryChangedNotifier>.Instance, + _providerManager.Object); + + // A folder passes the notifier's item filter without needing any of BaseItem's static services. + private void RaiseItemUpdated() + => _libraryManager.Raise( + e => e.ItemUpdated += null, + _libraryManager.Object, + new ItemChangeEventArgs { Item = new Folder { Id = Guid.NewGuid() } }); + + private async Task<bool> WaitForFlushAsync(int expected) + { + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < _flushTimeout) + { + if (Volatile.Read(ref _flushCount) >= expected) + { + return true; + } + + await Task.Delay(25, TestContext.Current.CancellationToken); + } + + return false; + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/EntryPoints/UserDataChangeNotifierTests.cs b/tests/Jellyfin.Server.Implementations.Tests/EntryPoints/UserDataChangeNotifierTests.cs new file mode 100644 index 0000000000..0274398f89 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/EntryPoints/UserDataChangeNotifierTests.cs @@ -0,0 +1,78 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations.EntryPoints; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Session; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Session; +using Moq; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.EntryPoints; + +public class UserDataChangeNotifierTests +{ + // How long a test waits for the notifier's timer callback to run. Generous: the assertions are + // about a batch being sent at all, not about how promptly. + private static readonly TimeSpan _flushTimeout = TimeSpan.FromSeconds(15); + + private readonly Mock<IUserDataManager> _userDataManager = new(); + private readonly Mock<ISessionManager> _sessionManager = new(); + private readonly Mock<IUserManager> _userManager = new(); + + private int _flushCount; + + public UserDataChangeNotifierTests() + { + _sessionManager + .Setup(e => e.SendMessageToUserSessions( + It.IsAny<System.Collections.Generic.List<Guid>>(), + SessionMessageType.UserDataChanged, + It.IsAny<Func<UserDataChangeInfo>>(), + It.IsAny<CancellationToken>())) + .Callback(() => Interlocked.Increment(ref _flushCount)) + .Returns(Task.CompletedTask); + } + + [Fact] + public async Task OnUserDataSaved_ChangesNeverPause_StillSendsOnTheWindow() + { + // A scan changes user data continuously. The window must run from the first change of a batch, + // or the batch never closes and holds every item it named alive for the length of the scan. + var notifier = CreateNotifier(); + await notifier.StartAsync(TestContext.Current.CancellationToken); + + var userId = Guid.NewGuid(); + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < _flushTimeout && Volatile.Read(ref _flushCount) == 0) + { + // Well below the window, and well below the size cap over the whole loop. + RaiseUserDataSaved(userId); + await Task.Delay(25, TestContext.Current.CancellationToken); + } + + Assert.True(Volatile.Read(ref _flushCount) > 0, "The batch was never sent while changes kept arriving."); + + await notifier.StopAsync(TestContext.Current.CancellationToken); + notifier.Dispose(); + } + + private UserDataChangeNotifier CreateNotifier() + => new(_userDataManager.Object, _sessionManager.Object, _userManager.Object); + + // A folder needs none of BaseItem's static services, and PlaybackProgress is the one reason the + // notifier ignores outright. + private void RaiseUserDataSaved(Guid userId) + => _userDataManager.Raise( + e => e.UserDataSaved += null, + _userDataManager.Object, + new UserDataSaveEventArgs + { + UserId = userId, + SaveReason = UserDataSaveReason.UpdateUserRating, + Item = new Folder { Id = Guid.NewGuid() } + }); +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/Item/BaseItemRepositoryItemValueTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Item/BaseItemRepositoryItemValueTests.cs new file mode 100644 index 0000000000..91148501ce --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Item/BaseItemRepositoryItemValueTests.cs @@ -0,0 +1,174 @@ +using System; +using Emby.Server.Implementations.Data; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Server.Implementations.Item; +using MediaBrowser.Controller.Entities; +using Xunit; +using BaseItemKind = Jellyfin.Data.Enums.BaseItemKind; + +namespace Jellyfin.Server.Implementations.Tests.Item; + +public sealed class BaseItemRepositoryItemValueTests : SqliteDbTestFixture +{ + private readonly BaseItemRepository _repository; + private readonly string _audioTypeName; + private readonly string _movieTypeName; + + public BaseItemRepositoryItemValueTests() + { + var itemTypeLookup = new ItemTypeLookup(); + _audioTypeName = itemTypeLookup.BaseItemKindNames[BaseItemKind.Audio]; + _movieTypeName = itemTypeLookup.BaseItemKindNames[BaseItemKind.Movie]; + _repository = CreateBaseItemRepository(itemTypeLookup); + } + + [Fact] + public void GetQueryFiltersLegacy_GroupsAndFiltersItemValues() + { + var firstItem = CreateMovieEntity(Guid.NewGuid(), "First"); + var secondItem = CreateMovieEntity(Guid.NewGuid(), "Second"); + var excludedItem = new BaseItemEntity + { + Id = Guid.NewGuid(), + Type = _audioTypeName, + Name = "Excluded Audio", + MediaType = "Audio", + IsMovie = false, + IsFolder = false, + IsVirtualItem = false + }; + var firstTag = new ItemValue + { + ItemValueId = Guid.NewGuid(), + Type = ItemValueType.Tags, + Value = "Alpha", + CleanValue = "alpha" + }; + var duplicateTag = new ItemValue + { + ItemValueId = Guid.NewGuid(), + Type = ItemValueType.Tags, + Value = "alpha", + CleanValue = "alpha" + }; + var secondTag = new ItemValue + { + ItemValueId = Guid.NewGuid(), + Type = ItemValueType.Tags, + Value = "Beta", + CleanValue = "beta" + }; + var genre = new ItemValue + { + ItemValueId = Guid.NewGuid(), + Type = ItemValueType.Genre, + Value = "Genre Leak", + CleanValue = "genre leak" + }; + var excludedTag = new ItemValue + { + ItemValueId = Guid.NewGuid(), + Type = ItemValueType.Tags, + Value = "Excluded Tag", + CleanValue = "excluded tag" + }; + var excludedGenre = new ItemValue + { + ItemValueId = Guid.NewGuid(), + Type = ItemValueType.Genre, + Value = "Excluded Genre", + CleanValue = "excluded genre" + }; + + using (var context = CreateDbContext()) + { + context.BaseItems.AddRange(firstItem, secondItem, excludedItem); + context.ItemValues.AddRange(firstTag, duplicateTag, secondTag, genre, excludedTag, excludedGenre); + context.ItemValuesMap.AddRange( + CreateMap(firstItem, firstTag), + CreateMap(firstItem, duplicateTag), + CreateMap(secondItem, secondTag), + CreateMap(firstItem, genre), + CreateMap(excludedItem, excludedTag), + CreateMap(excludedItem, excludedGenre)); + context.SaveChanges(); + } + + var result = _repository.GetQueryFiltersLegacy(new InternalItemsQuery(new Database.Implementations.Entities.User("test", "auth", "reset")) + { + IncludeItemTypes = [BaseItemKind.Movie] + }); + + Assert.Equal(["Alpha", "Beta"], result.Tags); + Assert.Equal(["Genre Leak"], result.Genres); + } + + [Fact] + public void GetGenreNames_GroupsAndFiltersMappedItemValues() + { + var movie = CreateMovieEntity(Guid.NewGuid(), "Movie"); + var audio = new BaseItemEntity + { + Id = Guid.NewGuid(), + Type = _audioTypeName, + Name = "Audio", + MediaType = "Audio", + IsFolder = false, + IsVirtualItem = false + }; + var movieGenre = CreateItemValue(ItemValueType.Genre, "Movie Genre", "movie genre"); + var duplicateMovieGenre = CreateItemValue(ItemValueType.Genre, "movie genre", "movie genre"); + var musicGenre = CreateItemValue(ItemValueType.Genre, "Music Genre", "music genre"); + var orphanedGenre = CreateItemValue(ItemValueType.Genre, "Orphaned Genre", "orphaned genre"); + + using (var context = CreateDbContext()) + { + context.BaseItems.AddRange(movie, audio); + context.ItemValues.AddRange(movieGenre, duplicateMovieGenre, musicGenre, orphanedGenre); + context.ItemValuesMap.AddRange( + CreateMap(movie, movieGenre), + CreateMap(movie, duplicateMovieGenre), + CreateMap(audio, musicGenre)); + context.SaveChanges(); + } + + Assert.Equal(["Movie Genre"], _repository.GetGenreNames()); + Assert.Equal(["Music Genre"], _repository.GetMusicGenreNames()); + } + + private BaseItemEntity CreateMovieEntity(Guid id, string name) + { + return new BaseItemEntity + { + Id = id, + Type = _movieTypeName, + Name = name, + MediaType = "Video", + IsMovie = true, + IsFolder = false, + IsVirtualItem = false + }; + } + + private static ItemValueMap CreateMap(BaseItemEntity item, ItemValue itemValue) + { + return new ItemValueMap + { + ItemId = item.Id, + ItemValueId = itemValue.ItemValueId, + Item = item, + ItemValue = itemValue + }; + } + + private static ItemValue CreateItemValue(ItemValueType type, string value, string cleanValue) + { + return new ItemValue + { + ItemValueId = Guid.NewGuid(), + Type = type, + Value = value, + CleanValue = cleanValue + }; + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/Item/ItemPersistenceServiceSaveImagesTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Item/ItemPersistenceServiceSaveImagesTests.cs new file mode 100644 index 0000000000..7997c6d771 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Item/ItemPersistenceServiceSaveImagesTests.cs @@ -0,0 +1,77 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Server.Implementations.Item; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Model.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.Item; + +public class ItemPersistenceServiceSaveImagesTests : SqliteDbTestFixture +{ + private readonly ItemPersistenceService _service; + + public ItemPersistenceServiceSaveImagesTests() + { + _service = new ItemPersistenceService( + CreateDbContextFactory(), + Mock.Of<IServerApplicationHost>(), + NullLogger<ItemPersistenceService>.Instance); + } + + [Fact] + public async Task SaveImagesAsync_ReplacesThePreviousImages() + { + var itemId = Guid.NewGuid(); + Seed(itemId); + + await _service.SaveImagesAsync(CreateItem(itemId, "/first.jpg"), TestContext.Current.CancellationToken); + await _service.SaveImagesAsync(CreateItem(itemId, "/second.jpg"), TestContext.Current.CancellationToken); + + using var context = CreateDbContext(); + var paths = context.BaseItemImageInfos + .Where(e => e.ItemId.Equals(itemId)) + .Select(e => e.Path) + .ToList(); + + Assert.Equal(["/second.jpg"], paths); + } + + [Fact] + public async Task SaveImagesAsync_ItemDeletedFromUnderIt_IsANoOp() + { + // A scan can delete the item between the refresh reading it and the images being written. That + // must not fail the whole refresh, and must not leave the images of an item that is gone. + var itemId = Guid.NewGuid(); + + await _service.SaveImagesAsync(CreateItem(itemId, "/gone.jpg"), TestContext.Current.CancellationToken); + + using var context = CreateDbContext(); + Assert.Empty(context.BaseItemImageInfos.Where(e => e.ItemId.Equals(itemId))); + } + + private static BaseItem CreateItem(Guid itemId, string imagePath) + => new Folder + { + Id = itemId, + ImageInfos = [new ItemImageInfo { Path = imagePath, Type = ImageType.Primary }] + }; + + private void Seed(Guid itemId) + { + using var context = CreateDbContext(); + context.BaseItems.Add(new BaseItemEntity + { + Id = itemId, + Type = "Folder", + IsFolder = true + }); + context.SaveChanges(); + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/Item/SqliteDbTestFixture.cs b/tests/Jellyfin.Server.Implementations.Tests/Item/SqliteDbTestFixture.cs index 87efa8fea5..cfc9c9496c 100644 --- a/tests/Jellyfin.Server.Implementations.Tests/Item/SqliteDbTestFixture.cs +++ b/tests/Jellyfin.Server.Implementations.Tests/Item/SqliteDbTestFixture.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using Emby.Server.Implementations.Data; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Locking; @@ -58,6 +59,8 @@ public abstract class SqliteDbTestFixture : IDisposable { var factory = new Mock<IDbContextFactory<JellyfinDbContext>>(); factory.Setup(f => f.CreateDbContext()).Returns(CreateDbContext); + factory.Setup(f => f.CreateDbContextAsync(It.IsAny<CancellationToken>())) + .ReturnsAsync(CreateDbContext); return factory.Object; } diff --git a/tests/Jellyfin.Server.Implementations.Tests/Library/PeopleValidatorPartitionTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Library/PeopleValidatorPartitionTests.cs new file mode 100644 index 0000000000..30f7bed208 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Library/PeopleValidatorPartitionTests.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using Emby.Server.Implementations.Library.Validators; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.Library; + +/// <summary> +/// Tests for how the people validator decides which credits need a person item and which person items +/// nothing credits any more. Keying either half on the item's name rather than its id put the two halves +/// in a loop that created, refreshed and deleted the same people on every run, so these pin the id. +/// </summary> +public class PeopleValidatorPartitionTests +{ + // Stands in for the real item-by-name id: derived from the credit name, case-insensitively, and + // from nothing else. The property that matters is that it does not depend on the item's own name. + private static Guid PersonId(string creditName) + { +#pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms + var hash = System.Security.Cryptography.MD5.HashData( + System.Text.Encoding.Unicode.GetBytes(creditName.ToLowerInvariant())); +#pragma warning restore CA5351 // Do Not Use Broken Cryptographic Algorithms + return new Guid(hash); + } + + [Fact] + public void PartitionCreditsByPersonId_ProviderRenamedThePerson_KeepsThemAndCreatesNothing() + { + // The credit still says "AURORA"; the item it made has been renamed to "Aurora" by the provider + // that refreshed it. Nothing about the library changed, so nothing should be created or deleted. + var credits = new[] { "AURORA" }; + var existing = new HashSet<Guid> { PersonId("AURORA") }; + + var (newNames, deadIds) = PeopleValidator.PartitionCreditsByPersonId(credits, PersonId, existing); + + Assert.Empty(newNames); + Assert.Empty(deadIds); + } + + [Theory] + // Every shape of rename seen in the wild on a real library. + [InlineData("AURORA")] + [InlineData("Amir AboulEla")] + [InlineData("Miguel Ángel Fuentes")] + [InlineData("a‐ha")] + [InlineData("윤현민")] + public void PartitionCreditsByPersonId_CreditWithAnItem_IsNeverBothCreatedAndDeleted(string creditName) + { + var existing = new HashSet<Guid> { PersonId(creditName) }; + + var (newNames, deadIds) = PeopleValidator.PartitionCreditsByPersonId([creditName], PersonId, existing); + + Assert.Empty(newNames); + Assert.Empty(deadIds); + } + + [Fact] + public void PartitionCreditsByPersonId_CreditWithNoItem_IsCreated() + { + var (newNames, deadIds) = PeopleValidator.PartitionCreditsByPersonId( + ["Wanted Person"], + PersonId, + new HashSet<Guid>()); + + Assert.Equal(["Wanted Person"], newNames); + Assert.Empty(deadIds); + } + + [Fact] + public void PartitionCreditsByPersonId_ItemNoCreditNames_IsDead() + { + var orphan = PersonId("Nobody Credits Me"); + var existing = new HashSet<Guid> { PersonId("Credited"), orphan }; + + var (newNames, deadIds) = PeopleValidator.PartitionCreditsByPersonId(["Credited"], PersonId, existing); + + Assert.Empty(newNames); + Assert.Equal([orphan], deadIds); + } + + [Fact] + public void PartitionCreditsByPersonId_CreditsNormalizingOntoOneId_CreateOneItem() + { + // "AURORA" and "Aurora" are one person as far as the item-by-name id is concerned, so exactly + // one of them should create the item and neither should end up dead. + var (newNames, deadIds) = PeopleValidator.PartitionCreditsByPersonId( + ["AURORA", "Aurora", "aurora"], + PersonId, + new HashSet<Guid>()); + + Assert.Single(newNames); + Assert.Empty(deadIds); + } + + [Fact] + public void PartitionCreditsByPersonId_SecondRunAfterCreating_AsksForNothingFurther() + { + // The churn showed up as a run that never settled, so drive two rounds: whatever round one + // created must leave round two with nothing to do. + string[] credits = ["AURORA", "Amir AboulEla", "Miguel Ángel Fuentes"]; + var existing = new HashSet<Guid>(); + + var (firstNames, firstDead) = PeopleValidator.PartitionCreditsByPersonId(credits, PersonId, existing); + Assert.Equal(3, firstNames.Count); + Assert.Empty(firstDead); + + foreach (var created in firstNames) + { + existing.Add(PersonId(created)); + } + + var (secondNames, secondDead) = PeopleValidator.PartitionCreditsByPersonId(credits, PersonId, existing); + + Assert.Empty(secondNames); + Assert.Empty(secondDead); + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/PlayQueueManagerTests.cs b/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/PlayQueueManagerTests.cs index 32685556b2..05e8a40de1 100644 --- a/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/PlayQueueManagerTests.cs +++ b/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/PlayQueueManagerTests.cs @@ -143,6 +143,36 @@ public class PlayQueueManagerTests } [Fact] + public void SetShuffleMode_SortedWhileAlreadySorted_KeepsPlayingItem() + { + var queue = CreateQueue(3); + queue.SetPlayingItemByIndex(1); + var expectedItemId = queue.GetPlayingItemId(); + + queue.SetShuffleMode(GroupShuffleMode.Sorted); + + Assert.Equal(GroupShuffleMode.Sorted, queue.ShuffleMode); + Assert.Equal(1, queue.PlayingItemIndex); + Assert.Equal(expectedItemId, queue.GetPlayingItemId()); + } + + [Fact] + public void SetShuffleMode_SortedTwiceAfterShuffle_KeepsPlayingItem() + { + var queue = CreateQueue(5); + queue.SetPlayingItemByIndex(2); + var expectedItemId = queue.GetPlayingItemId(); + + queue.SetShuffleMode(GroupShuffleMode.Shuffle); + queue.SetShuffleMode(GroupShuffleMode.Sorted); + queue.SetShuffleMode(GroupShuffleMode.Sorted); + + Assert.Equal(GroupShuffleMode.Sorted, queue.ShuffleMode); + Assert.Equal(5, queue.GetPlaylist().Count); + Assert.Equal(expectedItemId, queue.GetPlayingItemId()); + } + + [Fact] public void SetPlayingItemByIndex_InBounds_SetsPlayingItem() { var queue = CreateQueue(2); diff --git a/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/SyncPlayManagerTests.cs b/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/SyncPlayManagerTests.cs new file mode 100644 index 0000000000..b1221f6f71 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/SyncPlayManagerTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Threading; +using Jellyfin.Database.Implementations.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Session; +using MediaBrowser.Controller.SyncPlay.Requests; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; +using SyncPlayManager = Emby.Server.Implementations.SyncPlay.SyncPlayManager; + +namespace Jellyfin.Server.Implementations.Tests.SyncPlay; + +public class SyncPlayManagerTests +{ + [Fact] + public void LeaveGroup_AfterJoiningTheSameGroupTwice_ClearsTheActiveSessionCounter() + { + var harness = new ManagerHarness(); + + var info = harness.Manager.NewGroup(harness.Session, new NewGroupRequest("group"), CancellationToken.None); + Assert.True(harness.Manager.IsUserActive(harness.User.Id)); + + // A client that re-sends Join for the group it is already in must not be counted twice. + harness.Manager.JoinGroup(harness.Session, new JoinGroupRequest(info.GroupId), CancellationToken.None); + harness.Manager.LeaveGroup(harness.Session, new LeaveGroupRequest(), CancellationToken.None); + + Assert.False(harness.Manager.IsUserActive(harness.User.Id)); + } + + [Fact] + public void LeaveGroup_AfterASingleJoin_ClearsTheActiveSessionCounter() + { + var harness = new ManagerHarness(); + + harness.Manager.NewGroup(harness.Session, new NewGroupRequest("group"), CancellationToken.None); + harness.Manager.LeaveGroup(harness.Session, new LeaveGroupRequest(), CancellationToken.None); + + Assert.False(harness.Manager.IsUserActive(harness.User.Id)); + } + + [Fact] + public void IsUserActive_WithTwoSessionsOfTheSameUser_TracksBothSeparately() + { + var harness = new ManagerHarness(); + var second = harness.CreateSession("session-2"); + + var info = harness.Manager.NewGroup(harness.Session, new NewGroupRequest("group"), CancellationToken.None); + harness.Manager.JoinGroup(second, new JoinGroupRequest(info.GroupId), CancellationToken.None); + + harness.Manager.LeaveGroup(harness.Session, new LeaveGroupRequest(), CancellationToken.None); + Assert.True(harness.Manager.IsUserActive(harness.User.Id)); + + harness.Manager.LeaveGroup(second, new LeaveGroupRequest(), CancellationToken.None); + Assert.False(harness.Manager.IsUserActive(harness.User.Id)); + } + + private sealed class ManagerHarness + { + private readonly Mock<ISessionManager> _sessionManager = new(); + + public ManagerHarness() + { + var userManager = new Mock<IUserManager>(); + var libraryManager = new Mock<ILibraryManager>(); + + User = new User("tester", "auth-provider", "pwdreset-provider"); + userManager.Setup(m => m.GetUserById(It.IsAny<Guid>())).Returns(User); + + Manager = new SyncPlayManager( + NullLoggerFactory.Instance, + userManager.Object, + _sessionManager.Object, + libraryManager.Object); + + Session = CreateSession("session-1"); + } + + public SyncPlayManager Manager { get; } + + public User User { get; } + + public SessionInfo Session { get; } + + public SessionInfo CreateSession(string id) + { + return new SessionInfo(_sessionManager.Object, NullLogger.Instance) + { + Id = id, + UserId = User.Id, + UserName = User.Username + }; + } + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/WaitingGroupStateTests.cs b/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/WaitingGroupStateTests.cs new file mode 100644 index 0000000000..0cccd5d4ca --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/SyncPlay/WaitingGroupStateTests.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Database.Implementations.Entities; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Session; +using MediaBrowser.Controller.SyncPlay.GroupStates; +using MediaBrowser.Controller.SyncPlay.PlaybackRequests; +using MediaBrowser.Controller.SyncPlay.Requests; +using MediaBrowser.Model.SyncPlay; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; +using SyncPlayGroup = Emby.Server.Implementations.SyncPlay.Group; + +namespace Jellyfin.Server.Implementations.Tests.SyncPlay; + +public class WaitingGroupStateTests +{ + [Fact] + public void Ready_ClientResumedWithLowPing_AppliesTheDefaultPingFloorInMilliseconds() + { + var harness = new GroupHarness(); + var group = harness.Group; + + // Both members report a ping well under the default, so the floor is what decides the delay. + group.UpdatePing(harness.First, 10); + group.UpdatePing(harness.Second, 10); + + group.PositionTicks = TimeSpan.FromMinutes(5).Ticks; + group.LastActivity = DateTime.UtcNow; + group.SetBuffering(harness.First, true); + group.SetBuffering(harness.Second, false); + + var state = new WaitingGroupState(NullLoggerFactory.Instance) { ResumePlaying = true }; + + var before = DateTime.UtcNow; + state.HandleRequest( + new ReadyGroupRequest(DateTime.UtcNow, group.PositionTicks, true, harness.PlaylistItemId), + group, + GroupStateType.Waiting, + harness.First, + CancellationToken.None); + + // DefaultPing is expressed in milliseconds, so the floor must be converted before being + // compared against a tick count. Without the conversion the floor is 500 ticks (0.05 ms) + // and never applies. + var scheduledDelay = group.LastActivity - before; + Assert.True( + scheduledDelay >= TimeSpan.FromMilliseconds(group.DefaultPing), + $"expected a resume delay of at least {group.DefaultPing} ms, got {scheduledDelay.TotalMilliseconds} ms"); + } + + [Theory] + [InlineData(4_000_000_000L)] + [InlineData(1_000_000_000_000_000L)] + [InlineData(long.MaxValue)] + [InlineData(-1L)] + public void UpdatePing_ClientReportsAnUnusablePing_IsClampedAndCannotStallTheGroup(long reportedPing) + { + var harness = new GroupHarness(); + var group = harness.Group; + + group.UpdatePing(harness.First, reportedPing); + + Assert.InRange(group.GetHighestPing(), 0, group.MaxPing); + + // The reported ping is scaled into the group's resume point, so an unclamped value either + // pushes playback months out or overflows the arithmetic outright. + var state = new PlayingGroupState(NullLoggerFactory.Instance); + var before = DateTime.UtcNow; + state.HandleRequest( + new UnpauseGroupRequest(), + group, + GroupStateType.Paused, + harness.First, + CancellationToken.None); + + Assert.InRange(group.LastActivity - before, TimeSpan.Zero, TimeSpan.FromMinutes(1)); + } + + private sealed class GroupHarness + { + public GroupHarness() + { + var userManager = new Mock<IUserManager>(); + var sessionManager = new Mock<ISessionManager>(); + var libraryManager = new Mock<ILibraryManager>(); + + var user = new User("tester", "auth-provider", "pwdreset-provider"); + userManager.Setup(m => m.GetUserById(It.IsAny<Guid>())).Returns(user); + + var item = new Mock<BaseItem>(); + item.Setup(i => i.IsVisibleStandalone(It.IsAny<User>())).Returns(true); + item.Object.RunTimeTicks = TimeSpan.FromHours(2).Ticks; + libraryManager.Setup(m => m.GetItemById(It.IsAny<Guid>())).Returns(item.Object); + + sessionManager + .Setup(m => m.SendSyncPlayCommand(It.IsAny<string>(), It.IsAny<SendCommand>(), It.IsAny<CancellationToken>())) + .Returns(Task.CompletedTask); + + sessionManager + .Setup(m => m.SendSyncPlayGroupUpdate(It.IsAny<string>(), It.IsAny<GroupUpdate<GroupStateUpdate>>(), It.IsAny<CancellationToken>())) + .Returns(Task.CompletedTask); + + Group = new SyncPlayGroup( + NullLoggerFactory.Instance, + userManager.Object, + sessionManager.Object, + libraryManager.Object); + + First = new SessionInfo(sessionManager.Object, NullLogger.Instance) + { + Id = "first", + UserId = user.Id, + UserName = "first" + }; + Second = new SessionInfo(sessionManager.Object, NullLogger.Instance) + { + Id = "second", + UserId = user.Id, + UserName = "second" + }; + + Group.CreateGroup(First, new NewGroupRequest("group"), CancellationToken.None); + Group.SessionJoin(Second, new JoinGroupRequest(Group.GroupId), CancellationToken.None); + Group.SetPlayQueue(new List<Guid> { Guid.NewGuid() }, 0, 0); + PlaylistItemId = Group.PlayQueue.GetPlayingItemPlaylistId(); + } + + public SyncPlayGroup Group { get; } + + public SessionInfo First { get; } + + public SessionInfo Second { get; } + + public Guid PlaylistItemId { get; } + } +} diff --git a/tests/Jellyfin.Server.Integration.Tests/Controllers/LibraryStructureControllerTests.cs b/tests/Jellyfin.Server.Integration.Tests/Controllers/LibraryStructureControllerTests.cs index 0a5838c545..79b9d1e2c5 100644 --- a/tests/Jellyfin.Server.Integration.Tests/Controllers/LibraryStructureControllerTests.cs +++ b/tests/Jellyfin.Server.Integration.Tests/Controllers/LibraryStructureControllerTests.cs @@ -6,8 +6,11 @@ using System.Text.Json; using System.Threading.Tasks; using Jellyfin.Api.Models.LibraryStructureDto; using Jellyfin.Extensions.Json; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Entities; +using Microsoft.Extensions.DependencyInjection; using Xunit; using Xunit.v3.Priority; @@ -26,6 +29,45 @@ public sealed class LibraryStructureControllerTests : IClassFixture<JellyfinAppl } [Fact] + [Priority(-3)] + public async Task AddVirtualFolder_WithWarmDirectoryServiceCache_InvalidatesTheParentListing() + { + const string Name = "stale-cache-test"; + + var client = _factory.CreateClient(); + client.DefaultRequestHeaders.AddAuthHeader(_accessToken ??= await AuthHelper.CompleteStartupAsync(client)); + + var directoryService = _factory.Services.GetRequiredService<IDirectoryService>(); + var rootFolderPath = _factory.Services.GetRequiredService<IServerApplicationPaths>().DefaultUserViewsPath; + + // Cache a listing of the libraries root taken before the new folder exists. Everything + // resolving through this DirectoryService keeps reading that listing until it is dropped, + // so the library stays invisible. Making the caches shared once turned this into a real + // test failure, see UpdateLibraryOptions_Valid_Success. + Assert.DoesNotContain( + directoryService.GetFileSystemEntries(rootFolderPath), + x => string.Equals(x.Name, Name, StringComparison.Ordinal)); + + var body = new AddVirtualFolderDto() + { + LibraryOptions = new LibraryOptions() + { + Enabled = false + } + }; + + using var response = await client.PostAsJsonAsync($"Library/VirtualFolders?name={Name}&refreshLibrary=false", body, _jsonOptions, TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.NoContent, response.StatusCode); + + Assert.Contains( + directoryService.GetFileSystemEntries(rootFolderPath), + x => string.Equals(x.Name, Name, StringComparison.Ordinal)); + + using var cleanup = await client.DeleteAsync($"Library/VirtualFolders?name={Name}&refreshLibrary=false", TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.NoContent, cleanup.StatusCode); + } + + [Fact] [Priority(-1)] public async Task Post_NewVirtualFolder_NotFound() { diff --git a/tests/Jellyfin.Server.Tests/Migrations/CodeMigrationTests.cs b/tests/Jellyfin.Server.Tests/Migrations/CodeMigrationTests.cs new file mode 100644 index 0000000000..3bd8581a5f --- /dev/null +++ b/tests/Jellyfin.Server.Tests/Migrations/CodeMigrationTests.cs @@ -0,0 +1,112 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Server.Migrations; +using Jellyfin.Server.Migrations.Stages; +using Jellyfin.Server.ServerSetupApp; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace Jellyfin.Server.Tests.Migrations; + +public class CodeMigrationTests +{ + [Fact] + public async Task Perform_LeavesApplicationSingletonsAlive() + { + var services = new ServiceCollection() + .AddLogging() + .RegisterStartupLogger() + .AddSingleton<ApplicationSingleton>() + .AddTransient<MigrationTransient>(); + + await using var serviceProvider = services.BuildServiceProvider(); + var applicationSingleton = serviceProvider.GetRequiredService<ApplicationSingleton>(); + var logger = new StartupLogger(NullLogger.Instance).BeginGroup($"Test migration"); + + var migration = new CodeMigration( + typeof(TestMigration), + new JellyfinMigrationAttribute("2026-09-05T10:00:00", nameof(TestMigration)), + null); + await migration.Perform(serviceProvider, logger, CancellationToken.None); + + var performed = TestMigration.Performed; + Assert.NotNull(performed); + // The migration has to run against the applications own services, and they have to outlive it. + Assert.Same(applicationSingleton, performed.Singleton); + Assert.False(applicationSingleton.IsDisposed); + Assert.Same(applicationSingleton, serviceProvider.GetRequiredService<ApplicationSingleton>()); + // Services created for the migration itself are still owned by the migration. + Assert.True(performed.Transient.IsDisposed); + // The startup logger has to stay attached to the topic of the running migration. + Assert.Same(logger.Topic, performed.Logger.Topic); + } + + [Fact] + public async Task Perform_DoesNotLeakTheMigrationTopic() + { + var services = new ServiceCollection() + .AddLogging() + .RegisterStartupLogger() + .AddSingleton<ApplicationSingleton>() + .AddTransient<MigrationTransient>(); + + await using var serviceProvider = services.BuildServiceProvider(); + var logger = new StartupLogger(NullLogger.Instance).BeginGroup($"Test migration"); + + var migration = new CodeMigration( + typeof(TestMigration), + new JellyfinMigrationAttribute("2026-09-05T10:00:00", nameof(TestMigration)), + null); + await migration.Perform(serviceProvider, logger, CancellationToken.None); + + // The topic belongs to the migration that ran, so loggers resolved afterwards must not still write into it. + Assert.Null(serviceProvider.GetRequiredService<IStartupLogger<CodeMigrationTests>>().Topic); + Assert.Null(new StartupLogger(NullLogger.Instance).Topic); + } + + private sealed class ApplicationSingleton : IDisposable + { + public bool IsDisposed { get; private set; } + + public void Dispose() + { + IsDisposed = true; + } + } + + private sealed class MigrationTransient : IDisposable + { + public bool IsDisposed { get; private set; } + + public void Dispose() + { + IsDisposed = true; + } + } + + private sealed class TestMigration : IAsyncMigrationRoutine + { + public TestMigration(ApplicationSingleton singleton, MigrationTransient transient, IStartupLogger<TestMigration> logger) + { + Singleton = singleton; + Transient = transient; + Logger = logger; + } + + public static TestMigration? Performed { get; private set; } + + public ApplicationSingleton Singleton { get; } + + public MigrationTransient Transient { get; } + + public IStartupLogger<TestMigration> Logger { get; } + + public Task PerformAsync(CancellationToken cancellationToken) + { + Performed = this; + return Task.CompletedTask; + } + } +} diff --git a/tests/Jellyfin.Server.Tests/ServerSetupApp/StartupLoggerTests.cs b/tests/Jellyfin.Server.Tests/ServerSetupApp/StartupLoggerTests.cs new file mode 100644 index 0000000000..c2894e9647 --- /dev/null +++ b/tests/Jellyfin.Server.Tests/ServerSetupApp/StartupLoggerTests.cs @@ -0,0 +1,54 @@ +using Jellyfin.Server.ServerSetupApp; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace Jellyfin.Server.Tests.ServerSetupApp; + +public class StartupLoggerTests +{ + [Fact] + public void BeginAmbientTopic_AttachesNewLoggersToTheTopic() + { + var migration = new StartupLogger(NullLogger.Instance).BeginGroup($"Migration"); + + using (StartupLogger.BeginAmbientTopic(migration.Topic)) + { + Assert.Same(migration.Topic, new StartupLogger(NullLogger.Instance).Topic); + } + } + + [Fact] + public void BeginAmbientTopic_RestoresThePreviousTopic() + { + var root = new StartupLogger(NullLogger.Instance); + var outer = root.BeginGroup($"Outer"); + var inner = outer.BeginGroup($"Inner"); + + Assert.Null(new StartupLogger(NullLogger.Instance).Topic); + + using (StartupLogger.BeginAmbientTopic(outer.Topic)) + { + using (StartupLogger.BeginAmbientTopic(inner.Topic)) + { + Assert.Same(inner.Topic, new StartupLogger(NullLogger.Instance).Topic); + } + + // Leaving a nested topic has to fall back to the enclosing one, not to the setup UI root. + Assert.Same(outer.Topic, new StartupLogger(NullLogger.Instance).Topic); + } + + Assert.Null(new StartupLogger(NullLogger.Instance).Topic); + } + + [Fact] + public void BeginGroup_KeepsAnExplicitTopicOverTheAmbientOne() + { + var migration = new StartupLogger(NullLogger.Instance).BeginGroup($"Migration"); + var unrelated = new StartupLogger(NullLogger.Instance).BeginGroup($"Unrelated"); + + using (StartupLogger.BeginAmbientTopic(migration.Topic)) + { + Assert.Same(unrelated.Topic, unrelated.With(NullLogger.Instance).Topic); + } + } +} diff --git a/tests/Jellyfin.XbmcMetadata.Tests/Parsers/EpisodeNfoProviderTests.cs b/tests/Jellyfin.XbmcMetadata.Tests/Parsers/EpisodeNfoProviderTests.cs index a04b37f215..3767b5c954 100644 --- a/tests/Jellyfin.XbmcMetadata.Tests/Parsers/EpisodeNfoProviderTests.cs +++ b/tests/Jellyfin.XbmcMetadata.Tests/Parsers/EpisodeNfoProviderTests.cs @@ -124,6 +124,27 @@ namespace Jellyfin.XbmcMetadata.Tests.Parsers } [Fact] + public void Fetch_Valid_MultiEpisode_Unordered_Success() + { + var result = new MetadataResult<Episode>() + { + Item = new Episode() + }; + + _parser.Fetch(result, "Test Data/Rising-Reversed.nfo", CancellationToken.None); + + var item = result.Item; + // The episodedetails blocks are stored in descending order, the merged episode must still be in ascending order + Assert.Equal("Rising (1) / Rising (2)", item.Name); + Assert.Equal(1, item.IndexNumber); + Assert.Equal(2, item.IndexNumberEnd); + Assert.Equal(1, item.ParentIndexNumber); + Assert.Equal("A new Stargate team embarks on a dangerous mission to a distant galaxy, where they discover a mythical lost city -- and a deadly new enemy. / Sheppard tries to convince Weir to mount a rescue mission to free Colonel Sumner, Teyla, and the others captured by the Wraith.", item.Overview); + Assert.Equal(new DateTime(2004, 7, 16), item.PremiereDate); + Assert.Equal(2004, item.ProductionYear); + } + + [Fact] public void Fetch_Valid_MultiEpisode_With_Missing_Tags_Success() { var result = new MetadataResult<Episode>() diff --git a/tests/Jellyfin.XbmcMetadata.Tests/Test Data/Rising-Reversed.nfo b/tests/Jellyfin.XbmcMetadata.Tests/Test Data/Rising-Reversed.nfo new file mode 100644 index 0000000000..6dbab13566 --- /dev/null +++ b/tests/Jellyfin.XbmcMetadata.Tests/Test Data/Rising-Reversed.nfo @@ -0,0 +1,43 @@ +<episodedetails> + <title>Rising (2)</title> + <season>1</season> + <episode>2</episode> + <aired>2004-07-16</aired> + <plot>Sheppard tries to convince Weir to mount a rescue mission to free Colonel Sumner, Teyla, and the others captured by the Wraith.</plot> + <thumb>https://artworks.thetvdb.com/banners/episodes/70851/25334.jpg</thumb> + <watched>false</watched> + <rating>7.9</rating> + <actor> + <name>Joe Flanigan</name> + <role>John Sheppard</role> + <order>0</order> + <thumb>https://image.tmdb.org/t/p/w300_and_h450_bestv2/5AA1ORKIsnMakT6fCVy3JKlzMs6.jpg</thumb> + </actor> + <actor> + <name>David Hewlett</name> + <role>Rodney McKay</role> + <order>1</order> + <thumb>https://image.tmdb.org/t/p/w300_and_h450_bestv2/hUcYyssAPCqnZ4GjolhOWXHTWSa.jpg</thumb> + </actor> +</episodedetails><episodedetails> + <title>Rising (1)</title> + <season>1</season> + <episode>1</episode> + <aired>2004-07-16</aired> + <plot>A new Stargate team embarks on a dangerous mission to a distant galaxy, where they discover a mythical lost city -- and a deadly new enemy.</plot> + <thumb>https://artworks.thetvdb.com/banners/episodes/70851/25333.jpg</thumb> + <watched>false</watched> + <rating>8.0</rating> + <actor> + <name>Joe Flanigan</name> + <role>John Sheppard</role> + <order>0</order> + <thumb>https://image.tmdb.org/t/p/w300_and_h450_bestv2/5AA1ORKIsnMakT6fCVy3JKlzMs6.jpg</thumb> + </actor> + <actor> + <name>David Hewlett</name> + <role>Rodney McKay</role> + <order>1</order> + <thumb>https://image.tmdb.org/t/p/w300_and_h450_bestv2/hUcYyssAPCqnZ4GjolhOWXHTWSa.jpg</thumb> + </actor> +</episodedetails> |