diff options
22 files changed, 289 insertions, 269 deletions
diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index 5498f5a10..d74ea0352 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -183,6 +183,8 @@ namespace Emby.Server.Implementations private IPlugin[] _plugins; + private IReadOnlyList<LocalPlugin> _pluginsManifests; + /// <summary> /// Gets the plugins. /// </summary> @@ -531,8 +533,6 @@ namespace Emby.Server.Implementations ServiceCollection.AddSingleton(NetManager); - ServiceCollection.AddSingleton<IIsoManager, IsoManager>(); - ServiceCollection.AddSingleton<ITaskManager, TaskManager>(); ServiceCollection.AddSingleton(_xmlSerializer); @@ -774,17 +774,27 @@ namespace Emby.Server.Implementations if (Plugins != null) { - var pluginBuilder = new StringBuilder(); - foreach (var plugin in Plugins) { - pluginBuilder.Append(plugin.Name) - .Append(' ') - .Append(plugin.Version) - .AppendLine(); - } + if (_pluginsManifests != null && plugin is IPluginAssembly assemblyPlugin) + { + // Ensure the version number matches the Plugin Manifest information. + foreach (var item in _pluginsManifests) + { + if (Path.GetDirectoryName(plugin.AssemblyFilePath).Equals(item.Path, StringComparison.OrdinalIgnoreCase)) + { + // Update version number to that of the manifest. + assemblyPlugin.SetAttributes( + plugin.AssemblyFilePath, + Path.Combine(ApplicationPaths.PluginsPath, Path.GetFileNameWithoutExtension(plugin.AssemblyFilePath)), + item.Version); + break; + } + } + } - Logger.LogInformation("Plugins: {Plugins}", pluginBuilder.ToString()); + Logger.LogInformation("Loaded plugin: {PluginName} {PluginVersion}", plugin.Name, plugin.Version); + } } _urlPrefixes = GetUrlPrefixes().ToArray(); @@ -812,8 +822,6 @@ namespace Emby.Server.Implementations Resolve<IMediaSourceManager>().AddParts(GetExports<IMediaSourceProvider>()); Resolve<INotificationManager>().AddParts(GetExports<INotificationService>(), GetExports<INotificationTypeFactory>()); - - Resolve<IIsoManager>().AddParts(GetExports<IIsoMounter>()); } /// <summary> @@ -1100,7 +1108,8 @@ namespace Emby.Server.Implementations { if (Directory.Exists(ApplicationPaths.PluginsPath)) { - foreach (var plugin in GetLocalPlugins(ApplicationPaths.PluginsPath)) + _pluginsManifests = GetLocalPlugins(ApplicationPaths.PluginsPath).ToList(); + foreach (var plugin in _pluginsManifests) { foreach (var file in plugin.DllFiles) { diff --git a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs index ff64e217a..ae1b51b4c 100644 --- a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs +++ b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs @@ -1,6 +1,7 @@ #pragma warning disable CS1591 using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -44,7 +45,7 @@ namespace Emby.Server.Implementations.EntryPoints private readonly List<BaseItem> _itemsAdded = new List<BaseItem>(); private readonly List<BaseItem> _itemsRemoved = new List<BaseItem>(); private readonly List<BaseItem> _itemsUpdated = new List<BaseItem>(); - private readonly Dictionary<Guid, DateTime> _lastProgressMessageTimes = new Dictionary<Guid, DateTime>(); + private readonly ConcurrentDictionary<Guid, DateTime> _lastProgressMessageTimes = new ConcurrentDictionary<Guid, DateTime>(); public LibraryChangedNotifier( ILibraryManager libraryManager, @@ -98,7 +99,7 @@ namespace Emby.Server.Implementations.EntryPoints } } - _lastProgressMessageTimes[item.Id] = DateTime.UtcNow; + _lastProgressMessageTimes.AddOrUpdate(item.Id, key => DateTime.UtcNow, (key, existing) => DateTime.UtcNow); var dict = new Dictionary<string, string>(); dict["ItemId"] = item.Id.ToString("N", CultureInfo.InvariantCulture); @@ -140,6 +141,8 @@ namespace Emby.Server.Implementations.EntryPoints private void OnProviderRefreshCompleted(object sender, GenericEventArgs<BaseItem> e) { OnProviderRefreshProgress(sender, new GenericEventArgs<Tuple<BaseItem, double>>(new Tuple<BaseItem, double>(e.Argument, 100))); + + _lastProgressMessageTimes.TryRemove(e.Argument.Id, out DateTime removed); } private static bool EnableRefreshMessage(BaseItem item) diff --git a/Emby.Server.Implementations/IO/IsoManager.cs b/Emby.Server.Implementations/IO/IsoManager.cs deleted file mode 100644 index 94e92c2a6..000000000 --- a/Emby.Server.Implementations/IO/IsoManager.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using MediaBrowser.Model.IO; - -namespace Emby.Server.Implementations.IO -{ - /// <summary> - /// Class IsoManager. - /// </summary> - public class IsoManager : IIsoManager - { - /// <summary> - /// The _mounters. - /// </summary> - private readonly List<IIsoMounter> _mounters = new List<IIsoMounter>(); - - /// <summary> - /// Mounts the specified iso path. - /// </summary> - /// <param name="isoPath">The iso path.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns><see creaf="IsoMount" />.</returns> - public Task<IIsoMount> Mount(string isoPath, CancellationToken cancellationToken) - { - if (string.IsNullOrEmpty(isoPath)) - { - throw new ArgumentNullException(nameof(isoPath)); - } - - var mounter = _mounters.FirstOrDefault(i => i.CanMount(isoPath)); - - if (mounter == null) - { - throw new ArgumentException( - string.Format( - CultureInfo.InvariantCulture, - "No mounters are able to mount {0}", - isoPath)); - } - - return mounter.Mount(isoPath, cancellationToken); - } - - /// <summary> - /// Determines whether this instance can mount the specified path. - /// </summary> - /// <param name="path">The path.</param> - /// <returns><c>true</c> if this instance can mount the specified path; otherwise, <c>false</c>.</returns> - public bool CanMount(string path) - { - return _mounters.Any(i => i.CanMount(path)); - } - - /// <summary> - /// Adds the parts. - /// </summary> - /// <param name="mounters">The mounters.</param> - public void AddParts(IEnumerable<IIsoMounter> mounters) - { - _mounters.AddRange(mounters); - } - } -} diff --git a/Emby.Server.Implementations/Localization/Core/es-MX.json b/Emby.Server.Implementations/Localization/Core/es-MX.json index ab54c0ea6..05181116d 100644 --- a/Emby.Server.Implementations/Localization/Core/es-MX.json +++ b/Emby.Server.Implementations/Localization/Core/es-MX.json @@ -113,5 +113,9 @@ "TasksChannelsCategory": "Canales de Internet", "TasksApplicationCategory": "Aplicación", "TasksLibraryCategory": "Biblioteca", - "TasksMaintenanceCategory": "Mantenimiento" + "TasksMaintenanceCategory": "Mantenimiento", + "TaskCleanActivityLogDescription": "Elimina entradas del registro de actividad que sean más antiguas al periodo establecido.", + "TaskCleanActivityLog": "Limpiar registro de actividades", + "Undefined": "Sin definir", + "Forced": "Forzado" } diff --git a/Emby.Server.Implementations/Localization/Core/fi.json b/Emby.Server.Implementations/Localization/Core/fi.json index 61bef29ed..954759b5c 100644 --- a/Emby.Server.Implementations/Localization/Core/fi.json +++ b/Emby.Server.Implementations/Localization/Core/fi.json @@ -114,5 +114,8 @@ "TasksApplicationCategory": "Sovellus", "TasksLibraryCategory": "Kirjasto", "Forced": "Pakotettu", - "Default": "Oletus" + "Default": "Oletus", + "TaskCleanActivityLogDescription": "Poistaa määritettyä vanhemmat tapahtumat aktiviteettilokista.", + "TaskCleanActivityLog": "Tyhjennä aktiviteettiloki", + "Undefined": "Määrittelemätön" } diff --git a/Emby.Server.Implementations/Localization/Core/fr-CA.json b/Emby.Server.Implementations/Localization/Core/fr-CA.json index 3d7592e3c..5aa65a525 100644 --- a/Emby.Server.Implementations/Localization/Core/fr-CA.json +++ b/Emby.Server.Implementations/Localization/Core/fr-CA.json @@ -113,5 +113,6 @@ "TaskCleanCache": "Nettoyer le répertoire des fichiers temporaires", "TasksApplicationCategory": "Application", "TaskCleanCacheDescription": "Supprime les fichiers temporaires qui ne sont plus nécessaire pour le système.", - "TasksChannelsCategory": "Canaux Internet" + "TasksChannelsCategory": "Canaux Internet", + "Default": "Par défaut" } diff --git a/Jellyfin.Api/Helpers/TranscodingJobHelper.cs b/Jellyfin.Api/Helpers/TranscodingJobHelper.cs index bb2265dba..240d132b1 100644 --- a/Jellyfin.Api/Helpers/TranscodingJobHelper.cs +++ b/Jellyfin.Api/Helpers/TranscodingJobHelper.cs @@ -12,7 +12,6 @@ using Jellyfin.Api.Models.PlaybackDtos; using Jellyfin.Api.Models.StreamingDtos; using Jellyfin.Data.Enums; using MediaBrowser.Common.Configuration; -using MediaBrowser.Common.Extensions; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; @@ -46,7 +45,6 @@ namespace Jellyfin.Api.Helpers private readonly IAuthorizationContext _authorizationContext; private readonly EncodingHelper _encodingHelper; private readonly IFileSystem _fileSystem; - private readonly IIsoManager _isoManager; private readonly ILogger<TranscodingJobHelper> _logger; private readonly IMediaEncoder _mediaEncoder; private readonly IMediaSourceManager _mediaSourceManager; @@ -64,7 +62,6 @@ namespace Jellyfin.Api.Helpers /// <param name="serverConfigurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> /// <param name="sessionManager">Instance of the <see cref="ISessionManager"/> interface.</param> /// <param name="authorizationContext">Instance of the <see cref="IAuthorizationContext"/> interface.</param> - /// <param name="isoManager">Instance of the <see cref="IIsoManager"/> interface.</param> /// <param name="subtitleEncoder">Instance of the <see cref="ISubtitleEncoder"/> interface.</param> /// <param name="configuration">Instance of the <see cref="IConfiguration"/> interface.</param> /// <param name="loggerFactory">Instance of the <see cref="ILoggerFactory"/> interface.</param> @@ -76,7 +73,6 @@ namespace Jellyfin.Api.Helpers IServerConfigurationManager serverConfigurationManager, ISessionManager sessionManager, IAuthorizationContext authorizationContext, - IIsoManager isoManager, ISubtitleEncoder subtitleEncoder, IConfiguration configuration, ILoggerFactory loggerFactory) @@ -88,7 +84,6 @@ namespace Jellyfin.Api.Helpers _serverConfigurationManager = serverConfigurationManager; _sessionManager = sessionManager; _authorizationContext = authorizationContext; - _isoManager = isoManager; _loggerFactory = loggerFactory; _encodingHelper = new EncodingHelper(mediaEncoder, fileSystem, subtitleEncoder, configuration); diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs index 42b87ec5f..9137ea234 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs @@ -35,8 +35,14 @@ namespace Jellyfin.Server.Migrations.Routines _logger.LogInformation("Removing 'RemoveDownloadImagesInAdvance' settings in all the libraries"); foreach (var virtualFolder in virtualFolders) { + // Some virtual folders don't have a proper item id. + if (!Guid.TryParse(virtualFolder.ItemId, out var folderId)) + { + continue; + } + var libraryOptions = virtualFolder.LibraryOptions; - var collectionFolder = (CollectionFolder)_libraryManager.GetItemById(virtualFolder.ItemId); + var collectionFolder = (CollectionFolder)_libraryManager.GetItemById(folderId); // The property no longer exists in LibraryOptions, so we just re-save the options to get old data removed. collectionFolder.UpdateLibraryOptions(libraryOptions); _logger.LogInformation("Removed from '{VirtualFolder}'", virtualFolder.Name); diff --git a/Jellyfin.Server/Startup.cs b/Jellyfin.Server/Startup.cs index 7f1d332ee..aa3ef5350 100644 --- a/Jellyfin.Server/Startup.cs +++ b/Jellyfin.Server/Startup.cs @@ -133,8 +133,9 @@ namespace Jellyfin.Server { var extensionProvider = new FileExtensionContentTypeProvider(); - // subtitles octopus requires .data files. + // subtitles octopus requires .data, .mem files. extensionProvider.Mappings.Add(".data", MediaTypeNames.Application.Octet); + extensionProvider.Mappings.Add(".mem", MediaTypeNames.Application.Octet); mainApp.UseStaticFiles(new StaticFileOptions { FileProvider = new PhysicalFileProvider(_serverConfigurationManager.ApplicationPaths.WebPath), diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs index 67aa7f338..085f769d0 100644 --- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs @@ -1,5 +1,7 @@ -using System; +using System; using System.Linq; +using System.Threading; +using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Channels; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -12,6 +14,8 @@ namespace MediaBrowser.Controller.BaseItemManager { private readonly IServerConfigurationManager _serverConfigurationManager; + private int _metadataRefreshConcurrency = 0; + /// <summary> /// Initializes a new instance of the <see cref="BaseItemManager"/> class. /// </summary> @@ -19,9 +23,17 @@ namespace MediaBrowser.Controller.BaseItemManager public BaseItemManager(IServerConfigurationManager serverConfigurationManager) { _serverConfigurationManager = serverConfigurationManager; + + _metadataRefreshConcurrency = GetMetadataRefreshConcurrency(); + SetupMetadataThrottler(); + + _serverConfigurationManager.ConfigurationUpdated += OnConfigurationUpdated; } /// <inheritdoc /> + public SemaphoreSlim MetadataRefreshThrottler { get; private set; } + + /// <inheritdoc /> public bool IsMetadataFetcherEnabled(BaseItem baseItem, LibraryOptions libraryOptions, string name) { if (baseItem is Channel) @@ -82,5 +94,42 @@ namespace MediaBrowser.Controller.BaseItemManager return itemConfig == null || !itemConfig.DisabledImageFetchers.Contains(name, StringComparer.OrdinalIgnoreCase); } + + /// <summary> + /// Called when the configuration is updated. + /// It will refresh the metadata throttler if the relevant config changed. + /// </summary> + private void OnConfigurationUpdated(object sender, EventArgs e) + { + int newMetadataRefreshConcurrency = GetMetadataRefreshConcurrency(); + if (_metadataRefreshConcurrency != newMetadataRefreshConcurrency) + { + _metadataRefreshConcurrency = newMetadataRefreshConcurrency; + SetupMetadataThrottler(); + } + } + + /// <summary> + /// Creates the metadata refresh throttler. + /// </summary> + private void SetupMetadataThrottler() + { + MetadataRefreshThrottler = new SemaphoreSlim(_metadataRefreshConcurrency); + } + + /// <summary> + /// Returns the metadata refresh concurrency. + /// </summary> + private int GetMetadataRefreshConcurrency() + { + var concurrency = _serverConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency; + + if (concurrency <= 0) + { + concurrency = Environment.ProcessorCount; + } + + return concurrency; + } } } diff --git a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs index ee4d3dcdc..e1f5d05a6 100644 --- a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs +++ b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Controller.Entities; +using System; +using System.Threading; +using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Configuration; namespace MediaBrowser.Controller.BaseItemManager @@ -9,6 +11,11 @@ namespace MediaBrowser.Controller.BaseItemManager public interface IBaseItemManager { /// <summary> + /// Gets the semaphore used to limit the amount of concurrent metadata refreshes. + /// </summary> + SemaphoreSlim MetadataRefreshThrottler { get; } + + /// <summary> /// Is metadata fetcher enabled. /// </summary> /// <param name="baseItem">The base item.</param> diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 675cdbd96..23f4c00c1 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; using Jellyfin.Data.Entities; using Jellyfin.Data.Enums; using MediaBrowser.Common.Progress; @@ -328,11 +329,11 @@ namespace MediaBrowser.Controller.Entities return; } - progress.Report(5); + progress.Report(ProgressHelpers.RetrievedChildren); if (recursive) { - ProviderManager.OnRefreshProgress(this, 5); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.RetrievedChildren); } // Build a dictionary of the current children we have now by Id so we can compare quickly and easily @@ -388,11 +389,11 @@ namespace MediaBrowser.Controller.Entities validChildrenNeedGeneration = true; } - progress.Report(10); + progress.Report(ProgressHelpers.UpdatedChildItems); if (recursive) { - ProviderManager.OnRefreshProgress(this, 10); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.UpdatedChildItems); } cancellationToken.ThrowIfCancellationRequested(); @@ -402,11 +403,13 @@ namespace MediaBrowser.Controller.Entities var innerProgress = new ActionableProgress<double>(); var folder = this; - innerProgress.RegisterAction(p => + innerProgress.RegisterAction(innerPercent => { - double newPct = 0.80 * p + 10; - progress.Report(newPct); - ProviderManager.OnRefreshProgress(folder, newPct); + var percent = ProgressHelpers.GetProgress(ProgressHelpers.UpdatedChildItems, ProgressHelpers.ScannedSubfolders, innerPercent); + + progress.Report(percent); + + ProviderManager.OnRefreshProgress(folder, percent); }); if (validChildrenNeedGeneration) @@ -420,11 +423,11 @@ namespace MediaBrowser.Controller.Entities if (refreshChildMetadata) { - progress.Report(90); + progress.Report(ProgressHelpers.ScannedSubfolders); if (recursive) { - ProviderManager.OnRefreshProgress(this, 90); + ProviderManager.OnRefreshProgress(this, ProgressHelpers.ScannedSubfolders); } var container = this as IMetadataContainer; @@ -432,13 +435,15 @@ namespace MediaBrowser.Controller.Entities var innerProgress = new ActionableProgress<double>(); var folder = this; - innerProgress.RegisterAction(p => + innerProgress.RegisterAction(innerPercent => { - double newPct = 0.10 * p + 90; - progress.Report(newPct); + var percent = ProgressHelpers.GetProgress(ProgressHelpers.ScannedSubfolders, ProgressHelpers.RefreshedMetadata, innerPercent); + + progress.Report(percent); + if (recursive) { - ProviderManager.OnRefreshProgress(folder, newPct); + ProviderManager.OnRefreshProgress(folder, percent); } }); @@ -453,55 +458,35 @@ namespace MediaBrowser.Controller.Entities validChildren = Children.ToList(); } - await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken); + await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken).ConfigureAwait(false); } } } - private async Task RefreshMetadataRecursive(List<BaseItem> children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken) + private Task RefreshMetadataRecursive(IList<BaseItem> children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken) { - var numComplete = 0; - var count = children.Count; - double currentPercent = 0; - - foreach (var child in children) - { - cancellationToken.ThrowIfCancellationRequested(); - - var innerProgress = new ActionableProgress<double>(); - - // Avoid implicitly captured closure - var currentInnerPercent = currentPercent; - - innerProgress.RegisterAction(p => - { - double innerPercent = currentInnerPercent; - innerPercent += p / count; - progress.Report(innerPercent); - }); - - await RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken) - .ConfigureAwait(false); - - numComplete++; - double percent = numComplete; - percent /= count; - percent *= 100; - currentPercent = percent; - - progress.Report(percent); - } + return RunTasks( + (baseItem, innerProgress) => RefreshChildMetadata(baseItem, refreshOptions, recursive && baseItem.IsFolder, innerProgress, cancellationToken), + children, + progress, + cancellationToken); } private async Task RefreshAllMetadataForContainer(IMetadataContainer container, MetadataRefreshOptions refreshOptions, IProgress<double> progress, CancellationToken cancellationToken) { - var series = container as Series; - if (series != null) - { - await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); - } + // limit the amount of concurrent metadata refreshes + await ProviderManager.RunMetadataRefresh( + async () => + { + var series = container as Series; + if (series != null) + { + await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); + } - await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false); + await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false); + }, + cancellationToken).ConfigureAwait(false); } private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken) @@ -516,12 +501,15 @@ namespace MediaBrowser.Controller.Entities { if (refreshOptions.RefreshItem(child)) { - await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false); + // limit the amount of concurrent metadata refreshes + await ProviderManager.RunMetadataRefresh( + async () => await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false), + cancellationToken).ConfigureAwait(false); } if (recursive && child is Folder folder) { - await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken); + await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken).ConfigureAwait(false); } } } @@ -534,39 +522,72 @@ namespace MediaBrowser.Controller.Entities /// <param name="progress">The progress.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>Task.</returns> - private async Task ValidateSubFolders(IList<Folder> children, IDirectoryService directoryService, IProgress<double> progress, CancellationToken cancellationToken) + private Task ValidateSubFolders(IList<Folder> children, IDirectoryService directoryService, IProgress<double> progress, CancellationToken cancellationToken) { - var numComplete = 0; - var count = children.Count; - double currentPercent = 0; + return RunTasks( + (folder, innerProgress) => folder.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService), + children, + progress, + cancellationToken); + } - foreach (var child in children) + /// <summary> + /// Runs an action block on a list of children. + /// </summary> + /// <param name="task">The task to run for each child.</param> + /// <param name="children">The list of children.</param> + /// <param name="progress">The progress.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task.</returns> + private async Task RunTasks<T>(Func<T, IProgress<double>, Task> task, IList<T> children, IProgress<double> progress, CancellationToken cancellationToken) + { + var childrenCount = children.Count; + var childrenProgress = new double[childrenCount]; + + void UpdateProgress() { - cancellationToken.ThrowIfCancellationRequested(); + progress.Report(childrenProgress.Average()); + } - var innerProgress = new ActionableProgress<double>(); + var fanoutConcurrency = ConfigurationManager.Configuration.LibraryScanFanoutConcurrency; + var parallelism = fanoutConcurrency == 0 ? Environment.ProcessorCount : fanoutConcurrency; + + var actionBlock = new ActionBlock<int>( + async i => + { + var innerProgress = new ActionableProgress<double>(); + + innerProgress.RegisterAction(innerPercent => + { + // round the percent and only update progress if it changed to prevent excessive UpdateProgress calls + var innerPercentRounded = Math.Round(innerPercent); + if (childrenProgress[i] != innerPercentRounded) + { + childrenProgress[i] = innerPercentRounded; + UpdateProgress(); + } + }); + + await task(children[i], innerProgress).ConfigureAwait(false); - // Avoid implicitly captured closure - var currentInnerPercent = currentPercent; + childrenProgress[i] = 100; - innerProgress.RegisterAction(p => + UpdateProgress(); + }, + new ExecutionDataflowBlockOptions { - double innerPercent = currentInnerPercent; - innerPercent += p / count; - progress.Report(innerPercent); + MaxDegreeOfParallelism = parallelism, + CancellationToken = cancellationToken, }); - await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService) - .ConfigureAwait(false); + for (var i = 0; i < childrenCount; i++) + { + actionBlock.Post(i); + } - numComplete++; - double percent = numComplete; - percent /= count; - percent *= 100; - currentPercent = percent; + actionBlock.Complete(); - progress.Report(percent); - } + await actionBlock.Completion.ConfigureAwait(false); } /// <summary> @@ -1763,5 +1784,45 @@ namespace MediaBrowser.Controller.Entities } } } + + /// <summary> + /// Contains constants used when reporting scan progress. + /// </summary> + private static class ProgressHelpers + { + /// <summary> + /// Reported after the folders immediate children are retrieved. + /// </summary> + public const int RetrievedChildren = 5; + + /// <summary> + /// Reported after add, updating, or deleting child items from the LibraryManager. + /// </summary> + public const int UpdatedChildItems = 10; + + /// <summary> + /// Reported once subfolders are scanned. + /// When scanning subfolders, the progress will be between [UpdatedItems, ScannedSubfolders]. + /// </summary> + public const int ScannedSubfolders = 50; + + /// <summary> + /// Reported once metadata is refreshed. + /// When refreshing metadata, the progress will be between [ScannedSubfolders, MetadataRefreshed]. + /// </summary> + public const int RefreshedMetadata = 100; + + /// <summary> + /// Gets the current progress given the previous step, next step, and progress in between. + /// </summary> + /// <param name="previousProgressStep">The previous progress step.</param> + /// <param name="nextProgressStep">The next progress step.</param> + /// <param name="currentProgress">The current progress step.</param> + /// <returns>The progress.</returns> + public static double GetProgress(int previousProgressStep, int nextProgressStep, double currentProgress) + { + return previousProgressStep + ((nextProgressStep - previousProgressStep) * (currentProgress / 100)); + } + } } } diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 9acc98dce..5f75df54e 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -17,6 +17,7 @@ <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="5.0.0" /> <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" /> <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All"/> + <PackageReference Include="System.Threading.Tasks.Dataflow" Version="5.0.0" /> </ItemGroup> <ItemGroup> diff --git a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs index e7f042d2f..34fe895cc 100644 --- a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs +++ b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs @@ -117,6 +117,6 @@ namespace MediaBrowser.Controller.MediaEncoding void UpdateEncoderPath(string path, string pathType); - IEnumerable<string> GetPrimaryPlaylistVobFiles(string path, IIsoMount isoMount, uint? titleNumber); + IEnumerable<string> GetPrimaryPlaylistVobFiles(string path, uint? titleNumber); } } diff --git a/MediaBrowser.Controller/Providers/IProviderManager.cs b/MediaBrowser.Controller/Providers/IProviderManager.cs index 996ec27c0..0a4967223 100644 --- a/MediaBrowser.Controller/Providers/IProviderManager.cs +++ b/MediaBrowser.Controller/Providers/IProviderManager.cs @@ -46,6 +46,14 @@ namespace MediaBrowser.Controller.Providers Task<ItemUpdateType> RefreshSingleItem(BaseItem item, MetadataRefreshOptions options, CancellationToken cancellationToken); /// <summary> + /// Runs multiple metadata refreshes concurrently. + /// </summary> + /// <param name="action">The action to run.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns> + Task RunMetadataRefresh(Func<Task> action, CancellationToken cancellationToken); + + /// <summary> /// Saves the image. /// </summary> /// <param name="item">The item.</param> diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index 380894278..b1da9c712 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -949,16 +949,14 @@ namespace MediaBrowser.MediaEncoding.Encoder } /// <inheritdoc /> - public IEnumerable<string> GetPrimaryPlaylistVobFiles(string path, IIsoMount isoMount, uint? titleNumber) + public IEnumerable<string> GetPrimaryPlaylistVobFiles(string path, uint? titleNumber) { // min size 300 mb const long MinPlayableSize = 314572800; - var root = isoMount != null ? isoMount.MountedPath : path; - // Try to eliminate menus and intros by skipping all files at the front of the list that are less than the minimum size // Once we reach a file that is at least the minimum, return all subsequent ones - var allVobs = _fileSystem.GetFiles(root, true) + var allVobs = _fileSystem.GetFiles(path, true) .Where(file => string.Equals(file.Extension, ".vob", StringComparison.OrdinalIgnoreCase)) .OrderBy(i => i.FullName) .ToList(); diff --git a/MediaBrowser.Model/Configuration/ServerConfiguration.cs b/MediaBrowser.Model/Configuration/ServerConfiguration.cs index 830c8bd10..0dbd51bdc 100644 --- a/MediaBrowser.Model/Configuration/ServerConfiguration.cs +++ b/MediaBrowser.Model/Configuration/ServerConfiguration.cs @@ -439,5 +439,15 @@ namespace MediaBrowser.Model.Configuration /// Gets or sets the number of days we should retain activity logs. /// </summary> public int? ActivityLogRetentionDays { get; set; } = 30; + + /// <summary> + /// Gets or sets the how the library scan fans out. + /// </summary> + public int LibraryScanFanoutConcurrency { get; set; } + + /// <summary> + /// Gets or sets the how many metadata refreshes can run concurrently. + /// </summary> + public int LibraryMetadataRefreshConcurrency { get; set; } } } diff --git a/MediaBrowser.Model/IO/IIsoManager.cs b/MediaBrowser.Model/IO/IIsoManager.cs deleted file mode 100644 index 299bb0a21..000000000 --- a/MediaBrowser.Model/IO/IIsoManager.cs +++ /dev/null @@ -1,35 +0,0 @@ -#pragma warning disable CS1591 - -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -namespace MediaBrowser.Model.IO -{ - public interface IIsoManager - { - /// <summary> - /// Mounts the specified iso path. - /// </summary> - /// <param name="isoPath">The iso path.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>IsoMount.</returns> - /// <exception cref="IOException">Unable to create mount.</exception> - Task<IIsoMount> Mount(string isoPath, CancellationToken cancellationToken); - - /// <summary> - /// Determines whether this instance can mount the specified path. - /// </summary> - /// <param name="path">The path.</param> - /// <returns><c>true</c> if this instance can mount the specified path; otherwise, <c>false</c>.</returns> - bool CanMount(string path); - - /// <summary> - /// Adds the parts. - /// </summary> - /// <param name="mounters">The mounters.</param> - void AddParts(IEnumerable<IIsoMounter> mounters); - } -} diff --git a/MediaBrowser.Model/IO/IIsoMount.cs b/MediaBrowser.Model/IO/IIsoMount.cs deleted file mode 100644 index ea65d976a..000000000 --- a/MediaBrowser.Model/IO/IIsoMount.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; - -namespace MediaBrowser.Model.IO -{ - /// <summary> - /// Interface IIsoMount. - /// </summary> - public interface IIsoMount : IDisposable - { - /// <summary> - /// Gets the iso path. - /// </summary> - /// <value>The iso path.</value> - string IsoPath { get; } - - /// <summary> - /// Gets the mounted path. - /// </summary> - /// <value>The mounted path.</value> - string MountedPath { get; } - } -} diff --git a/MediaBrowser.Model/IO/IIsoMounter.cs b/MediaBrowser.Model/IO/IIsoMounter.cs deleted file mode 100644 index 0d257395a..000000000 --- a/MediaBrowser.Model/IO/IIsoMounter.cs +++ /dev/null @@ -1,35 +0,0 @@ -#pragma warning disable CS1591 - -using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -namespace MediaBrowser.Model.IO -{ - public interface IIsoMounter - { - /// <summary> - /// Gets the name. - /// </summary> - /// <value>The name.</value> - string Name { get; } - - /// <summary> - /// Mounts the specified iso path. - /// </summary> - /// <param name="isoPath">The iso path.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>IsoMount.</returns> - /// <exception cref="ArgumentNullException">isoPath</exception> - /// <exception cref="IOException">Unable to create mount.</exception> - Task<IIsoMount> Mount(string isoPath, CancellationToken cancellationToken); - - /// <summary> - /// Determines whether this instance can mount the specified path. - /// </summary> - /// <param name="path">The path.</param> - /// <returns><c>true</c> if this instance can mount the specified path; otherwise, <c>false</c>.</returns> - bool CanMount(string path); - } -} diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index e7e44876d..a20c47cf2 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -1167,6 +1167,29 @@ namespace MediaBrowser.Providers.Manager return RefreshItem(item, options, cancellationToken); } + /// <summary> + /// Runs multiple metadata refreshes concurrently. + /// </summary> + /// <param name="action">The action to run.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns> + public async Task RunMetadataRefresh(Func<Task> action, CancellationToken cancellationToken) + { + // create a variable for this since it is possible MetadataRefreshThrottler could change due to a config update during a scan + var metadataRefreshThrottler = _baseItemManager.MetadataRefreshThrottler; + + await metadataRefreshThrottler.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + await action().ConfigureAwait(false); + } + finally + { + metadataRefreshThrottler.Release(); + } + } + /// <inheritdoc/> public void Dispose() { diff --git a/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs b/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs index 6d39c091e..74849a522 100644 --- a/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs +++ b/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs @@ -619,7 +619,7 @@ namespace MediaBrowser.Providers.MediaInfo item.RunTimeTicks = GetRuntime(primaryTitle); } - return _mediaEncoder.GetPrimaryPlaylistVobFiles(item.Path, null, titleNumber) + return _mediaEncoder.GetPrimaryPlaylistVobFiles(item.Path, titleNumber) .Select(Path.GetFileName) .ToArray(); } |