Merge pull request #4242 from Spacetech/library_scan_speed

Increase library scan and metadata refresh speed

8 files changed, 243 insertions, 81 deletions

diff --git a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs
index ff64e217a..ae1b51b4c 100644
--- a/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs
+++ b/Emby.Server.Implementations/EntryPoints/LibraryChangedNotifier.cs
@@ -1,6 +1,7 @@
 #pragma warning disable CS1591
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
@@ -44,7 +45,7 @@ namespace Emby.Server.Implementations.EntryPoints
         private readonly List<BaseItem> _itemsAdded = new List<BaseItem>();
         private readonly List<BaseItem> _itemsRemoved = new List<BaseItem>();
         private readonly List<BaseItem> _itemsUpdated = new List<BaseItem>();
-        private readonly Dictionary<Guid, DateTime> _lastProgressMessageTimes = new Dictionary<Guid, DateTime>();
+        private readonly ConcurrentDictionary<Guid, DateTime> _lastProgressMessageTimes = new ConcurrentDictionary<Guid, DateTime>();
 
         public LibraryChangedNotifier(
             ILibraryManager libraryManager,
@@ -98,7 +99,7 @@ namespace Emby.Server.Implementations.EntryPoints
                 }
             }
 
-            _lastProgressMessageTimes[item.Id] = DateTime.UtcNow;
+            _lastProgressMessageTimes.AddOrUpdate(item.Id, key => DateTime.UtcNow, (key, existing) => DateTime.UtcNow);
 
             var dict = new Dictionary<string, string>();
             dict["ItemId"] = item.Id.ToString("N", CultureInfo.InvariantCulture);
@@ -140,6 +141,8 @@ namespace Emby.Server.Implementations.EntryPoints
         private void OnProviderRefreshCompleted(object sender, GenericEventArgs<BaseItem> e)
         {
             OnProviderRefreshProgress(sender, new GenericEventArgs<Tuple<BaseItem, double>>(new Tuple<BaseItem, double>(e.Argument, 100)));
+
+            _lastProgressMessageTimes.TryRemove(e.Argument.Id, out DateTime removed);
         }
 
         private static bool EnableRefreshMessage(BaseItem item)
diff --git a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs
index 67aa7f338..085f769d0 100644
--- a/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs
+++ b/MediaBrowser.Controller/BaseItemManager/BaseItemManager.cs
@@ -1,5 +1,7 @@
-using System;
+using System;
 using System.Linq;
+using System.Threading;
+using MediaBrowser.Common.Configuration;
 using MediaBrowser.Controller.Channels;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
@@ -12,6 +14,8 @@ namespace MediaBrowser.Controller.BaseItemManager
     {
         private readonly IServerConfigurationManager _serverConfigurationManager;
 
+        private int _metadataRefreshConcurrency = 0;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="BaseItemManager"/> class.
         /// </summary>
@@ -19,9 +23,17 @@ namespace MediaBrowser.Controller.BaseItemManager
         public BaseItemManager(IServerConfigurationManager serverConfigurationManager)
         {
             _serverConfigurationManager = serverConfigurationManager;
+
+            _metadataRefreshConcurrency = GetMetadataRefreshConcurrency();
+            SetupMetadataThrottler();
+
+            _serverConfigurationManager.ConfigurationUpdated += OnConfigurationUpdated;
         }
 
         /// <inheritdoc />
+        public SemaphoreSlim MetadataRefreshThrottler { get; private set; }
+
+        /// <inheritdoc />
         public bool IsMetadataFetcherEnabled(BaseItem baseItem, LibraryOptions libraryOptions, string name)
         {
             if (baseItem is Channel)
@@ -82,5 +94,42 @@ namespace MediaBrowser.Controller.BaseItemManager
 
             return itemConfig == null || !itemConfig.DisabledImageFetchers.Contains(name, StringComparer.OrdinalIgnoreCase);
         }
+
+        /// <summary>
+        /// Called when the configuration is updated.
+        /// It will refresh the metadata throttler if the relevant config changed.
+        /// </summary>
+        private void OnConfigurationUpdated(object sender, EventArgs e)
+        {
+            int newMetadataRefreshConcurrency = GetMetadataRefreshConcurrency();
+            if (_metadataRefreshConcurrency != newMetadataRefreshConcurrency)
+            {
+                _metadataRefreshConcurrency = newMetadataRefreshConcurrency;
+                SetupMetadataThrottler();
+            }
+        }
+
+        /// <summary>
+        /// Creates the metadata refresh throttler.
+        /// </summary>
+        private void SetupMetadataThrottler()
+        {
+            MetadataRefreshThrottler = new SemaphoreSlim(_metadataRefreshConcurrency);
+        }
+
+        /// <summary>
+        /// Returns the metadata refresh concurrency.
+        /// </summary>
+        private int GetMetadataRefreshConcurrency()
+        {
+            var concurrency = _serverConfigurationManager.Configuration.LibraryMetadataRefreshConcurrency;
+
+            if (concurrency <= 0)
+            {
+                concurrency = Environment.ProcessorCount;
+            }
+
+            return concurrency;
+        }
     }
 }
diff --git a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs
index ee4d3dcdc..e1f5d05a6 100644
--- a/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs
+++ b/MediaBrowser.Controller/BaseItemManager/IBaseItemManager.cs
@@ -1,4 +1,6 @@
-using MediaBrowser.Controller.Entities;
+using System;
+using System.Threading;
+using MediaBrowser.Controller.Entities;
 using MediaBrowser.Model.Configuration;
 
 namespace MediaBrowser.Controller.BaseItemManager
@@ -9,6 +11,11 @@ namespace MediaBrowser.Controller.BaseItemManager
     public interface IBaseItemManager
     {
         /// <summary>
+        /// Gets the semaphore used to limit the amount of concurrent metadata refreshes.
+        /// </summary>
+        SemaphoreSlim MetadataRefreshThrottler { get; }
+
+        /// <summary>
         /// Is metadata fetcher enabled.
         /// </summary>
         /// <param name="baseItem">The base item.</param>
diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs
index 675cdbd96..23f4c00c1 100644
--- a/MediaBrowser.Controller/Entities/Folder.cs
+++ b/MediaBrowser.Controller/Entities/Folder.cs
@@ -8,6 +8,7 @@ using System.Linq;
 using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
 using Jellyfin.Data.Entities;
 using Jellyfin.Data.Enums;
 using MediaBrowser.Common.Progress;
@@ -328,11 +329,11 @@ namespace MediaBrowser.Controller.Entities
                     return;
                 }
 
-                progress.Report(5);
+                progress.Report(ProgressHelpers.RetrievedChildren);
 
                 if (recursive)
                 {
-                    ProviderManager.OnRefreshProgress(this, 5);
+                    ProviderManager.OnRefreshProgress(this, ProgressHelpers.RetrievedChildren);
                 }
 
                 // Build a dictionary of the current children we have now by Id so we can compare quickly and easily
@@ -388,11 +389,11 @@ namespace MediaBrowser.Controller.Entities
                 validChildrenNeedGeneration = true;
             }
 
-            progress.Report(10);
+            progress.Report(ProgressHelpers.UpdatedChildItems);
 
             if (recursive)
             {
-                ProviderManager.OnRefreshProgress(this, 10);
+                ProviderManager.OnRefreshProgress(this, ProgressHelpers.UpdatedChildItems);
             }
 
             cancellationToken.ThrowIfCancellationRequested();
@@ -402,11 +403,13 @@ namespace MediaBrowser.Controller.Entities
                 var innerProgress = new ActionableProgress<double>();
 
                 var folder = this;
-                innerProgress.RegisterAction(p =>
+                innerProgress.RegisterAction(innerPercent =>
                 {
-                    double newPct = 0.80 * p + 10;
-                    progress.Report(newPct);
-                    ProviderManager.OnRefreshProgress(folder, newPct);
+                    var percent = ProgressHelpers.GetProgress(ProgressHelpers.UpdatedChildItems, ProgressHelpers.ScannedSubfolders, innerPercent);
+
+                    progress.Report(percent);
+
+                    ProviderManager.OnRefreshProgress(folder, percent);
                 });
 
                 if (validChildrenNeedGeneration)
@@ -420,11 +423,11 @@ namespace MediaBrowser.Controller.Entities
 
             if (refreshChildMetadata)
             {
-                progress.Report(90);
+                progress.Report(ProgressHelpers.ScannedSubfolders);
 
                 if (recursive)
                 {
-                    ProviderManager.OnRefreshProgress(this, 90);
+                    ProviderManager.OnRefreshProgress(this, ProgressHelpers.ScannedSubfolders);
                 }
 
                 var container = this as IMetadataContainer;
@@ -432,13 +435,15 @@ namespace MediaBrowser.Controller.Entities
                 var innerProgress = new ActionableProgress<double>();
 
                 var folder = this;
-                innerProgress.RegisterAction(p =>
+                innerProgress.RegisterAction(innerPercent =>
                 {
-                    double newPct = 0.10 * p + 90;
-                    progress.Report(newPct);
+                    var percent = ProgressHelpers.GetProgress(ProgressHelpers.ScannedSubfolders, ProgressHelpers.RefreshedMetadata, innerPercent);
+
+                    progress.Report(percent);
+
                     if (recursive)
                     {
-                        ProviderManager.OnRefreshProgress(folder, newPct);
+                        ProviderManager.OnRefreshProgress(folder, percent);
                     }
                 });
 
@@ -453,55 +458,35 @@ namespace MediaBrowser.Controller.Entities
                         validChildren = Children.ToList();
                     }
 
-                    await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken);
+                    await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken).ConfigureAwait(false);
                 }
             }
         }
 
-        private async Task RefreshMetadataRecursive(List<BaseItem> children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
+        private Task RefreshMetadataRecursive(IList<BaseItem> children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var numComplete = 0;
-            var count = children.Count;
-            double currentPercent = 0;
-
-            foreach (var child in children)
-            {
-                cancellationToken.ThrowIfCancellationRequested();
-
-                var innerProgress = new ActionableProgress<double>();
-
-                // Avoid implicitly captured closure
-                var currentInnerPercent = currentPercent;
-
-                innerProgress.RegisterAction(p =>
-                {
-                    double innerPercent = currentInnerPercent;
-                    innerPercent += p / count;
-                    progress.Report(innerPercent);
-                });
-
-                await RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken)
-                    .ConfigureAwait(false);
-
-                numComplete++;
-                double percent = numComplete;
-                percent /= count;
-                percent *= 100;
-                currentPercent = percent;
-
-                progress.Report(percent);
-            }
+            return RunTasks(
+                (baseItem, innerProgress) => RefreshChildMetadata(baseItem, refreshOptions, recursive && baseItem.IsFolder, innerProgress, cancellationToken),
+                children,
+                progress,
+                cancellationToken);
         }
 
         private async Task RefreshAllMetadataForContainer(IMetadataContainer container, MetadataRefreshOptions refreshOptions, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var series = container as Series;
-            if (series != null)
-            {
-                await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
-            }
+            // limit the amount of concurrent metadata refreshes
+            await ProviderManager.RunMetadataRefresh(
+                async () =>
+                {
+                    var series = container as Series;
+                    if (series != null)
+                    {
+                        await series.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
+                    }
 
-            await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false);
+                    await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false);
+                },
+                cancellationToken).ConfigureAwait(false);
         }
 
         private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
@@ -516,12 +501,15 @@ namespace MediaBrowser.Controller.Entities
             {
                 if (refreshOptions.RefreshItem(child))
                 {
-                    await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
+                    // limit the amount of concurrent metadata refreshes
+                    await ProviderManager.RunMetadataRefresh(
+                        async () => await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false),
+                        cancellationToken).ConfigureAwait(false);
                 }
 
                 if (recursive && child is Folder folder)
                 {
-                    await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken);
+                    await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken).ConfigureAwait(false);
                 }
             }
         }
@@ -534,39 +522,72 @@ namespace MediaBrowser.Controller.Entities
         /// <param name="progress">The progress.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns>Task.</returns>
-        private async Task ValidateSubFolders(IList<Folder> children, IDirectoryService directoryService, IProgress<double> progress, CancellationToken cancellationToken)
+        private Task ValidateSubFolders(IList<Folder> children, IDirectoryService directoryService, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var numComplete = 0;
-            var count = children.Count;
-            double currentPercent = 0;
+            return RunTasks(
+                (folder, innerProgress) => folder.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService),
+                children,
+                progress,
+                cancellationToken);
+        }
 
-            foreach (var child in children)
+        /// <summary>
+        /// Runs an action block on a list of children.
+        /// </summary>
+        /// <param name="task">The task to run for each child.</param>
+        /// <param name="children">The list of children.</param>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        private async Task RunTasks<T>(Func<T, IProgress<double>, Task> task, IList<T> children, IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            var childrenCount = children.Count;
+            var childrenProgress = new double[childrenCount];
+
+            void UpdateProgress()
             {
-                cancellationToken.ThrowIfCancellationRequested();
+                progress.Report(childrenProgress.Average());
+            }
 
-                var innerProgress = new ActionableProgress<double>();
+            var fanoutConcurrency = ConfigurationManager.Configuration.LibraryScanFanoutConcurrency;
+            var parallelism = fanoutConcurrency == 0 ? Environment.ProcessorCount : fanoutConcurrency;
+
+            var actionBlock = new ActionBlock<int>(
+                async i =>
+                {
+                    var innerProgress = new ActionableProgress<double>();
+
+                    innerProgress.RegisterAction(innerPercent =>
+                    {
+                        // round the percent and only update progress if it changed to prevent excessive UpdateProgress calls
+                        var innerPercentRounded = Math.Round(innerPercent);
+                        if (childrenProgress[i] != innerPercentRounded)
+                        {
+                            childrenProgress[i] = innerPercentRounded;
+                            UpdateProgress();
+                        }
+                    });
+
+                    await task(children[i], innerProgress).ConfigureAwait(false);
 
-                // Avoid implicitly captured closure
-                var currentInnerPercent = currentPercent;
+                    childrenProgress[i] = 100;
 
-                innerProgress.RegisterAction(p =>
+                    UpdateProgress();
+                },
+                new ExecutionDataflowBlockOptions
                 {
-                    double innerPercent = currentInnerPercent;
-                    innerPercent += p / count;
-                    progress.Report(innerPercent);
+                    MaxDegreeOfParallelism = parallelism,
+                    CancellationToken = cancellationToken,
                 });
 
-                await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService)
-                        .ConfigureAwait(false);
+            for (var i = 0; i < childrenCount; i++)
+            {
+                actionBlock.Post(i);
+            }
 
-                numComplete++;
-                double percent = numComplete;
-                percent /= count;
-                percent *= 100;
-                currentPercent = percent;
+            actionBlock.Complete();
 
-                progress.Report(percent);
-            }
+            await actionBlock.Completion.ConfigureAwait(false);
         }
 
         /// <summary>
@@ -1763,5 +1784,45 @@ namespace MediaBrowser.Controller.Entities
                 }
             }
         }
+
+        /// <summary>
+        /// Contains constants used when reporting scan progress.
+        /// </summary>
+        private static class ProgressHelpers
+        {
+            /// <summary>
+            /// Reported after the folders immediate children are retrieved.
+            /// </summary>
+            public const int RetrievedChildren = 5;
+
+            /// <summary>
+            /// Reported after add, updating, or deleting child items from the LibraryManager.
+            /// </summary>
+            public const int UpdatedChildItems = 10;
+
+            /// <summary>
+            /// Reported once subfolders are scanned.
+            /// When scanning subfolders, the progress will be between [UpdatedItems, ScannedSubfolders].
+            /// </summary>
+            public const int ScannedSubfolders = 50;
+
+            /// <summary>
+            /// Reported once metadata is refreshed.
+            /// When refreshing metadata, the progress will be between [ScannedSubfolders, MetadataRefreshed].
+            /// </summary>
+            public const int RefreshedMetadata = 100;
+
+            /// <summary>
+            /// Gets the current progress given the previous step, next step, and progress in between.
+            /// </summary>
+            /// <param name="previousProgressStep">The previous progress step.</param>
+            /// <param name="nextProgressStep">The next progress step.</param>
+            /// <param name="currentProgress">The current progress step.</param>
+            /// <returns>The progress.</returns>
+            public static double GetProgress(int previousProgressStep, int nextProgressStep, double currentProgress)
+            {
+                return previousProgressStep + ((nextProgressStep - previousProgressStep) * (currentProgress / 100));
+            }
+        }
     }
 }
diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj
index 9acc98dce..5f75df54e 100644
--- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj
+++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj
@@ -17,6 +17,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All"/>
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="5.0.0" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/MediaBrowser.Controller/Providers/IProviderManager.cs b/MediaBrowser.Controller/Providers/IProviderManager.cs
index 996ec27c0..0a4967223 100644
--- a/MediaBrowser.Controller/Providers/IProviderManager.cs
+++ b/MediaBrowser.Controller/Providers/IProviderManager.cs
@@ -46,6 +46,14 @@ namespace MediaBrowser.Controller.Providers
         Task<ItemUpdateType> RefreshSingleItem(BaseItem item, MetadataRefreshOptions options, CancellationToken cancellationToken);
 
         /// <summary>
+        /// Runs multiple metadata refreshes concurrently.
+        /// </summary>
+        /// <param name="action">The action to run.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns>
+        Task RunMetadataRefresh(Func<Task> action, CancellationToken cancellationToken);
+
+        /// <summary>
         /// Saves the image.
         /// </summary>
         /// <param name="item">The item.</param>
diff --git a/MediaBrowser.Model/Configuration/ServerConfiguration.cs b/MediaBrowser.Model/Configuration/ServerConfiguration.cs
index 830c8bd10..0dbd51bdc 100644
--- a/MediaBrowser.Model/Configuration/ServerConfiguration.cs
+++ b/MediaBrowser.Model/Configuration/ServerConfiguration.cs
@@ -439,5 +439,15 @@ namespace MediaBrowser.Model.Configuration
         /// Gets or sets the number of days we should retain activity logs.
         /// </summary>
         public int? ActivityLogRetentionDays { get; set; } = 30;
+
+        /// <summary>
+        /// Gets or sets the how the library scan fans out.
+        /// </summary>
+        public int LibraryScanFanoutConcurrency { get; set; }
+
+        /// <summary>
+        /// Gets or sets the how many metadata refreshes can run concurrently.
+        /// </summary>
+        public int LibraryMetadataRefreshConcurrency { get; set; }
     }
 }
diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs
index e7e44876d..a20c47cf2 100644
--- a/MediaBrowser.Providers/Manager/ProviderManager.cs
+++ b/MediaBrowser.Providers/Manager/ProviderManager.cs
@@ -1167,6 +1167,29 @@ namespace MediaBrowser.Providers.Manager
             return RefreshItem(item, options, cancellationToken);
         }
 
+        /// <summary>
+        /// Runs multiple metadata refreshes concurrently.
+        /// </summary>
+        /// <param name="action">The action to run.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns>
+        public async Task RunMetadataRefresh(Func<Task> action, CancellationToken cancellationToken)
+        {
+            // create a variable for this since it is possible MetadataRefreshThrottler could change due to a config update during a scan
+            var metadataRefreshThrottler = _baseItemManager.MetadataRefreshThrottler;
+
+            await metadataRefreshThrottler.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+            try
+            {
+                await action().ConfigureAwait(false);
+            }
+            finally
+            {
+                metadataRefreshThrottler.Release();
+            }
+        }
+
         /// <inheritdoc/>
         public void Dispose()
         {