From 1e5c3db9eba730fe8b52995e5c699c22983fa62a Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Fri, 23 Jun 2017 12:04:45 -0400 Subject: support individual library refreshing --- MediaBrowser.Controller/Channels/Channel.cs | 3 +- MediaBrowser.Controller/Dto/DtoOptions.cs | 3 +- .../Entities/Audio/MusicAlbum.cs | 2 - .../Entities/Audio/MusicArtist.cs | 2 - MediaBrowser.Controller/Entities/BaseItem.cs | 38 +++- .../Entities/CollectionFolder.cs | 30 ++++ MediaBrowser.Controller/Entities/Folder.cs | 196 +++++++++++++-------- MediaBrowser.Controller/Entities/TV/Series.cs | 2 - MediaBrowser.Controller/Library/ILibraryManager.cs | 4 +- .../MediaBrowser.Controller.csproj | 1 - .../Providers/IProviderManager.cs | 19 +- .../Providers/ProviderRefreshStatus.cs | 22 --- 12 files changed, 208 insertions(+), 114 deletions(-) delete mode 100644 MediaBrowser.Controller/Providers/ProviderRefreshStatus.cs (limited to 'MediaBrowser.Controller') diff --git a/MediaBrowser.Controller/Channels/Channel.cs b/MediaBrowser.Controller/Channels/Channel.cs index 862c8e5ace..8e2faa20e2 100644 --- a/MediaBrowser.Controller/Channels/Channel.cs +++ b/MediaBrowser.Controller/Channels/Channel.cs @@ -6,6 +6,7 @@ using System.Linq; using MediaBrowser.Model.Serialization; using System.Threading; using System.Threading.Tasks; +using MediaBrowser.Common.Progress; namespace MediaBrowser.Controller.Channels { @@ -51,7 +52,7 @@ namespace MediaBrowser.Controller.Channels SortBy = query.SortBy, SortOrder = query.SortOrder - }, new Progress(), CancellationToken.None).Result; + }, new SimpleProgress(), CancellationToken.None).Result; } catch { diff --git a/MediaBrowser.Controller/Dto/DtoOptions.cs b/MediaBrowser.Controller/Dto/DtoOptions.cs index b9d9d7ddad..098ba558f8 100644 --- a/MediaBrowser.Controller/Dto/DtoOptions.cs +++ b/MediaBrowser.Controller/Dto/DtoOptions.cs @@ -10,7 +10,8 @@ namespace MediaBrowser.Controller.Dto { private static readonly List DefaultExcludedFields = new List { - ItemFields.SeasonUserData + ItemFields.SeasonUserData, + ItemFields.RefreshState }; public List Fields { get; set; } diff --git a/MediaBrowser.Controller/Entities/Audio/MusicAlbum.cs b/MediaBrowser.Controller/Entities/Audio/MusicAlbum.cs index e9eca19cf7..516ab50530 100644 --- a/MediaBrowser.Controller/Entities/Audio/MusicAlbum.cs +++ b/MediaBrowser.Controller/Entities/Audio/MusicAlbum.cs @@ -235,8 +235,6 @@ namespace MediaBrowser.Controller.Entities.Audio { await RefreshArtists(refreshOptions, cancellationToken).ConfigureAwait(false); } - - progress.Report(100); } private async Task RefreshArtists(MetadataRefreshOptions refreshOptions, CancellationToken cancellationToken) diff --git a/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs b/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs index 4b232de499..ebd83205e5 100644 --- a/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs +++ b/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs @@ -250,8 +250,6 @@ namespace MediaBrowser.Controller.Entities.Audio percent /= totalItems; progress.Report(percent * 100); } - - progress.Report(100); } public ArtistInfo GetLookupInfo() diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index 4efea94d8e..b4a3d89eac 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -1058,6 +1058,16 @@ namespace MediaBrowser.Controller.Entities return RefreshMetadata(new MetadataRefreshOptions(new DirectoryService(Logger, FileSystem)), cancellationToken); } + protected virtual void TriggerOnRefreshStart() + { + + } + + protected virtual void TriggerOnRefreshComplete() + { + + } + /// /// Overrides the base implementation to refresh metadata for local trailers /// @@ -1066,6 +1076,8 @@ namespace MediaBrowser.Controller.Entities /// true if a provider reports we changed public async Task RefreshMetadata(MetadataRefreshOptions options, CancellationToken cancellationToken) { + TriggerOnRefreshStart(); + var locationType = LocationType; var requiresSave = false; @@ -1091,14 +1103,21 @@ namespace MediaBrowser.Controller.Entities } } - var refreshOptions = requiresSave - ? new MetadataRefreshOptions(options) - { - ForceSave = true - } - : options; + try + { + var refreshOptions = requiresSave + ? new MetadataRefreshOptions(options) + { + ForceSave = true + } + : options; - return await ProviderManager.RefreshSingleItem(this, refreshOptions, cancellationToken).ConfigureAwait(false); + return await ProviderManager.RefreshSingleItem(this, refreshOptions, cancellationToken).ConfigureAwait(false); + } + finally + { + TriggerOnRefreshComplete(); + } } [IgnoreDataMember] @@ -2421,5 +2440,10 @@ namespace MediaBrowser.Controller.Entities { return new List(); } + + public virtual double? GetRefreshProgress() + { + return null; + } } } \ No newline at end of file diff --git a/MediaBrowser.Controller/Entities/CollectionFolder.cs b/MediaBrowser.Controller/Entities/CollectionFolder.cs index 8bc23a5819..0bd8606b96 100644 --- a/MediaBrowser.Controller/Entities/CollectionFolder.cs +++ b/MediaBrowser.Controller/Entities/CollectionFolder.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using MediaBrowser.Controller.Configuration; using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.Entities; using MediaBrowser.Model.Extensions; using MediaBrowser.Model.IO; using MediaBrowser.Model.Serialization; @@ -199,6 +200,30 @@ namespace MediaBrowser.Controller.Entities return changed; } + public override double? GetRefreshProgress() + { + var folders = GetPhysicalFolders(true).ToList(); + double totalProgresses = 0; + var foldersWithProgress = 0; + + foreach (var folder in folders) + { + var progress = ProviderManager.GetRefreshProgress(folder.Id); + if (progress.HasValue) + { + totalProgresses += progress.Value; + foldersWithProgress++; + } + } + + if (foldersWithProgress == 0) + { + return null; + } + + return (totalProgresses / foldersWithProgress); + } + protected override bool RefreshLinkedChildren(IEnumerable fileSystemChildren) { return RefreshLinkedChildrenInternal(true); @@ -321,6 +346,11 @@ namespace MediaBrowser.Controller.Entities return GetPhysicalFolders(true).SelectMany(c => c.Children); } + public IEnumerable GetPhysicalFolders() + { + return GetPhysicalFolders(true); + } + private IEnumerable GetPhysicalFolders(bool enableCache) { if (enableCache) diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 727b7dbebe..34b33fde0d 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -271,6 +271,11 @@ namespace MediaBrowser.Controller.Entities return GetCachedChildren(); } + public override double? GetRefreshProgress() + { + return ProviderManager.GetRefreshProgress(Id); + } + public Task ValidateChildren(IProgress progress, CancellationToken cancellationToken) { return ValidateChildren(progress, cancellationToken, new MetadataRefreshOptions(new DirectoryService(Logger, FileSystem))); @@ -318,6 +323,14 @@ namespace MediaBrowser.Controller.Entities return current.IsValidFromResolver(newItem); } + protected override void TriggerOnRefreshStart() + { + } + + protected override void TriggerOnRefreshComplete() + { + } + /// /// Validates the children internal. /// @@ -328,7 +341,27 @@ namespace MediaBrowser.Controller.Entities /// The refresh options. /// The directory service. /// Task. - protected async virtual Task ValidateChildrenInternal(IProgress progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions, IDirectoryService directoryService) + protected virtual async Task ValidateChildrenInternal(IProgress progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions, IDirectoryService directoryService) + { + if (recursive) + { + ProviderManager.OnRefreshStart(this); + } + + try + { + await ValidateChildrenInternal2(progress, cancellationToken, recursive, refreshChildMetadata, refreshOptions, directoryService).ConfigureAwait(false); + } + finally + { + if (recursive) + { + ProviderManager.OnRefreshComplete(this); + } + } + } + + private async Task ValidateChildrenInternal2(IProgress progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions, IDirectoryService directoryService) { var locationType = LocationType; @@ -360,6 +393,11 @@ namespace MediaBrowser.Controller.Entities progress.Report(5); + if (recursive) + { + ProviderManager.OnRefreshProgress(this, 5); + } + //build a dictionary of the current children we have now by Id so we can compare quickly and easily var currentChildren = GetActualChildrenDictionary(); @@ -424,76 +462,99 @@ namespace MediaBrowser.Controller.Entities await LibraryManager.CreateItems(newItems, cancellationToken).ConfigureAwait(false); } } + else + { + if (recursive || refreshChildMetadata) + { + // used below + validChildren = Children.ToList(); + } + } progress.Report(10); - cancellationToken.ThrowIfCancellationRequested(); - if (recursive) { - await ValidateSubFolders(Children.ToList().OfType().ToList(), directoryService, progress, cancellationToken).ConfigureAwait(false); + ProviderManager.OnRefreshProgress(this, 10); } - progress.Report(20); + cancellationToken.ThrowIfCancellationRequested(); - if (refreshChildMetadata) + if (recursive) { - var container = this as IMetadataContainer; + using (var innerProgress = new ActionableProgress()) + { + var folder = this; + innerProgress.RegisterAction(p => + { + double newPct = .70 * p + 10; + progress.Report(newPct); + ProviderManager.OnRefreshProgress(folder, newPct); + }); - var innerProgress = new ActionableProgress(); + await ValidateSubFolders(validChildren.OfType().ToList(), directoryService, innerProgress, cancellationToken).ConfigureAwait(false); + } + } - innerProgress.RegisterAction(p => progress.Report(.80 * p + 20)); + if (refreshChildMetadata) + { + progress.Report(80); - if (container != null) + if (recursive) { - await container.RefreshAllMetadata(refreshOptions, innerProgress, cancellationToken).ConfigureAwait(false); + ProviderManager.OnRefreshProgress(this, 80); } - else + + var container = this as IMetadataContainer; + + using (var innerProgress = new ActionableProgress()) { - await RefreshMetadataRecursive(refreshOptions, recursive, innerProgress, cancellationToken); + var folder = this; + innerProgress.RegisterAction(p => + { + double newPct = .20 * p + 80; + progress.Report(newPct); + if (recursive) + { + ProviderManager.OnRefreshProgress(folder, newPct); + } + }); + + if (container != null) + { + await container.RefreshAllMetadata(refreshOptions, innerProgress, cancellationToken).ConfigureAwait(false); + } + else + { + await RefreshMetadataRecursive(validChildren, refreshOptions, recursive, innerProgress, cancellationToken); + } } } - - progress.Report(100); } - private async Task RefreshMetadataRecursive(MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) + private async Task RefreshMetadataRecursive(List children, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) { - var children = Children.ToList(); - - var percentages = new Dictionary(children.Count); var numComplete = 0; var count = children.Count; + double currentPercent = 0; foreach (var child in children) { cancellationToken.ThrowIfCancellationRequested(); - if (child.IsFolder) + using (var innerProgress = new ActionableProgress()) { - var innerProgress = new ActionableProgress(); - // Avoid implicitly captured closure - var currentChild = child; + var currentInnerPercent = currentPercent; + innerProgress.RegisterAction(p => { - lock (percentages) - { - percentages[currentChild.Id] = p / 100; - - var innerPercent = percentages.Values.Sum(); - innerPercent /= count; - innerPercent *= 100; - progress.Report(innerPercent); - } + double innerPercent = currentInnerPercent; + innerPercent += p / (count); + progress.Report(innerPercent); }); - await RefreshChildMetadata(child, refreshOptions, recursive, innerProgress, cancellationToken) - .ConfigureAwait(false); - } - else - { - await RefreshChildMetadata(child, refreshOptions, false, new Progress(), cancellationToken) + await RefreshChildMetadata(child, refreshOptions, recursive && child.IsFolder, innerProgress, cancellationToken) .ConfigureAwait(false); } @@ -501,11 +562,10 @@ namespace MediaBrowser.Controller.Entities double percent = numComplete; percent /= count; percent *= 100; + currentPercent = percent; progress.Report(percent); } - - progress.Report(100); } private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress progress, CancellationToken cancellationToken) @@ -526,11 +586,10 @@ namespace MediaBrowser.Controller.Entities if (folder != null) { - await folder.RefreshMetadataRecursive(refreshOptions, true, progress, cancellationToken); + await folder.RefreshMetadataRecursive(folder.Children.ToList(), refreshOptions, true, progress, cancellationToken); } } } - progress.Report(100); } /// @@ -543,47 +602,40 @@ namespace MediaBrowser.Controller.Entities /// Task. private async Task ValidateSubFolders(IList children, IDirectoryService directoryService, IProgress progress, CancellationToken cancellationToken) { - var list = children; - var childCount = list.Count; - - var percentages = new Dictionary(list.Count); + var numComplete = 0; + var count = children.Count; + double currentPercent = 0; - foreach (var item in list) + foreach (var child in children) { cancellationToken.ThrowIfCancellationRequested(); - var child = item; - - var innerProgress = new ActionableProgress(); - - innerProgress.RegisterAction(p => + using (var innerProgress = new ActionableProgress()) { - lock (percentages) + // Avoid implicitly captured closure + var currentInnerPercent = currentPercent; + + innerProgress.RegisterAction(p => { - percentages[child.Id] = p / 100; + double innerPercent = currentInnerPercent; + innerPercent += p / (count); + progress.Report(innerPercent); + }); - var percent = percentages.Values.Sum(); - percent /= childCount; + await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService) + .ConfigureAwait(false); + } - progress.Report(10 * percent + 10); - } - }); + numComplete++; + double percent = numComplete; + percent /= count; + percent *= 100; + currentPercent = percent; - await child.ValidateChildrenInternal(innerProgress, cancellationToken, true, false, null, directoryService) - .ConfigureAwait(false); + progress.Report(percent); } } - /// - /// Determines whether the specified path is offline. - /// - /// The path. - /// true if the specified path is offline; otherwise, false. - public static bool IsPathOffline(string path) - { - return IsPathOffline(path, LibraryManager.GetVirtualFolders().SelectMany(i => i.Locations).ToList()); - } - public static bool IsPathOffline(string path, List allLibraryPaths) { if (FileSystem.FileExists(path)) @@ -926,7 +978,7 @@ namespace MediaBrowser.Controller.Entities SortBy = query.SortBy, SortOrder = query.SortOrder - }, new Progress(), CancellationToken.None).Result; + }, new SimpleProgress(), CancellationToken.None).Result; } catch { diff --git a/MediaBrowser.Controller/Entities/TV/Series.cs b/MediaBrowser.Controller/Entities/TV/Series.cs index 0b96624505..1e4b3fdad3 100644 --- a/MediaBrowser.Controller/Entities/TV/Series.cs +++ b/MediaBrowser.Controller/Entities/TV/Series.cs @@ -414,8 +414,6 @@ namespace MediaBrowser.Controller.Entities.TV refreshOptions = new MetadataRefreshOptions(refreshOptions); refreshOptions.IsPostRecursiveRefresh = true; await ProviderManager.RefreshSingleItem(this, refreshOptions, cancellationToken).ConfigureAwait(false); - - progress.Report(100); } public IEnumerable GetSeasonEpisodes(Season parentSeason, User user, DtoOptions options) diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 5a702b4f08..b726c267c6 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -132,7 +132,9 @@ namespace MediaBrowser.Controller.Library /// Gets the default view. /// /// IEnumerable{VirtualFolderInfo}. - IEnumerable GetVirtualFolders(); + List GetVirtualFolders(); + + List GetVirtualFolders(bool includeRefreshState); /// /// Gets the item by id. diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 348cfd343f..22f94695d8 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -320,7 +320,6 @@ - diff --git a/MediaBrowser.Controller/Providers/IProviderManager.cs b/MediaBrowser.Controller/Providers/IProviderManager.cs index c0bc902140..0ba573da8b 100644 --- a/MediaBrowser.Controller/Providers/IProviderManager.cs +++ b/MediaBrowser.Controller/Providers/IProviderManager.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; +using MediaBrowser.Model.Events; namespace MediaBrowser.Controller.Providers { @@ -30,7 +31,7 @@ namespace MediaBrowser.Controller.Providers /// The cancellation token. /// Task. Task RefreshFullItem(IHasMetadata item, MetadataRefreshOptions options, CancellationToken cancellationToken); - + /// /// Refreshes the metadata. /// @@ -68,7 +69,7 @@ namespace MediaBrowser.Controller.Providers /// /// Task. Task SaveImage(IHasImages item, string source, string mimeType, ImageType type, int? imageIndex, bool? saveLocallyWithMedia, CancellationToken cancellationToken); - + /// /// Adds the metadata providers. /// @@ -128,7 +129,7 @@ namespace MediaBrowser.Controller.Providers /// The savers. /// Task. Task SaveMetadata(IHasMetadata item, ItemUpdateType updateType, IEnumerable savers); - + /// /// Gets the metadata options. /// @@ -158,6 +159,18 @@ namespace MediaBrowser.Controller.Providers /// The cancellation token. /// Task{HttpResponseInfo}. Task GetSearchImage(string providerName, string url, CancellationToken cancellationToken); + + Dictionary GetRefreshQueue(); + + void OnRefreshStart(BaseItem item); + void OnRefreshProgress(BaseItem item, double progress); + void OnRefreshComplete(BaseItem item); + + double? GetRefreshProgress(Guid id); + + event EventHandler> RefreshStarted; + event EventHandler> RefreshCompleted; + event EventHandler>> RefreshProgress; } public enum RefreshPriority diff --git a/MediaBrowser.Controller/Providers/ProviderRefreshStatus.cs b/MediaBrowser.Controller/Providers/ProviderRefreshStatus.cs deleted file mode 100644 index 6523dc4173..0000000000 --- a/MediaBrowser.Controller/Providers/ProviderRefreshStatus.cs +++ /dev/null @@ -1,22 +0,0 @@ - -namespace MediaBrowser.Controller.Providers -{ - /// - /// Enum ProviderRefreshStatus - /// - public enum ProviderRefreshStatus - { - /// - /// The success - /// - Success = 0, - /// - /// The completed with errors - /// - CompletedWithErrors = 1, - /// - /// The failure - /// - Failure = 2 - } -} -- cgit v1.2.3