From 17106ea5c72511f5871178c7f1def629c20191ac Mon Sep 17 00:00:00 2001 From: ebr11 Eric Reed spam Date: Mon, 17 Sep 2012 11:12:43 -0400 Subject: Initial commit changing to on-demand child loading and validations --- MediaBrowser.Common/Extensions/BaseExtensions.cs | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 MediaBrowser.Common/Extensions/BaseExtensions.cs (limited to 'MediaBrowser.Common/Extensions/BaseExtensions.cs') diff --git a/MediaBrowser.Common/Extensions/BaseExtensions.cs b/MediaBrowser.Common/Extensions/BaseExtensions.cs new file mode 100644 index 0000000000..89cfbecad2 --- /dev/null +++ b/MediaBrowser.Common/Extensions/BaseExtensions.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Security.Cryptography; + +namespace MediaBrowser.Common.Extensions +{ + public static class BaseExtensions + { + static MD5CryptoServiceProvider md5Provider = new MD5CryptoServiceProvider(); + + public static Guid GetMD5(this string str) + { + lock (md5Provider) + { + return new Guid(md5Provider.ComputeHash(Encoding.Unicode.GetBytes(str))); + } + } + + public static bool ContainsStartsWith(this List lst, string value) + { + foreach (var str in lst) + { + if (str.StartsWith(value, StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + } +} -- cgit v1.2.3 From e5b5861abf1ebd940e9b369de3cd3ab0432338c5 Mon Sep 17 00:00:00 2001 From: ebr11 Eric Reed spam Date: Tue, 18 Sep 2012 07:22:49 -0400 Subject: Need to be more intelligent finding paths we're already watching --- MediaBrowser.Common/Extensions/BaseExtensions.cs | 13 +++++++++++-- MediaBrowser.Controller/IO/DirectoryWatchers.cs | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'MediaBrowser.Common/Extensions/BaseExtensions.cs') diff --git a/MediaBrowser.Common/Extensions/BaseExtensions.cs b/MediaBrowser.Common/Extensions/BaseExtensions.cs index 89cfbecad2..ceb008629c 100644 --- a/MediaBrowser.Common/Extensions/BaseExtensions.cs +++ b/MediaBrowser.Common/Extensions/BaseExtensions.cs @@ -19,11 +19,20 @@ namespace MediaBrowser.Common.Extensions } } - public static bool ContainsStartsWith(this List lst, string value) + /// + /// Examine a list of strings assumed to be file paths to see if it contains a parent of + /// the provided path. + /// + /// + /// + /// + public static bool ContainsParentFolder(this List lst, string path) { foreach (var str in lst) { - if (str.StartsWith(value, StringComparison.OrdinalIgnoreCase)) return true; + //this should be a little quicker than examining each actual parent folder... + if (path.Equals(str,StringComparison.OrdinalIgnoreCase) + || (path.StartsWith(str, StringComparison.OrdinalIgnoreCase) && path[str.Length-1] == '\\')) return true; } return false; } diff --git a/MediaBrowser.Controller/IO/DirectoryWatchers.cs b/MediaBrowser.Controller/IO/DirectoryWatchers.cs index 91f74e6777..8243c8174a 100644 --- a/MediaBrowser.Controller/IO/DirectoryWatchers.cs +++ b/MediaBrowser.Controller/IO/DirectoryWatchers.cs @@ -30,7 +30,7 @@ namespace MediaBrowser.Controller.IO { foreach (string path in folder.PhysicalLocations) { - if (Path.IsPathRooted(path) && !pathsToWatch.ContainsStartsWith(path)) + if (Path.IsPathRooted(path) && !pathsToWatch.ContainsParentFolder(path)) { pathsToWatch.Add(path); } -- cgit v1.2.3 From 0f078d8098e60804c4f9b13c82566286362a83f2 Mon Sep 17 00:00:00 2001 From: ebr11 Eric Reed spam Date: Tue, 18 Sep 2012 15:18:15 -0400 Subject: Move NeedsRefresh to async task and fix problem finding episode metadata --- MediaBrowser.Common/Extensions/BaseExtensions.cs | 21 +++++++++++++++++++++ MediaBrowser.Controller/Entities/Folder.cs | 3 ++- MediaBrowser.Controller/Kernel.cs | 13 ++++++------- .../Providers/BaseItemXmlParser.cs | 1 + .../Providers/BaseMetadataProvider.cs | 14 +++++++++++--- .../Resolvers/EntityResolutionHelper.cs | 6 +++++- 6 files changed, 46 insertions(+), 12 deletions(-) (limited to 'MediaBrowser.Common/Extensions/BaseExtensions.cs') diff --git a/MediaBrowser.Common/Extensions/BaseExtensions.cs b/MediaBrowser.Common/Extensions/BaseExtensions.cs index ceb008629c..10ff019aa4 100644 --- a/MediaBrowser.Common/Extensions/BaseExtensions.cs +++ b/MediaBrowser.Common/Extensions/BaseExtensions.cs @@ -36,5 +36,26 @@ namespace MediaBrowser.Common.Extensions } return false; } + + /// + /// Helper method for Dictionaries since they throw on not-found keys + /// + /// + /// + /// + /// + /// + /// + public static U GetValueOrDefault(this Dictionary dictionary, T key, U defaultValue) + { + U val; + if (!dictionary.TryGetValue(key, out val)) + { + val = defaultValue; + } + return val; + + } + } } diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 1cad13f22e..249921f0fd 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -153,9 +153,9 @@ namespace MediaBrowser.Controller.Entities //brand new item - needs to be added changed = true; changedArgs.ItemsAdded.Add(child); - //Logger.LogInfo("New Item Added to Library: ("+child.GetType().Name+")"+ child.Name + "(" + child.Path + ")"); //refresh it child.RefreshMetadata(); + //Logger.LogInfo("New Item Added to Library: ("+child.GetType().Name+") "+ child.Name + " (" + child.Path + ")"); //save it in repo... //and add it to our valid children @@ -180,6 +180,7 @@ namespace MediaBrowser.Controller.Entities // will identify this item as the same one currentChild.ResolveArgs = child.ResolveArgs; currentChild.RefreshMetadata(); + Logger.LogInfo("Item Changed: ("+currentChild.GetType().Name+") "+ currentChild.Name + " (" + currentChild.Path + ")"); //save it in repo... validChildren.Add(currentChild); } diff --git a/MediaBrowser.Controller/Kernel.cs b/MediaBrowser.Controller/Kernel.cs index 6baae88113..1daa43dcd2 100644 --- a/MediaBrowser.Controller/Kernel.cs +++ b/MediaBrowser.Controller/Kernel.cs @@ -198,8 +198,13 @@ namespace MediaBrowser.Controller //re-start the directory watchers DirectoryWatchers.Stop(); DirectoryWatchers.Start(); + //Task.Delay(30000); //let's wait and see if more data gets filled in... var allChildren = RootFolder.RecursiveChildren; Logger.LogInfo(string.Format("Loading complete. Movies: {0} Episodes: {1}", allChildren.OfType().Count(), allChildren.OfType().Count())); + foreach (var child in allChildren) + { + Logger.LogDebugInfo("(" + child.GetType().Name + ") " + child.Name + " Overview " + (child.Overview != null ? child.Overview.Substring(0,Math.Min(25,child.Overview.Length)): "") + " (" + child.Path + ")"); + } } /// @@ -368,15 +373,9 @@ namespace MediaBrowser.Controller continue; } - // Skip if provider says we don't need to run - if (!provider.NeedsRefresh(item)) - { - continue; - } - try { - await provider.FetchAsync(item, item.ResolveArgs).ConfigureAwait(false); + await provider.FetchIfNeededAsync(item).ConfigureAwait(false); } catch (Exception ex) { diff --git a/MediaBrowser.Controller/Providers/BaseItemXmlParser.cs b/MediaBrowser.Controller/Providers/BaseItemXmlParser.cs index 1cb2e2af34..45343b0511 100644 --- a/MediaBrowser.Controller/Providers/BaseItemXmlParser.cs +++ b/MediaBrowser.Controller/Providers/BaseItemXmlParser.cs @@ -1,6 +1,7 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Xml; using MediaBrowser.Model.Entities; +using MediaBrowser.Common.Logging; using System; using System.Collections.Generic; using System.Linq; diff --git a/MediaBrowser.Controller/Providers/BaseMetadataProvider.cs b/MediaBrowser.Controller/Providers/BaseMetadataProvider.cs index 80a264af27..50004be442 100644 --- a/MediaBrowser.Controller/Providers/BaseMetadataProvider.cs +++ b/MediaBrowser.Controller/Providers/BaseMetadataProvider.cs @@ -36,7 +36,7 @@ namespace MediaBrowser.Controller.Providers /// protected virtual DateTime LastRefreshed(BaseEntity item) { - return (item.ProviderData[this.Id] ?? new BaseProviderInfo()).LastRefreshed; + return (item.ProviderData.GetValueOrDefault(this.Id, new BaseProviderInfo())).LastRefreshed; } /// @@ -46,7 +46,7 @@ namespace MediaBrowser.Controller.Providers /// protected virtual void SetLastRefreshed(BaseEntity item, DateTime value) { - var data = item.ProviderData[this.Id] ?? new BaseProviderInfo(); + var data = item.ProviderData.GetValueOrDefault(this.Id, new BaseProviderInfo()); data.LastRefreshed = value; item.ProviderData[this.Id] = data; } @@ -68,7 +68,15 @@ namespace MediaBrowser.Controller.Providers /// protected virtual DateTime CompareDate(BaseEntity item) { - return DateTime.MinValue; + return DateTime.MinValue.AddMinutes(1); // want this to be greater than mindate so new items will refresh + } + + public virtual Task FetchIfNeededAsync(BaseEntity item) + { + if (this.NeedsRefresh(item)) + return FetchAsync(item, item.ResolveArgs); + else + return new Task(() => { }); } public abstract Task FetchAsync(BaseEntity item, ItemResolveEventArgs args); diff --git a/MediaBrowser.Controller/Resolvers/EntityResolutionHelper.cs b/MediaBrowser.Controller/Resolvers/EntityResolutionHelper.cs index 76541e0efe..ab2f07ecfd 100644 --- a/MediaBrowser.Controller/Resolvers/EntityResolutionHelper.cs +++ b/MediaBrowser.Controller/Resolvers/EntityResolutionHelper.cs @@ -18,7 +18,6 @@ namespace MediaBrowser.Controller.Resolvers public static List IgnoreFolders = new List() { "trailers", - "metadata", "bdmv", "certificate", "backup", @@ -61,6 +60,11 @@ namespace MediaBrowser.Controller.Resolvers // Ignore any folders containing a file called .ignore resolve = false; } + else if (args.FileInfo.cFileName.Equals("metadata", StringComparison.OrdinalIgnoreCase)) + { + // Ignore metadata folders + resolve = false; + } return resolve; } -- cgit v1.2.3 From da618f13e23cbef6ca71c8c0099dfd563a394fa5 Mon Sep 17 00:00:00 2001 From: ebr11 Eric Reed spam Date: Thu, 20 Sep 2012 13:53:10 -0400 Subject: Fix up directory watchers --- MediaBrowser.Common/Extensions/BaseExtensions.cs | 6 ++++-- MediaBrowser.Controller/IO/DirectoryWatchers.cs | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'MediaBrowser.Common/Extensions/BaseExtensions.cs') diff --git a/MediaBrowser.Common/Extensions/BaseExtensions.cs b/MediaBrowser.Common/Extensions/BaseExtensions.cs index 10ff019aa4..77eb9fbb4a 100644 --- a/MediaBrowser.Common/Extensions/BaseExtensions.cs +++ b/MediaBrowser.Common/Extensions/BaseExtensions.cs @@ -28,11 +28,13 @@ namespace MediaBrowser.Common.Extensions /// public static bool ContainsParentFolder(this List lst, string path) { + path = path.TrimEnd('\\'); foreach (var str in lst) { //this should be a little quicker than examining each actual parent folder... - if (path.Equals(str,StringComparison.OrdinalIgnoreCase) - || (path.StartsWith(str, StringComparison.OrdinalIgnoreCase) && path[str.Length-1] == '\\')) return true; + var compare = str.TrimEnd('\\'); + if (path.Equals(compare,StringComparison.OrdinalIgnoreCase) + || (path.StartsWith(compare, StringComparison.OrdinalIgnoreCase) && path[compare.Length] == '\\')) return true; } return false; } diff --git a/MediaBrowser.Controller/IO/DirectoryWatchers.cs b/MediaBrowser.Controller/IO/DirectoryWatchers.cs index 2e33400892..eb1358e16f 100644 --- a/MediaBrowser.Controller/IO/DirectoryWatchers.cs +++ b/MediaBrowser.Controller/IO/DirectoryWatchers.cs @@ -62,11 +62,9 @@ namespace MediaBrowser.Controller.IO Logger.LogDebugInfo("****** Watcher sees change of type " + e.ChangeType.ToString() + " to " + e.FullPath); lock (affectedPaths) { - if (!affectedPaths.Contains(e.FullPath)) - { - Logger.LogDebugInfo("****** Adding " + e.FullPath + " to affected paths."); - affectedPaths.Add(e.FullPath); - } + //Since we're watching created, deleted and renamed we always want the parent of the item to be the affected path + var affectedPath = Path.GetDirectoryName(e.FullPath); + if (e.ChangeType == WatcherChangeTypes.Renamed) { var renamedArgs = e as RenamedEventArgs; @@ -76,6 +74,15 @@ namespace MediaBrowser.Controller.IO affectedPaths.Remove(renamedArgs.OldFullPath); } } + + //If anything underneath this path was already marked as affected - remove it as it will now get captured by this one + affectedPaths.RemoveAll(p => p.StartsWith(e.FullPath, StringComparison.OrdinalIgnoreCase)); + + if (!affectedPaths.ContainsParentFolder(affectedPath)) + { + Logger.LogDebugInfo("****** Adding " + affectedPath + " to affected paths."); + affectedPaths.Add(affectedPath); + } } if (updateTimer == null) @@ -125,7 +132,7 @@ namespace MediaBrowser.Controller.IO } foreach (var p in paths) Logger.LogDebugInfo("********* "+ p + " reports change."); - foreach (var i in itemsToRefresh) Logger.LogDebugInfo("********* "+i.Name + " will be refreshed."); + foreach (var i in itemsToRefresh) Logger.LogDebugInfo("********* "+i.Name + " ("+ i.Path + ") will be refreshed."); return Task.WhenAll(itemsToRefresh.Select(i => i.ChangedExternally())); } -- cgit v1.2.3