diff options
Diffstat (limited to 'MediaBrowser.Server.Implementations')
6 files changed, 1067 insertions, 5 deletions
diff --git a/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs b/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs new file mode 100644 index 000000000..c1371a32b --- /dev/null +++ b/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs @@ -0,0 +1,543 @@ +using MediaBrowser.Common.ScheduledTasks; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.ScheduledTasks; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.IO +{ + /// <summary> + /// Class DirectoryWatchers + /// </summary> + public class DirectoryWatchers : IDirectoryWatchers + { + /// <summary> + /// The file system watchers + /// </summary> + private ConcurrentBag<FileSystemWatcher> _fileSystemWatchers = new ConcurrentBag<FileSystemWatcher>(); + /// <summary> + /// The update timer + /// </summary> + private Timer _updateTimer; + /// <summary> + /// The affected paths + /// </summary> + private readonly ConcurrentDictionary<string, string> _affectedPaths = new ConcurrentDictionary<string, string>(); + + /// <summary> + /// A dynamic list of paths that should be ignored. Added to during our own file sytem modifications. + /// </summary> + private readonly ConcurrentDictionary<string,string> _tempIgnoredPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase); + + /// <summary> + /// The timer lock + /// </summary> + private readonly object _timerLock = new object(); + + /// <summary> + /// Add the path to our temporary ignore list. Use when writing to a path within our listening scope. + /// </summary> + /// <param name="path">The path.</param> + public void TemporarilyIgnore(string path) + { + _tempIgnoredPaths[path] = path; + } + + /// <summary> + /// Removes the temp ignore. + /// </summary> + /// <param name="path">The path.</param> + public void RemoveTempIgnore(string path) + { + string val; + _tempIgnoredPaths.TryRemove(path, out val); + } + + /// <summary> + /// Gets or sets the logger. + /// </summary> + /// <value>The logger.</value> + private ILogger Logger { get; set; } + + /// <summary> + /// Gets or sets the task manager. + /// </summary> + /// <value>The task manager.</value> + private ITaskManager TaskManager { get; set; } + + private ILibraryManager LibraryManager { get; set; } + private IServerConfigurationManager ConfigurationManager { get; set; } + + /// <summary> + /// Initializes a new instance of the <see cref="DirectoryWatchers" /> class. + /// </summary> + public DirectoryWatchers(ILogManager logManager, ITaskManager taskManager, ILibraryManager libraryManager, IServerConfigurationManager configurationManager) + { + if (taskManager == null) + { + throw new ArgumentNullException("taskManager"); + } + + LibraryManager = libraryManager; + TaskManager = taskManager; + Logger = logManager.GetLogger("DirectoryWatchers"); + ConfigurationManager = configurationManager; + } + + /// <summary> + /// Starts this instance. + /// </summary> + public void Start() + { + LibraryManager.LibraryChanged += Instance_LibraryChanged; + + var pathsToWatch = new List<string> { LibraryManager.RootFolder.Path }; + + var paths = LibraryManager.RootFolder.Children.OfType<Folder>() + .SelectMany(f => + { + try + { + // Accessing ResolveArgs could involve file system access + return f.ResolveArgs.PhysicalLocations; + } + catch (IOException) + { + return new string[] {}; + } + + }) + .Where(Path.IsPathRooted); + + foreach (var path in paths) + { + if (!ContainsParentFolder(pathsToWatch, path)) + { + pathsToWatch.Add(path); + } + } + + foreach (var path in pathsToWatch) + { + StartWatchingPath(path); + } + } + + /// <summary> + /// Examine a list of strings assumed to be file paths to see if it contains a parent of + /// the provided path. + /// </summary> + /// <param name="lst">The LST.</param> + /// <param name="path">The path.</param> + /// <returns><c>true</c> if [contains parent folder] [the specified LST]; otherwise, <c>false</c>.</returns> + /// <exception cref="System.ArgumentNullException">path</exception> + private static bool ContainsParentFolder(IEnumerable<string> lst, string path) + { + if (string.IsNullOrEmpty(path)) + { + throw new ArgumentNullException("path"); + } + + path = path.TrimEnd(Path.DirectorySeparatorChar); + + return lst.Any(str => + { + //this should be a little quicker than examining each actual parent folder... + var compare = str.TrimEnd(Path.DirectorySeparatorChar); + + return (path.Equals(compare, StringComparison.OrdinalIgnoreCase) || (path.StartsWith(compare, StringComparison.OrdinalIgnoreCase) && path[compare.Length] == Path.DirectorySeparatorChar)); + }); + } + + /// <summary> + /// Starts the watching path. + /// </summary> + /// <param name="path">The path.</param> + private void StartWatchingPath(string path) + { + // Creating a FileSystemWatcher over the LAN can take hundreds of milliseconds, so wrap it in a Task to do them all in parallel + Task.Run(() => + { + var newWatcher = new FileSystemWatcher(path, "*") { IncludeSubdirectories = true, InternalBufferSize = 32767 }; + + newWatcher.Created += watcher_Changed; + newWatcher.Deleted += watcher_Changed; + newWatcher.Renamed += watcher_Changed; + newWatcher.Changed += watcher_Changed; + + newWatcher.Error += watcher_Error; + + try + { + newWatcher.EnableRaisingEvents = true; + _fileSystemWatchers.Add(newWatcher); + + Logger.Info("Watching directory " + path); + } + catch (IOException ex) + { + Logger.ErrorException("Error watching path: {0}", ex, path); + } + catch (PlatformNotSupportedException ex) + { + Logger.ErrorException("Error watching path: {0}", ex, path); + } + }); + } + + /// <summary> + /// Stops the watching path. + /// </summary> + /// <param name="path">The path.</param> + private void StopWatchingPath(string path) + { + var watcher = _fileSystemWatchers.FirstOrDefault(f => f.Path.Equals(path, StringComparison.OrdinalIgnoreCase)); + + if (watcher != null) + { + DisposeWatcher(watcher); + } + } + + /// <summary> + /// Disposes the watcher. + /// </summary> + /// <param name="watcher">The watcher.</param> + private void DisposeWatcher(FileSystemWatcher watcher) + { + Logger.Info("Stopping directory watching for path {0}", watcher.Path); + + watcher.EnableRaisingEvents = false; + watcher.Dispose(); + + var watchers = _fileSystemWatchers.ToList(); + + watchers.Remove(watcher); + + _fileSystemWatchers = new ConcurrentBag<FileSystemWatcher>(watchers); + } + + /// <summary> + /// Handles the LibraryChanged event of the Kernel + /// </summary> + /// <param name="sender">The source of the event.</param> + /// <param name="e">The <see cref="MediaBrowser.Controller.Library.ChildrenChangedEventArgs" /> instance containing the event data.</param> + void Instance_LibraryChanged(object sender, ChildrenChangedEventArgs e) + { + if (e.Folder is AggregateFolder && e.HasAddOrRemoveChange) + { + if (e.ItemsRemoved != null) + { + foreach (var item in e.ItemsRemoved.OfType<Folder>()) + { + StopWatchingPath(item.Path); + } + } + if (e.ItemsAdded != null) + { + foreach (var item in e.ItemsAdded.OfType<Folder>()) + { + StartWatchingPath(item.Path); + } + } + } + } + + /// <summary> + /// Handles the Error event of the watcher control. + /// </summary> + /// <param name="sender">The source of the event.</param> + /// <param name="e">The <see cref="ErrorEventArgs" /> instance containing the event data.</param> + async void watcher_Error(object sender, ErrorEventArgs e) + { + var ex = e.GetException(); + var dw = (FileSystemWatcher) sender; + + Logger.ErrorException("Error in Directory watcher for: "+dw.Path, ex); + + if (ex.Message.Contains("network name is no longer available")) + { + //Network either dropped or, we are coming out of sleep and it hasn't reconnected yet - wait and retry + Logger.Warn("Network connection lost - will retry..."); + var retries = 0; + var success = false; + while (!success && retries < 10) + { + await Task.Delay(500).ConfigureAwait(false); + + try + { + dw.EnableRaisingEvents = false; + dw.EnableRaisingEvents = true; + success = true; + } + catch (IOException) + { + Logger.Warn("Network still unavailable..."); + retries++; + } + } + if (!success) + { + Logger.Warn("Unable to access network. Giving up."); + DisposeWatcher(dw); + } + + } + else + { + if (!ex.Message.Contains("BIOS command limit")) + { + Logger.Info("Attempting to re-start watcher."); + + dw.EnableRaisingEvents = false; + dw.EnableRaisingEvents = true; + } + + } + } + + /// <summary> + /// Handles the Changed event of the watcher control. + /// </summary> + /// <param name="sender">The source of the event.</param> + /// <param name="e">The <see cref="FileSystemEventArgs" /> instance containing the event data.</param> + void watcher_Changed(object sender, FileSystemEventArgs e) + { + if (e.ChangeType == WatcherChangeTypes.Created && e.Name == "New folder") + { + return; + } + if (_tempIgnoredPaths.ContainsKey(e.FullPath)) + { + Logger.Info("Watcher requested to ignore change to " + e.FullPath); + return; + } + + Logger.Info("Watcher sees change of type " + e.ChangeType.ToString() + " to " + e.FullPath); + + //Since we're watching created, deleted and renamed we always want the parent of the item to be the affected path + var affectedPath = e.FullPath; + + _affectedPaths.AddOrUpdate(affectedPath, affectedPath, (key, oldValue) => affectedPath); + + lock (_timerLock) + { + if (_updateTimer == null) + { + _updateTimer = new Timer(TimerStopped, null, TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1)); + } + else + { + _updateTimer.Change(TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1)); + } + } + } + + /// <summary> + /// Timers the stopped. + /// </summary> + /// <param name="stateInfo">The state info.</param> + private async void TimerStopped(object stateInfo) + { + lock (_timerLock) + { + // Extend the timer as long as any of the paths are still being written to. + if (_affectedPaths.Any(p => IsFileLocked(p.Key))) + { + Logger.Info("Timer extended."); + _updateTimer.Change(TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1)); + return; + } + + Logger.Info("Timer stopped."); + + _updateTimer.Dispose(); + _updateTimer = null; + } + + var paths = _affectedPaths.Keys.ToList(); + _affectedPaths.Clear(); + + await ProcessPathChanges(paths).ConfigureAwait(false); + } + + /// <summary> + /// Try and determine if a file is locked + /// This is not perfect, and is subject to race conditions, so I'd rather not make this a re-usable library method. + /// </summary> + /// <param name="path">The path.</param> + /// <returns><c>true</c> if [is file locked] [the specified path]; otherwise, <c>false</c>.</returns> + private bool IsFileLocked(string path) + { + try + { + var data = FileSystem.GetFileData(path); + + if (!data.HasValue || data.Value.IsDirectory) + { + return false; + } + } + catch (IOException) + { + return false; + } + + FileStream stream = null; + + try + { + stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); + } + catch + { + //the file is unavailable because it is: + //still being written to + //or being processed by another thread + //or does not exist (has already been processed) + return true; + } + finally + { + if (stream != null) + stream.Close(); + } + + //file is not locked + return false; + } + + /// <summary> + /// Processes the path changes. + /// </summary> + /// <param name="paths">The paths.</param> + /// <returns>Task.</returns> + private async Task ProcessPathChanges(List<string> paths) + { + var itemsToRefresh = paths.Select(Path.GetDirectoryName) + .Select(GetAffectedBaseItem) + .Where(item => item != null) + .Distinct() + .ToList(); + + foreach (var p in paths) Logger.Info(p + " reports change."); + + // If the root folder changed, run the library task so the user can see it + if (itemsToRefresh.Any(i => i is AggregateFolder)) + { + TaskManager.CancelIfRunningAndQueue<RefreshMediaLibraryTask>(); + return; + } + + await Task.WhenAll(itemsToRefresh.Select(i => Task.Run(async () => + { + Logger.Info(i.Name + " (" + i.Path + ") will be refreshed."); + + try + { + await i.ChangedExternally().ConfigureAwait(false); + } + catch (IOException ex) + { + // For now swallow and log. + // Research item: If an IOException occurs, the item may be in a disconnected state (media unavailable) + // Should we remove it from it's parent? + Logger.ErrorException("Error refreshing {0}", ex, i.Name); + } + + }))).ConfigureAwait(false); + } + + /// <summary> + /// Gets the affected base item. + /// </summary> + /// <param name="path">The path.</param> + /// <returns>BaseItem.</returns> + private BaseItem GetAffectedBaseItem(string path) + { + BaseItem item = null; + + while (item == null && !string.IsNullOrEmpty(path)) + { + item = LibraryManager.RootFolder.FindByPath(path); + + path = Path.GetDirectoryName(path); + } + + if (item != null) + { + // If the item has been deleted find the first valid parent that still exists + while (!Directory.Exists(item.Path) && !File.Exists(item.Path)) + { + item = item.Parent; + + if (item == null) + { + break; + } + } + } + + return item; + } + + /// <summary> + /// Stops this instance. + /// </summary> + public void Stop() + { + LibraryManager.LibraryChanged -= Instance_LibraryChanged; + + FileSystemWatcher watcher; + + while (_fileSystemWatchers.TryTake(out watcher)) + { + watcher.Changed -= watcher_Changed; + watcher.EnableRaisingEvents = false; + watcher.Dispose(); + } + + lock (_timerLock) + { + if (_updateTimer != null) + { + _updateTimer.Dispose(); + _updateTimer = null; + } + } + + _affectedPaths.Clear(); + } + + /// <summary> + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// </summary> + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// <summary> + /// Releases unmanaged and - optionally - managed resources. + /// </summary> + /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param> + protected virtual void Dispose(bool dispose) + { + if (dispose) + { + Stop(); + } + } + } +} diff --git a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj index e56c79ded..781088672 100644 --- a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj +++ b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj @@ -111,6 +111,7 @@ <Compile Include="HttpServer\ServerFactory.cs" /> <Compile Include="HttpServer\StreamWriter.cs" /> <Compile Include="HttpServer\SwaggerService.cs" /> + <Compile Include="IO\DirectoryWatchers.cs" /> <Compile Include="Library\CoreResolutionIgnoreRule.cs" /> <Compile Include="Library\LibraryManager.cs" /> <Compile Include="Library\ResolverHelper.cs" /> @@ -128,6 +129,7 @@ <Compile Include="Library\Resolvers\VideoResolver.cs" /> <Compile Include="Library\UserManager.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="Providers\ProviderManager.cs" /> <Compile Include="Reflection\TypeMapper.cs" /> <Compile Include="ScheduledTasks\PeopleValidationTask.cs" /> <Compile Include="ScheduledTasks\ChapterImagesTask.cs" /> diff --git a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs new file mode 100644 index 000000000..5accc06d7 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs @@ -0,0 +1,487 @@ +using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.IO; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Providers +{ + /// <summary> + /// Class ProviderManager + /// </summary> + public class ProviderManager : IProviderManager + { + /// <summary> + /// The remote image cache + /// </summary> + private readonly FileSystemRepository _remoteImageCache; + + /// <summary> + /// The currently running metadata providers + /// </summary> + private readonly ConcurrentDictionary<string, Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource>> _currentlyRunningProviders = + new ConcurrentDictionary<string, Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource>>(); + + /// <summary> + /// The _logger + /// </summary> + private readonly ILogger _logger; + + /// <summary> + /// The _HTTP client + /// </summary> + private readonly IHttpClient _httpClient; + + /// <summary> + /// The _directory watchers + /// </summary> + private readonly IDirectoryWatchers _directoryWatchers; + + /// <summary> + /// Gets or sets the configuration manager. + /// </summary> + /// <value>The configuration manager.</value> + private IServerConfigurationManager ConfigurationManager { get; set; } + + /// <summary> + /// Gets the list of currently registered metadata prvoiders + /// </summary> + /// <value>The metadata providers enumerable.</value> + private BaseMetadataProvider[] MetadataProviders { get; set; } + + /// <summary> + /// Initializes a new instance of the <see cref="ProviderManager" /> class. + /// </summary> + /// <param name="httpClient">The HTTP client.</param> + /// <param name="configurationManager">The configuration manager.</param> + /// <param name="directoryWatchers">The directory watchers.</param> + /// <param name="logManager">The log manager.</param> + public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager) + { + _logger = logManager.GetLogger("ProviderManager"); + _httpClient = httpClient; + ConfigurationManager = configurationManager; + _directoryWatchers = directoryWatchers; + _remoteImageCache = new FileSystemRepository(configurationManager.ApplicationPaths.DownloadedImagesDataPath); + + configurationManager.ConfigurationUpdated += configurationManager_ConfigurationUpdated; + } + + /// <summary> + /// Handles the ConfigurationUpdated event of the configurationManager control. + /// </summary> + /// <param name="sender">The source of the event.</param> + /// <param name="e">The <see cref="EventArgs" /> instance containing the event data.</param> + void configurationManager_ConfigurationUpdated(object sender, EventArgs e) + { + // Validate currently executing providers, in the background + Task.Run(() => + { + ValidateCurrentlyRunningProviders(); + }); + } + + /// <summary> + /// Gets or sets the supported providers key. + /// </summary> + /// <value>The supported providers key.</value> + private Guid SupportedProvidersKey { get; set; } + + /// <summary> + /// Adds the metadata providers. + /// </summary> + /// <param name="providers">The providers.</param> + public void AddMetadataProviders(IEnumerable<BaseMetadataProvider> providers) + { + MetadataProviders = providers.ToArray(); + } + + /// <summary> + /// Runs all metadata providers for an entity, and returns true or false indicating if at least one was refreshed and requires persistence + /// </summary> + /// <param name="item">The item.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <param name="force">if set to <c>true</c> [force].</param> + /// <param name="allowSlowProviders">if set to <c>true</c> [allow slow providers].</param> + /// <returns>Task{System.Boolean}.</returns> + public async Task<bool> ExecuteMetadataProviders(BaseItem item, CancellationToken cancellationToken, bool force = false, bool allowSlowProviders = true) + { + // Allow providers of the same priority to execute in parallel + MetadataProviderPriority? currentPriority = null; + var currentTasks = new List<Task<bool>>(); + + var result = false; + + cancellationToken.ThrowIfCancellationRequested(); + + // Determine if supported providers have changed + var supportedProviders = MetadataProviders.Where(p => p.Supports(item)).ToList(); + + BaseProviderInfo supportedProvidersInfo; + + if (SupportedProvidersKey == Guid.Empty) + { + SupportedProvidersKey = "SupportedProviders".GetMD5(); + } + + var supportedProvidersHash = string.Join("+", supportedProviders.Select(i => i.GetType().Name)).GetMD5(); + bool providersChanged; + + item.ProviderData.TryGetValue(SupportedProvidersKey, out supportedProvidersInfo); + if (supportedProvidersInfo == null) + { + // First time + supportedProvidersInfo = new BaseProviderInfo { ProviderId = SupportedProvidersKey, FileSystemStamp = supportedProvidersHash }; + providersChanged = force = true; + } + else + { + // Force refresh if the supported providers have changed + providersChanged = force = force || supportedProvidersInfo.FileSystemStamp != supportedProvidersHash; + } + + // If providers have changed, clear provider info and update the supported providers hash + if (providersChanged) + { + _logger.Debug("Providers changed for {0}. Clearing and forcing refresh.", item.Name); + item.ProviderData.Clear(); + supportedProvidersInfo.FileSystemStamp = supportedProvidersHash; + } + + if (force) item.ClearMetaValues(); + + // Run the normal providers sequentially in order of priority + foreach (var provider in supportedProviders) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Skip if internet providers are currently disabled + if (provider.RequiresInternet && !ConfigurationManager.Configuration.EnableInternetProviders) + { + continue; + } + + // Skip if is slow and we aren't allowing slow ones + if (provider.IsSlow && !allowSlowProviders) + { + continue; + } + + // Skip if internet provider and this type is not allowed + if (provider.RequiresInternet && ConfigurationManager.Configuration.EnableInternetProviders && ConfigurationManager.Configuration.InternetProviderExcludeTypes.Contains(item.GetType().Name, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + // When a new priority is reached, await the ones that are currently running and clear the list + if (currentPriority.HasValue && currentPriority.Value != provider.Priority && currentTasks.Count > 0) + { + var results = await Task.WhenAll(currentTasks).ConfigureAwait(false); + result |= results.Contains(true); + + currentTasks.Clear(); + } + + // Put this check below the await because the needs refresh of the next tier of providers may depend on the previous ones running + // This is the case for the fan art provider which depends on the movie and tv providers having run before them + if (!force && !provider.NeedsRefresh(item)) + { + continue; + } + + currentTasks.Add(FetchAsync(provider, item, force, cancellationToken)); + currentPriority = provider.Priority; + } + + if (currentTasks.Count > 0) + { + var results = await Task.WhenAll(currentTasks).ConfigureAwait(false); + result |= results.Contains(true); + } + + if (providersChanged) + { + item.ProviderData[SupportedProvidersKey] = supportedProvidersInfo; + } + + return result || providersChanged; + } + + /// <summary> + /// Fetches metadata and returns true or false indicating if any work that requires persistence was done + /// </summary> + /// <param name="provider">The provider.</param> + /// <param name="item">The item.</param> + /// <param name="force">if set to <c>true</c> [force].</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task{System.Boolean}.</returns> + /// <exception cref="System.ArgumentNullException"></exception> + private async Task<bool> FetchAsync(BaseMetadataProvider provider, BaseItem item, bool force, CancellationToken cancellationToken) + { + if (item == null) + { + throw new ArgumentNullException(); + } + + cancellationToken.ThrowIfCancellationRequested(); + + _logger.Info("Running {0} for {1}", provider.GetType().Name, item.Path ?? item.Name ?? "--Unknown--"); + + // This provides the ability to cancel just this one provider + var innerCancellationTokenSource = new CancellationTokenSource(); + + OnProviderRefreshBeginning(provider, item, innerCancellationTokenSource); + + try + { + return await provider.FetchAsync(item, force, CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, innerCancellationTokenSource.Token).Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _logger.Info("{0} cancelled for {1}", provider.GetType().Name, item.Name); + + // If the outer cancellation token is the one that caused the cancellation, throw it + if (cancellationToken.IsCancellationRequested && ex.CancellationToken == cancellationToken) + { + throw; + } + + return false; + } + catch (Exception ex) + { + _logger.ErrorException("{0} failed refreshing {1}", ex, provider.GetType().Name, item.Name); + + provider.SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.Failure); + return true; + } + finally + { + innerCancellationTokenSource.Dispose(); + + OnProviderRefreshCompleted(provider, item); + } + } + + /// <summary> + /// Notifies the kernal that a provider has begun refreshing + /// </summary> + /// <param name="provider">The provider.</param> + /// <param name="item">The item.</param> + /// <param name="cancellationTokenSource">The cancellation token source.</param> + public void OnProviderRefreshBeginning(BaseMetadataProvider provider, BaseItem item, CancellationTokenSource cancellationTokenSource) + { + var key = item.Id + provider.GetType().Name; + + Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource> current; + + if (_currentlyRunningProviders.TryGetValue(key, out current)) + { + try + { + current.Item3.Cancel(); + } + catch (ObjectDisposedException) + { + + } + } + + var tuple = new Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource>(provider, item, cancellationTokenSource); + + _currentlyRunningProviders.AddOrUpdate(key, tuple, (k, v) => tuple); + } + + /// <summary> + /// Notifies the kernal that a provider has completed refreshing + /// </summary> + /// <param name="provider">The provider.</param> + /// <param name="item">The item.</param> + public void OnProviderRefreshCompleted(BaseMetadataProvider provider, BaseItem item) + { + var key = item.Id + provider.GetType().Name; + + Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource> current; + + if (_currentlyRunningProviders.TryRemove(key, out current)) + { + current.Item3.Dispose(); + } + } + + /// <summary> + /// Validates the currently running providers and cancels any that should not be run due to configuration changes + /// </summary> + private void ValidateCurrentlyRunningProviders() + { + _logger.Info("Validing currently running providers"); + + var enableInternetProviders = ConfigurationManager.Configuration.EnableInternetProviders; + var internetProviderExcludeTypes = ConfigurationManager.Configuration.InternetProviderExcludeTypes; + + foreach (var tuple in _currentlyRunningProviders.Values + .Where(p => p.Item1.RequiresInternet && (!enableInternetProviders || internetProviderExcludeTypes.Contains(p.Item2.GetType().Name, StringComparer.OrdinalIgnoreCase))) + .ToList()) + { + tuple.Item3.Cancel(); + } + } + + /// <summary> + /// Downloads the and save image. + /// </summary> + /// <param name="item">The item.</param> + /// <param name="source">The source.</param> + /// <param name="targetName">Name of the target.</param> + /// <param name="resourcePool">The resource pool.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task{System.String}.</returns> + /// <exception cref="System.ArgumentNullException">item</exception> + public async Task<string> DownloadAndSaveImage(BaseItem item, string source, string targetName, SemaphoreSlim resourcePool, CancellationToken cancellationToken) + { + if (item == null) + { + throw new ArgumentNullException("item"); + } + if (string.IsNullOrEmpty(source)) + { + throw new ArgumentNullException("source"); + } + if (string.IsNullOrEmpty(targetName)) + { + throw new ArgumentNullException("targetName"); + } + if (resourcePool == null) + { + throw new ArgumentNullException("resourcePool"); + } + + //download and save locally + var localPath = ConfigurationManager.Configuration.SaveLocalMeta ? + Path.Combine(item.MetaLocation, targetName) : + _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName); + + var img = await _httpClient.GetMemoryStream(source, resourcePool, cancellationToken).ConfigureAwait(false); + + if (ConfigurationManager.Configuration.SaveLocalMeta) // queue to media directories + { + await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false); + } + else + { + // we can write directly here because it won't affect the watchers + + try + { + using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + _logger.ErrorException("Error downloading and saving image " + localPath, e); + throw; + } + finally + { + img.Dispose(); + } + + } + return localPath; + } + + + /// <summary> + /// Saves to library filesystem. + /// </summary> + /// <param name="item">The item.</param> + /// <param name="path">The path.</param> + /// <param name="dataToSave">The data to save.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task.</returns> + /// <exception cref="System.ArgumentNullException"></exception> + public async Task SaveToLibraryFilesystem(BaseItem item, string path, Stream dataToSave, CancellationToken cancellationToken) + { + if (item == null) + { + throw new ArgumentNullException(); + } + if (string.IsNullOrEmpty(path)) + { + throw new ArgumentNullException(); + } + if (dataToSave == null) + { + throw new ArgumentNullException(); + } + if (cancellationToken == null) + { + throw new ArgumentNullException(); + } + + cancellationToken.ThrowIfCancellationRequested(); + + //Tell the watchers to ignore + _directoryWatchers.TemporarilyIgnore(path); + + //Make the mod + + dataToSave.Position = 0; + + try + { + using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await dataToSave.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); + + dataToSave.Dispose(); + + // If this is ever used for something other than metadata we can add a file type param + item.ResolveArgs.AddMetadataFile(path); + } + } + finally + { + //Remove the ignore + _directoryWatchers.RemoveTempIgnore(path); + } + } + + /// <summary> + /// Releases unmanaged and - optionally - managed resources. + /// </summary> + /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param> + protected virtual void Dispose(bool dispose) + { + if (dispose) + { + _remoteImageCache.Dispose(); + } + } + + /// <summary> + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// </summary> + public void Dispose() + { + Dispose(true); + } + } +} diff --git a/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs b/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs index 8306e094b..b20630edb 100644 --- a/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs +++ b/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs @@ -27,17 +27,21 @@ namespace MediaBrowser.Server.Implementations.ScheduledTasks /// </summary> private readonly ILogger _logger; private readonly ILibraryManager _libraryManager; + private readonly IServerApplicationPaths _appPaths; /// <summary> /// Initializes a new instance of the <see cref="ImageCleanupTask" /> class. /// </summary> /// <param name="kernel">The kernel.</param> /// <param name="logger">The logger.</param> - public ImageCleanupTask(Kernel kernel, ILogger logger, ILibraryManager libraryManager) + /// <param name="libraryManager">The library manager.</param> + /// <param name="appPaths">The app paths.</param> + public ImageCleanupTask(Kernel kernel, ILogger logger, ILibraryManager libraryManager, IServerApplicationPaths appPaths) { _kernel = kernel; _logger = logger; _libraryManager = libraryManager; + _appPaths = appPaths; } /// <summary> @@ -65,7 +69,7 @@ namespace MediaBrowser.Server.Implementations.ScheduledTasks // First gather all image files var files = GetFiles(_kernel.FFMpegManager.AudioImagesDataPath) .Concat(GetFiles(_kernel.FFMpegManager.VideoImagesDataPath)) - .Concat(GetFiles(_kernel.ProviderManager.ImagesDataPath)) + .Concat(GetFiles(_appPaths.DownloadedImagesDataPath)) .ToList(); // Now gather all items diff --git a/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs b/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs index ff29badff..429c893ca 100644 --- a/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs +++ b/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs @@ -302,7 +302,7 @@ namespace MediaBrowser.Server.Implementations /// Gets the FF MPEG stream cache path. /// </summary> /// <value>The FF MPEG stream cache path.</value> - public string FFMpegStreamCachePath + public string EncodedMediaCachePath { get { @@ -345,5 +345,31 @@ namespace MediaBrowser.Server.Implementations return _mediaToolsPath; } } + + /// <summary> + /// The _images data path + /// </summary> + private string _downloadedImagesDataPath; + /// <summary> + /// Gets the images data path. + /// </summary> + /// <value>The images data path.</value> + public string DownloadedImagesDataPath + { + get + { + if (_downloadedImagesDataPath == null) + { + _downloadedImagesDataPath = Path.Combine(DataPath, "remote-images"); + + if (!Directory.Exists(_downloadedImagesDataPath)) + { + Directory.CreateDirectory(_downloadedImagesDataPath); + } + } + + return _downloadedImagesDataPath; + } + } } } diff --git a/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs b/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs index e4cb83e96..60628853e 100644 --- a/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs +++ b/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs @@ -104,8 +104,8 @@ namespace MediaBrowser.Server.Implementations.ServerManager /// <value>The web socket listeners.</value> private readonly List<IWebSocketListener> _webSocketListeners = new List<IWebSocketListener>(); - private Kernel _kernel; - + private readonly Kernel _kernel; + /// <summary> /// Initializes a new instance of the <see cref="ServerManager" /> class. /// </summary> |