From 9911df11e8a96d5d7fffcd8618b63a6adb27701f Mon Sep 17 00:00:00 2001 From: LukePulverenti Date: Fri, 8 Mar 2013 00:08:27 -0500 Subject: extracted provider manager. took more off the kernel --- .../IO/DirectoryWatchers.cs | 543 +++++++++++++++++++++ .../MediaBrowser.Server.Implementations.csproj | 2 + .../Providers/ProviderManager.cs | 487 ++++++++++++++++++ .../ScheduledTasks/ImageCleanupTask.cs | 8 +- .../ServerApplicationPaths.cs | 28 +- .../ServerManager/ServerManager.cs | 4 +- 6 files changed, 1067 insertions(+), 5 deletions(-) create mode 100644 MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs create mode 100644 MediaBrowser.Server.Implementations/Providers/ProviderManager.cs (limited to 'MediaBrowser.Server.Implementations') diff --git a/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs b/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs new file mode 100644 index 000000000..c1371a32b --- /dev/null +++ b/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs @@ -0,0 +1,543 @@ +using MediaBrowser.Common.ScheduledTasks; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.ScheduledTasks; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.IO +{ + /// + /// Class DirectoryWatchers + /// + public class DirectoryWatchers : IDirectoryWatchers + { + /// + /// The file system watchers + /// + private ConcurrentBag _fileSystemWatchers = new ConcurrentBag(); + /// + /// The update timer + /// + private Timer _updateTimer; + /// + /// The affected paths + /// + private readonly ConcurrentDictionary _affectedPaths = new ConcurrentDictionary(); + + /// + /// A dynamic list of paths that should be ignored. Added to during our own file sytem modifications. + /// + private readonly ConcurrentDictionary _tempIgnoredPaths = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + + /// + /// The timer lock + /// + private readonly object _timerLock = new object(); + + /// + /// Add the path to our temporary ignore list. Use when writing to a path within our listening scope. + /// + /// The path. + public void TemporarilyIgnore(string path) + { + _tempIgnoredPaths[path] = path; + } + + /// + /// Removes the temp ignore. + /// + /// The path. + public void RemoveTempIgnore(string path) + { + string val; + _tempIgnoredPaths.TryRemove(path, out val); + } + + /// + /// Gets or sets the logger. + /// + /// The logger. + private ILogger Logger { get; set; } + + /// + /// Gets or sets the task manager. + /// + /// The task manager. + private ITaskManager TaskManager { get; set; } + + private ILibraryManager LibraryManager { get; set; } + private IServerConfigurationManager ConfigurationManager { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public DirectoryWatchers(ILogManager logManager, ITaskManager taskManager, ILibraryManager libraryManager, IServerConfigurationManager configurationManager) + { + if (taskManager == null) + { + throw new ArgumentNullException("taskManager"); + } + + LibraryManager = libraryManager; + TaskManager = taskManager; + Logger = logManager.GetLogger("DirectoryWatchers"); + ConfigurationManager = configurationManager; + } + + /// + /// Starts this instance. + /// + public void Start() + { + LibraryManager.LibraryChanged += Instance_LibraryChanged; + + var pathsToWatch = new List { LibraryManager.RootFolder.Path }; + + var paths = LibraryManager.RootFolder.Children.OfType() + .SelectMany(f => + { + try + { + // Accessing ResolveArgs could involve file system access + return f.ResolveArgs.PhysicalLocations; + } + catch (IOException) + { + return new string[] {}; + } + + }) + .Where(Path.IsPathRooted); + + foreach (var path in paths) + { + if (!ContainsParentFolder(pathsToWatch, path)) + { + pathsToWatch.Add(path); + } + } + + foreach (var path in pathsToWatch) + { + StartWatchingPath(path); + } + } + + /// + /// Examine a list of strings assumed to be file paths to see if it contains a parent of + /// the provided path. + /// + /// The LST. + /// The path. + /// true if [contains parent folder] [the specified LST]; otherwise, false. + /// path + private static bool ContainsParentFolder(IEnumerable lst, string path) + { + if (string.IsNullOrEmpty(path)) + { + throw new ArgumentNullException("path"); + } + + path = path.TrimEnd(Path.DirectorySeparatorChar); + + return lst.Any(str => + { + //this should be a little quicker than examining each actual parent folder... + var compare = str.TrimEnd(Path.DirectorySeparatorChar); + + return (path.Equals(compare, StringComparison.OrdinalIgnoreCase) || (path.StartsWith(compare, StringComparison.OrdinalIgnoreCase) && path[compare.Length] == Path.DirectorySeparatorChar)); + }); + } + + /// + /// Starts the watching path. + /// + /// The path. + private void StartWatchingPath(string path) + { + // Creating a FileSystemWatcher over the LAN can take hundreds of milliseconds, so wrap it in a Task to do them all in parallel + Task.Run(() => + { + var newWatcher = new FileSystemWatcher(path, "*") { IncludeSubdirectories = true, InternalBufferSize = 32767 }; + + newWatcher.Created += watcher_Changed; + newWatcher.Deleted += watcher_Changed; + newWatcher.Renamed += watcher_Changed; + newWatcher.Changed += watcher_Changed; + + newWatcher.Error += watcher_Error; + + try + { + newWatcher.EnableRaisingEvents = true; + _fileSystemWatchers.Add(newWatcher); + + Logger.Info("Watching directory " + path); + } + catch (IOException ex) + { + Logger.ErrorException("Error watching path: {0}", ex, path); + } + catch (PlatformNotSupportedException ex) + { + Logger.ErrorException("Error watching path: {0}", ex, path); + } + }); + } + + /// + /// Stops the watching path. + /// + /// The path. + private void StopWatchingPath(string path) + { + var watcher = _fileSystemWatchers.FirstOrDefault(f => f.Path.Equals(path, StringComparison.OrdinalIgnoreCase)); + + if (watcher != null) + { + DisposeWatcher(watcher); + } + } + + /// + /// Disposes the watcher. + /// + /// The watcher. + private void DisposeWatcher(FileSystemWatcher watcher) + { + Logger.Info("Stopping directory watching for path {0}", watcher.Path); + + watcher.EnableRaisingEvents = false; + watcher.Dispose(); + + var watchers = _fileSystemWatchers.ToList(); + + watchers.Remove(watcher); + + _fileSystemWatchers = new ConcurrentBag(watchers); + } + + /// + /// Handles the LibraryChanged event of the Kernel + /// + /// The source of the event. + /// The instance containing the event data. + void Instance_LibraryChanged(object sender, ChildrenChangedEventArgs e) + { + if (e.Folder is AggregateFolder && e.HasAddOrRemoveChange) + { + if (e.ItemsRemoved != null) + { + foreach (var item in e.ItemsRemoved.OfType()) + { + StopWatchingPath(item.Path); + } + } + if (e.ItemsAdded != null) + { + foreach (var item in e.ItemsAdded.OfType()) + { + StartWatchingPath(item.Path); + } + } + } + } + + /// + /// Handles the Error event of the watcher control. + /// + /// The source of the event. + /// The instance containing the event data. + async void watcher_Error(object sender, ErrorEventArgs e) + { + var ex = e.GetException(); + var dw = (FileSystemWatcher) sender; + + Logger.ErrorException("Error in Directory watcher for: "+dw.Path, ex); + + if (ex.Message.Contains("network name is no longer available")) + { + //Network either dropped or, we are coming out of sleep and it hasn't reconnected yet - wait and retry + Logger.Warn("Network connection lost - will retry..."); + var retries = 0; + var success = false; + while (!success && retries < 10) + { + await Task.Delay(500).ConfigureAwait(false); + + try + { + dw.EnableRaisingEvents = false; + dw.EnableRaisingEvents = true; + success = true; + } + catch (IOException) + { + Logger.Warn("Network still unavailable..."); + retries++; + } + } + if (!success) + { + Logger.Warn("Unable to access network. Giving up."); + DisposeWatcher(dw); + } + + } + else + { + if (!ex.Message.Contains("BIOS command limit")) + { + Logger.Info("Attempting to re-start watcher."); + + dw.EnableRaisingEvents = false; + dw.EnableRaisingEvents = true; + } + + } + } + + /// + /// Handles the Changed event of the watcher control. + /// + /// The source of the event. + /// The instance containing the event data. + void watcher_Changed(object sender, FileSystemEventArgs e) + { + if (e.ChangeType == WatcherChangeTypes.Created && e.Name == "New folder") + { + return; + } + if (_tempIgnoredPaths.ContainsKey(e.FullPath)) + { + Logger.Info("Watcher requested to ignore change to " + e.FullPath); + return; + } + + Logger.Info("Watcher sees change of type " + e.ChangeType.ToString() + " to " + e.FullPath); + + //Since we're watching created, deleted and renamed we always want the parent of the item to be the affected path + var affectedPath = e.FullPath; + + _affectedPaths.AddOrUpdate(affectedPath, affectedPath, (key, oldValue) => affectedPath); + + lock (_timerLock) + { + if (_updateTimer == null) + { + _updateTimer = new Timer(TimerStopped, null, TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1)); + } + else + { + _updateTimer.Change(TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1)); + } + } + } + + /// + /// Timers the stopped. + /// + /// The state info. + private async void TimerStopped(object stateInfo) + { + lock (_timerLock) + { + // Extend the timer as long as any of the paths are still being written to. + if (_affectedPaths.Any(p => IsFileLocked(p.Key))) + { + Logger.Info("Timer extended."); + _updateTimer.Change(TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1)); + return; + } + + Logger.Info("Timer stopped."); + + _updateTimer.Dispose(); + _updateTimer = null; + } + + var paths = _affectedPaths.Keys.ToList(); + _affectedPaths.Clear(); + + await ProcessPathChanges(paths).ConfigureAwait(false); + } + + /// + /// Try and determine if a file is locked + /// This is not perfect, and is subject to race conditions, so I'd rather not make this a re-usable library method. + /// + /// The path. + /// true if [is file locked] [the specified path]; otherwise, false. + private bool IsFileLocked(string path) + { + try + { + var data = FileSystem.GetFileData(path); + + if (!data.HasValue || data.Value.IsDirectory) + { + return false; + } + } + catch (IOException) + { + return false; + } + + FileStream stream = null; + + try + { + stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); + } + catch + { + //the file is unavailable because it is: + //still being written to + //or being processed by another thread + //or does not exist (has already been processed) + return true; + } + finally + { + if (stream != null) + stream.Close(); + } + + //file is not locked + return false; + } + + /// + /// Processes the path changes. + /// + /// The paths. + /// Task. + private async Task ProcessPathChanges(List paths) + { + var itemsToRefresh = paths.Select(Path.GetDirectoryName) + .Select(GetAffectedBaseItem) + .Where(item => item != null) + .Distinct() + .ToList(); + + foreach (var p in paths) Logger.Info(p + " reports change."); + + // If the root folder changed, run the library task so the user can see it + if (itemsToRefresh.Any(i => i is AggregateFolder)) + { + TaskManager.CancelIfRunningAndQueue(); + return; + } + + await Task.WhenAll(itemsToRefresh.Select(i => Task.Run(async () => + { + Logger.Info(i.Name + " (" + i.Path + ") will be refreshed."); + + try + { + await i.ChangedExternally().ConfigureAwait(false); + } + catch (IOException ex) + { + // For now swallow and log. + // Research item: If an IOException occurs, the item may be in a disconnected state (media unavailable) + // Should we remove it from it's parent? + Logger.ErrorException("Error refreshing {0}", ex, i.Name); + } + + }))).ConfigureAwait(false); + } + + /// + /// Gets the affected base item. + /// + /// The path. + /// BaseItem. + private BaseItem GetAffectedBaseItem(string path) + { + BaseItem item = null; + + while (item == null && !string.IsNullOrEmpty(path)) + { + item = LibraryManager.RootFolder.FindByPath(path); + + path = Path.GetDirectoryName(path); + } + + if (item != null) + { + // If the item has been deleted find the first valid parent that still exists + while (!Directory.Exists(item.Path) && !File.Exists(item.Path)) + { + item = item.Parent; + + if (item == null) + { + break; + } + } + } + + return item; + } + + /// + /// Stops this instance. + /// + public void Stop() + { + LibraryManager.LibraryChanged -= Instance_LibraryChanged; + + FileSystemWatcher watcher; + + while (_fileSystemWatchers.TryTake(out watcher)) + { + watcher.Changed -= watcher_Changed; + watcher.EnableRaisingEvents = false; + watcher.Dispose(); + } + + lock (_timerLock) + { + if (_updateTimer != null) + { + _updateTimer.Dispose(); + _updateTimer = null; + } + } + + _affectedPaths.Clear(); + } + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Releases unmanaged and - optionally - managed resources. + /// + /// true to release both managed and unmanaged resources; false to release only unmanaged resources. + protected virtual void Dispose(bool dispose) + { + if (dispose) + { + Stop(); + } + } + } +} diff --git a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj index e56c79ded..781088672 100644 --- a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj +++ b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj @@ -111,6 +111,7 @@ + @@ -128,6 +129,7 @@ + diff --git a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs new file mode 100644 index 000000000..5accc06d7 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs @@ -0,0 +1,487 @@ +using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.IO; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Providers +{ + /// + /// Class ProviderManager + /// + public class ProviderManager : IProviderManager + { + /// + /// The remote image cache + /// + private readonly FileSystemRepository _remoteImageCache; + + /// + /// The currently running metadata providers + /// + private readonly ConcurrentDictionary> _currentlyRunningProviders = + new ConcurrentDictionary>(); + + /// + /// The _logger + /// + private readonly ILogger _logger; + + /// + /// The _HTTP client + /// + private readonly IHttpClient _httpClient; + + /// + /// The _directory watchers + /// + private readonly IDirectoryWatchers _directoryWatchers; + + /// + /// Gets or sets the configuration manager. + /// + /// The configuration manager. + private IServerConfigurationManager ConfigurationManager { get; set; } + + /// + /// Gets the list of currently registered metadata prvoiders + /// + /// The metadata providers enumerable. + private BaseMetadataProvider[] MetadataProviders { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// The HTTP client. + /// The configuration manager. + /// The directory watchers. + /// The log manager. + public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager) + { + _logger = logManager.GetLogger("ProviderManager"); + _httpClient = httpClient; + ConfigurationManager = configurationManager; + _directoryWatchers = directoryWatchers; + _remoteImageCache = new FileSystemRepository(configurationManager.ApplicationPaths.DownloadedImagesDataPath); + + configurationManager.ConfigurationUpdated += configurationManager_ConfigurationUpdated; + } + + /// + /// Handles the ConfigurationUpdated event of the configurationManager control. + /// + /// The source of the event. + /// The instance containing the event data. + void configurationManager_ConfigurationUpdated(object sender, EventArgs e) + { + // Validate currently executing providers, in the background + Task.Run(() => + { + ValidateCurrentlyRunningProviders(); + }); + } + + /// + /// Gets or sets the supported providers key. + /// + /// The supported providers key. + private Guid SupportedProvidersKey { get; set; } + + /// + /// Adds the metadata providers. + /// + /// The providers. + public void AddMetadataProviders(IEnumerable providers) + { + MetadataProviders = providers.ToArray(); + } + + /// + /// Runs all metadata providers for an entity, and returns true or false indicating if at least one was refreshed and requires persistence + /// + /// The item. + /// The cancellation token. + /// if set to true [force]. + /// if set to true [allow slow providers]. + /// Task{System.Boolean}. + public async Task ExecuteMetadataProviders(BaseItem item, CancellationToken cancellationToken, bool force = false, bool allowSlowProviders = true) + { + // Allow providers of the same priority to execute in parallel + MetadataProviderPriority? currentPriority = null; + var currentTasks = new List>(); + + var result = false; + + cancellationToken.ThrowIfCancellationRequested(); + + // Determine if supported providers have changed + var supportedProviders = MetadataProviders.Where(p => p.Supports(item)).ToList(); + + BaseProviderInfo supportedProvidersInfo; + + if (SupportedProvidersKey == Guid.Empty) + { + SupportedProvidersKey = "SupportedProviders".GetMD5(); + } + + var supportedProvidersHash = string.Join("+", supportedProviders.Select(i => i.GetType().Name)).GetMD5(); + bool providersChanged; + + item.ProviderData.TryGetValue(SupportedProvidersKey, out supportedProvidersInfo); + if (supportedProvidersInfo == null) + { + // First time + supportedProvidersInfo = new BaseProviderInfo { ProviderId = SupportedProvidersKey, FileSystemStamp = supportedProvidersHash }; + providersChanged = force = true; + } + else + { + // Force refresh if the supported providers have changed + providersChanged = force = force || supportedProvidersInfo.FileSystemStamp != supportedProvidersHash; + } + + // If providers have changed, clear provider info and update the supported providers hash + if (providersChanged) + { + _logger.Debug("Providers changed for {0}. Clearing and forcing refresh.", item.Name); + item.ProviderData.Clear(); + supportedProvidersInfo.FileSystemStamp = supportedProvidersHash; + } + + if (force) item.ClearMetaValues(); + + // Run the normal providers sequentially in order of priority + foreach (var provider in supportedProviders) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Skip if internet providers are currently disabled + if (provider.RequiresInternet && !ConfigurationManager.Configuration.EnableInternetProviders) + { + continue; + } + + // Skip if is slow and we aren't allowing slow ones + if (provider.IsSlow && !allowSlowProviders) + { + continue; + } + + // Skip if internet provider and this type is not allowed + if (provider.RequiresInternet && ConfigurationManager.Configuration.EnableInternetProviders && ConfigurationManager.Configuration.InternetProviderExcludeTypes.Contains(item.GetType().Name, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + // When a new priority is reached, await the ones that are currently running and clear the list + if (currentPriority.HasValue && currentPriority.Value != provider.Priority && currentTasks.Count > 0) + { + var results = await Task.WhenAll(currentTasks).ConfigureAwait(false); + result |= results.Contains(true); + + currentTasks.Clear(); + } + + // Put this check below the await because the needs refresh of the next tier of providers may depend on the previous ones running + // This is the case for the fan art provider which depends on the movie and tv providers having run before them + if (!force && !provider.NeedsRefresh(item)) + { + continue; + } + + currentTasks.Add(FetchAsync(provider, item, force, cancellationToken)); + currentPriority = provider.Priority; + } + + if (currentTasks.Count > 0) + { + var results = await Task.WhenAll(currentTasks).ConfigureAwait(false); + result |= results.Contains(true); + } + + if (providersChanged) + { + item.ProviderData[SupportedProvidersKey] = supportedProvidersInfo; + } + + return result || providersChanged; + } + + /// + /// Fetches metadata and returns true or false indicating if any work that requires persistence was done + /// + /// The provider. + /// The item. + /// if set to true [force]. + /// The cancellation token. + /// Task{System.Boolean}. + /// + private async Task FetchAsync(BaseMetadataProvider provider, BaseItem item, bool force, CancellationToken cancellationToken) + { + if (item == null) + { + throw new ArgumentNullException(); + } + + cancellationToken.ThrowIfCancellationRequested(); + + _logger.Info("Running {0} for {1}", provider.GetType().Name, item.Path ?? item.Name ?? "--Unknown--"); + + // This provides the ability to cancel just this one provider + var innerCancellationTokenSource = new CancellationTokenSource(); + + OnProviderRefreshBeginning(provider, item, innerCancellationTokenSource); + + try + { + return await provider.FetchAsync(item, force, CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, innerCancellationTokenSource.Token).Token).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + _logger.Info("{0} cancelled for {1}", provider.GetType().Name, item.Name); + + // If the outer cancellation token is the one that caused the cancellation, throw it + if (cancellationToken.IsCancellationRequested && ex.CancellationToken == cancellationToken) + { + throw; + } + + return false; + } + catch (Exception ex) + { + _logger.ErrorException("{0} failed refreshing {1}", ex, provider.GetType().Name, item.Name); + + provider.SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.Failure); + return true; + } + finally + { + innerCancellationTokenSource.Dispose(); + + OnProviderRefreshCompleted(provider, item); + } + } + + /// + /// Notifies the kernal that a provider has begun refreshing + /// + /// The provider. + /// The item. + /// The cancellation token source. + public void OnProviderRefreshBeginning(BaseMetadataProvider provider, BaseItem item, CancellationTokenSource cancellationTokenSource) + { + var key = item.Id + provider.GetType().Name; + + Tuple current; + + if (_currentlyRunningProviders.TryGetValue(key, out current)) + { + try + { + current.Item3.Cancel(); + } + catch (ObjectDisposedException) + { + + } + } + + var tuple = new Tuple(provider, item, cancellationTokenSource); + + _currentlyRunningProviders.AddOrUpdate(key, tuple, (k, v) => tuple); + } + + /// + /// Notifies the kernal that a provider has completed refreshing + /// + /// The provider. + /// The item. + public void OnProviderRefreshCompleted(BaseMetadataProvider provider, BaseItem item) + { + var key = item.Id + provider.GetType().Name; + + Tuple current; + + if (_currentlyRunningProviders.TryRemove(key, out current)) + { + current.Item3.Dispose(); + } + } + + /// + /// Validates the currently running providers and cancels any that should not be run due to configuration changes + /// + private void ValidateCurrentlyRunningProviders() + { + _logger.Info("Validing currently running providers"); + + var enableInternetProviders = ConfigurationManager.Configuration.EnableInternetProviders; + var internetProviderExcludeTypes = ConfigurationManager.Configuration.InternetProviderExcludeTypes; + + foreach (var tuple in _currentlyRunningProviders.Values + .Where(p => p.Item1.RequiresInternet && (!enableInternetProviders || internetProviderExcludeTypes.Contains(p.Item2.GetType().Name, StringComparer.OrdinalIgnoreCase))) + .ToList()) + { + tuple.Item3.Cancel(); + } + } + + /// + /// Downloads the and save image. + /// + /// The item. + /// The source. + /// Name of the target. + /// The resource pool. + /// The cancellation token. + /// Task{System.String}. + /// item + public async Task DownloadAndSaveImage(BaseItem item, string source, string targetName, SemaphoreSlim resourcePool, CancellationToken cancellationToken) + { + if (item == null) + { + throw new ArgumentNullException("item"); + } + if (string.IsNullOrEmpty(source)) + { + throw new ArgumentNullException("source"); + } + if (string.IsNullOrEmpty(targetName)) + { + throw new ArgumentNullException("targetName"); + } + if (resourcePool == null) + { + throw new ArgumentNullException("resourcePool"); + } + + //download and save locally + var localPath = ConfigurationManager.Configuration.SaveLocalMeta ? + Path.Combine(item.MetaLocation, targetName) : + _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName); + + var img = await _httpClient.GetMemoryStream(source, resourcePool, cancellationToken).ConfigureAwait(false); + + if (ConfigurationManager.Configuration.SaveLocalMeta) // queue to media directories + { + await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false); + } + else + { + // we can write directly here because it won't affect the watchers + + try + { + using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + _logger.ErrorException("Error downloading and saving image " + localPath, e); + throw; + } + finally + { + img.Dispose(); + } + + } + return localPath; + } + + + /// + /// Saves to library filesystem. + /// + /// The item. + /// The path. + /// The data to save. + /// The cancellation token. + /// Task. + /// + public async Task SaveToLibraryFilesystem(BaseItem item, string path, Stream dataToSave, CancellationToken cancellationToken) + { + if (item == null) + { + throw new ArgumentNullException(); + } + if (string.IsNullOrEmpty(path)) + { + throw new ArgumentNullException(); + } + if (dataToSave == null) + { + throw new ArgumentNullException(); + } + if (cancellationToken == null) + { + throw new ArgumentNullException(); + } + + cancellationToken.ThrowIfCancellationRequested(); + + //Tell the watchers to ignore + _directoryWatchers.TemporarilyIgnore(path); + + //Make the mod + + dataToSave.Position = 0; + + try + { + using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await dataToSave.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); + + dataToSave.Dispose(); + + // If this is ever used for something other than metadata we can add a file type param + item.ResolveArgs.AddMetadataFile(path); + } + } + finally + { + //Remove the ignore + _directoryWatchers.RemoveTempIgnore(path); + } + } + + /// + /// Releases unmanaged and - optionally - managed resources. + /// + /// true to release both managed and unmanaged resources; false to release only unmanaged resources. + protected virtual void Dispose(bool dispose) + { + if (dispose) + { + _remoteImageCache.Dispose(); + } + } + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() + { + Dispose(true); + } + } +} diff --git a/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs b/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs index 8306e094b..b20630edb 100644 --- a/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs +++ b/MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs @@ -27,17 +27,21 @@ namespace MediaBrowser.Server.Implementations.ScheduledTasks /// private readonly ILogger _logger; private readonly ILibraryManager _libraryManager; + private readonly IServerApplicationPaths _appPaths; /// /// Initializes a new instance of the class. /// /// The kernel. /// The logger. - public ImageCleanupTask(Kernel kernel, ILogger logger, ILibraryManager libraryManager) + /// The library manager. + /// The app paths. + public ImageCleanupTask(Kernel kernel, ILogger logger, ILibraryManager libraryManager, IServerApplicationPaths appPaths) { _kernel = kernel; _logger = logger; _libraryManager = libraryManager; + _appPaths = appPaths; } /// @@ -65,7 +69,7 @@ namespace MediaBrowser.Server.Implementations.ScheduledTasks // First gather all image files var files = GetFiles(_kernel.FFMpegManager.AudioImagesDataPath) .Concat(GetFiles(_kernel.FFMpegManager.VideoImagesDataPath)) - .Concat(GetFiles(_kernel.ProviderManager.ImagesDataPath)) + .Concat(GetFiles(_appPaths.DownloadedImagesDataPath)) .ToList(); // Now gather all items diff --git a/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs b/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs index ff29badff..429c893ca 100644 --- a/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs +++ b/MediaBrowser.Server.Implementations/ServerApplicationPaths.cs @@ -302,7 +302,7 @@ namespace MediaBrowser.Server.Implementations /// Gets the FF MPEG stream cache path. /// /// The FF MPEG stream cache path. - public string FFMpegStreamCachePath + public string EncodedMediaCachePath { get { @@ -345,5 +345,31 @@ namespace MediaBrowser.Server.Implementations return _mediaToolsPath; } } + + /// + /// The _images data path + /// + private string _downloadedImagesDataPath; + /// + /// Gets the images data path. + /// + /// The images data path. + public string DownloadedImagesDataPath + { + get + { + if (_downloadedImagesDataPath == null) + { + _downloadedImagesDataPath = Path.Combine(DataPath, "remote-images"); + + if (!Directory.Exists(_downloadedImagesDataPath)) + { + Directory.CreateDirectory(_downloadedImagesDataPath); + } + } + + return _downloadedImagesDataPath; + } + } } } diff --git a/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs b/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs index e4cb83e96..60628853e 100644 --- a/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs +++ b/MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs @@ -104,8 +104,8 @@ namespace MediaBrowser.Server.Implementations.ServerManager /// The web socket listeners. private readonly List _webSocketListeners = new List(); - private Kernel _kernel; - + private readonly Kernel _kernel; + /// /// Initializes a new instance of the class. /// -- cgit v1.2.3