diff options
| author | Luke <luke.pulverenti@gmail.com> | 2015-09-02 11:50:00 -0400 |
|---|---|---|
| committer | Luke <luke.pulverenti@gmail.com> | 2015-09-02 11:50:00 -0400 |
| commit | f868dd81e856488280978006cbb67afc2677049d (patch) | |
| tree | 616ba8ae846efe9ec889abeb12f6b2702c6b8592 /MediaBrowser.Server.Implementations/Channels | |
| parent | af89446c20fb302087b82c18c28da92076dbc5ac (diff) | |
| parent | e6d5901408ba7d8e344a27ea1f3b0046c40e56c1 (diff) | |
Merge pull request #1164 from MediaBrowser/dev
3.0.5724.1
Diffstat (limited to 'MediaBrowser.Server.Implementations/Channels')
4 files changed, 96 insertions, 526 deletions
diff --git a/MediaBrowser.Server.Implementations/Channels/ChannelDownloadScheduledTask.cs b/MediaBrowser.Server.Implementations/Channels/ChannelDownloadScheduledTask.cs deleted file mode 100644 index 18711c61e..000000000 --- a/MediaBrowser.Server.Implementations/Channels/ChannelDownloadScheduledTask.cs +++ /dev/null @@ -1,415 +0,0 @@ -using MediaBrowser.Common.IO; -using MediaBrowser.Common.Net; -using MediaBrowser.Common.Progress; -using MediaBrowser.Common.ScheduledTasks; -using MediaBrowser.Common.Security; -using MediaBrowser.Controller.Channels; -using MediaBrowser.Controller.Configuration; -using MediaBrowser.Controller.Entities; -using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.Channels; -using MediaBrowser.Model.Configuration; -using MediaBrowser.Model.Dto; -using MediaBrowser.Model.Logging; -using MediaBrowser.Model.MediaInfo; -using MediaBrowser.Model.Querying; -using MoreLinq; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace MediaBrowser.Server.Implementations.Channels -{ - public class ChannelDownloadScheduledTask : IScheduledTask, IConfigurableScheduledTask - { - private readonly IChannelManager _manager; - private readonly IServerConfigurationManager _config; - private readonly ILogger _logger; - private readonly IHttpClient _httpClient; - private readonly IFileSystem _fileSystem; - private readonly ILibraryManager _libraryManager; - private readonly IUserManager _userManager; - private readonly ISecurityManager _security; - - public ChannelDownloadScheduledTask(IChannelManager manager, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient, IFileSystem fileSystem, ILibraryManager libraryManager, IUserManager userManager, ISecurityManager security) - { - _manager = manager; - _config = config; - _logger = logger; - _httpClient = httpClient; - _fileSystem = fileSystem; - _libraryManager = libraryManager; - _userManager = userManager; - _security = security; - } - - public string Name - { - get { return "Download channel content"; } - } - - public string Description - { - get { return "Downloads channel content based on configuration."; } - } - - public string Category - { - get { return "Channels"; } - } - - public async Task Execute(CancellationToken cancellationToken, IProgress<double> progress) - { - CleanChannelContent(cancellationToken); - - var users = _userManager.Users - .DistinctBy(GetUserDistinctValue) - .Select(i => i.Id.ToString("N")) - .ToList(); - - var numComplete = 0; - - foreach (var user in users) - { - double percentPerUser = 1; - percentPerUser /= users.Count; - var startingPercent = numComplete * percentPerUser * 100; - - var innerProgress = new ActionableProgress<double>(); - innerProgress.RegisterAction(p => progress.Report(startingPercent + (percentPerUser * p))); - - await DownloadContent(user, cancellationToken, innerProgress).ConfigureAwait(false); - - numComplete++; - double percent = numComplete; - percent /= users.Count; - progress.Report(percent * 100); - } - - progress.Report(100); - } - - public static string GetUserDistinctValue(User user) - { - var channels = user.Policy.EnabledChannels - .OrderBy(i => i) - .ToList(); - - return string.Join("|", channels.ToArray()); - } - - private async Task DownloadContent(string user, - CancellationToken cancellationToken, - IProgress<double> progress) - { - var innerProgress = new ActionableProgress<double>(); - innerProgress.RegisterAction(p => progress.Report(0 + (.8 * p))); - await DownloadAllChannelContent(user, cancellationToken, innerProgress).ConfigureAwait(false); - progress.Report(80); - - innerProgress = new ActionableProgress<double>(); - innerProgress.RegisterAction(p => progress.Report(80 + (.2 * p))); - await DownloadLatestChannelContent(user, cancellationToken, progress).ConfigureAwait(false); - progress.Report(100); - } - - private async Task DownloadLatestChannelContent(string userId, - CancellationToken cancellationToken, - IProgress<double> progress) - { - var result = await _manager.GetLatestChannelItemsInternal(new AllChannelMediaQuery - { - UserId = userId - - }, cancellationToken).ConfigureAwait(false); - - progress.Report(5); - - var innerProgress = new ActionableProgress<double>(); - innerProgress.RegisterAction(p => progress.Report(5 + (.95 * p))); - - var path = _manager.ChannelDownloadPath; - - await DownloadChannelContent(result, path, cancellationToken, innerProgress).ConfigureAwait(false); - } - - private async Task DownloadAllChannelContent(string userId, - CancellationToken cancellationToken, - IProgress<double> progress) - { - var result = await _manager.GetAllMediaInternal(new AllChannelMediaQuery - { - UserId = userId - - }, cancellationToken).ConfigureAwait(false); - - progress.Report(5); - - var innerProgress = new ActionableProgress<double>(); - innerProgress.RegisterAction(p => progress.Report(5 + (.95 * p))); - - var path = _manager.ChannelDownloadPath; - - await DownloadChannelContent(result, path, cancellationToken, innerProgress).ConfigureAwait(false); - } - - private async Task DownloadChannelContent(QueryResult<BaseItem> result, - string path, - CancellationToken cancellationToken, - IProgress<double> progress) - { - var numComplete = 0; - - var options = _config.GetChannelsConfiguration(); - - foreach (var item in result.Items) - { - var channelItem = item as IChannelMediaItem; - - if (channelItem != null) - { - var channelFeatures = _manager.GetChannelFeatures(channelItem.ChannelId); - - if (channelFeatures.SupportsContentDownloading) - { - if (options.DownloadingChannels.Contains(channelItem.ChannelId)) - { - try - { - await DownloadChannelItem(channelItem, options, cancellationToken, path); - } - catch (OperationCanceledException) - { - break; - } - catch (ChannelDownloadException) - { - // Logged at lower levels - } - catch (Exception ex) - { - _logger.ErrorException("Error downloading channel content for {0}", ex, item.Name); - } - } - } - } - - numComplete++; - double percent = numComplete; - percent /= result.Items.Length; - progress.Report(percent * 100); - } - - progress.Report(100); - } - - private double? GetDownloadLimit(ChannelOptions channelOptions) - { - return channelOptions.DownloadSizeLimit; - } - - private async Task DownloadChannelItem(IChannelMediaItem item, - ChannelOptions channelOptions, - CancellationToken cancellationToken, - string path) - { - var itemId = item.Id.ToString("N"); - var sources = await _manager.GetStaticMediaSources(item, true, cancellationToken) - .ConfigureAwait(false); - - var cachedVersions = sources.Where(i => i.Protocol == MediaProtocol.File).ToList(); - - if (cachedVersions.Count > 0) - { - await RefreshMediaSourceItems(cachedVersions, cancellationToken).ConfigureAwait(false); - return; - } - - var limit = GetDownloadLimit(channelOptions); - - if (limit.HasValue) - { - if (IsSizeLimitReached(path, limit.Value)) - { - return; - } - } - - var destination = Path.Combine(path, item.ChannelId, itemId); - - await _manager.DownloadChannelItem(item, destination, new Progress<double>(), cancellationToken) - .ConfigureAwait(false); - - await RefreshMediaSourceItem(destination, cancellationToken).ConfigureAwait(false); - } - - private async Task RefreshMediaSourceItems(IEnumerable<MediaSourceInfo> items, CancellationToken cancellationToken) - { - foreach (var item in items) - { - await RefreshMediaSourceItem(item.Path, cancellationToken).ConfigureAwait(false); - } - } - - private async Task RefreshMediaSourceItem(string path, CancellationToken cancellationToken) - { - var item = _libraryManager.ResolvePath(new FileInfo(path)); - - if (item != null) - { - var forceSave = false; - - // Get the version from the database - var dbItem = _libraryManager.GetItemById(item.Id); - - if (dbItem == null) - { - forceSave = true; - } - else - { - item = dbItem; - } - - await item.RefreshMetadata(new MetadataRefreshOptions - { - ForceSave = forceSave - - }, cancellationToken).ConfigureAwait(false); - } - } - - private bool IsSizeLimitReached(string path, double gbLimit) - { - try - { - var byteLimit = gbLimit * 1000000000; - - long total = 0; - - foreach (var file in new DirectoryInfo(path).EnumerateFiles("*", SearchOption.AllDirectories)) - { - total += file.Length; - - if (total >= byteLimit) - { - return true; - } - } - - return false; - } - catch (DirectoryNotFoundException) - { - return false; - } - } - - public IEnumerable<ITaskTrigger> GetDefaultTriggers() - { - return new ITaskTrigger[] - { - new IntervalTrigger{ Interval = TimeSpan.FromHours(3)}, - }; - } - - private void CleanChannelContent(CancellationToken cancellationToken) - { - var options = _config.GetChannelsConfiguration(); - - if (!options.MaxDownloadAge.HasValue) - { - return; - } - - var minDateModified = DateTime.UtcNow.AddDays(0 - options.MaxDownloadAge.Value); - - var path = _manager.ChannelDownloadPath; - - try - { - DeleteCacheFilesFromDirectory(cancellationToken, path, minDateModified, new Progress<double>()); - } - catch (DirectoryNotFoundException) - { - // No biggie here. Nothing to delete - } - } - - /// <summary> - /// Deletes the cache files from directory with a last write time less than a given date - /// </summary> - /// <param name="cancellationToken">The task cancellation token.</param> - /// <param name="directory">The directory.</param> - /// <param name="minDateModified">The min date modified.</param> - /// <param name="progress">The progress.</param> - private void DeleteCacheFilesFromDirectory(CancellationToken cancellationToken, string directory, DateTime minDateModified, IProgress<double> progress) - { - var filesToDelete = new DirectoryInfo(directory).EnumerateFiles("*", SearchOption.AllDirectories) - .Where(f => _fileSystem.GetLastWriteTimeUtc(f) < minDateModified) - .ToList(); - - var index = 0; - - foreach (var file in filesToDelete) - { - double percent = index; - percent /= filesToDelete.Count; - - progress.Report(100 * percent); - - cancellationToken.ThrowIfCancellationRequested(); - - DeleteFile(file.FullName); - - index++; - } - - progress.Report(100); - } - - /// <summary> - /// Deletes the file. - /// </summary> - /// <param name="path">The path.</param> - private void DeleteFile(string path) - { - try - { - _fileSystem.DeleteFile(path); - } - catch (IOException ex) - { - _logger.ErrorException("Error deleting file {0}", ex, path); - } - } - - /// <summary> - /// Gets a value indicating whether this instance is hidden. - /// </summary> - /// <value><c>true</c> if this instance is hidden; otherwise, <c>false</c>.</value> - public bool IsHidden - { - get - { - return !_manager.GetAllChannelFeatures().Any(); - } - } - - /// <summary> - /// Gets a value indicating whether this instance is enabled. - /// </summary> - /// <value><c>true</c> if this instance is enabled; otherwise, <c>false</c>.</value> - public bool IsEnabled - { - get - { - return true; - } - } - } -} diff --git a/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs b/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs index 0cd4b0a5c..f670176e6 100644 --- a/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs +++ b/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs @@ -23,7 +23,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -255,7 +254,7 @@ namespace MediaBrowser.Server.Implementations.Channels sources.InsertRange(0, cachedVersions); } - return sources.Where(IsValidMediaSource); + return sources; } public async Task<IEnumerable<MediaSourceInfo>> GetDynamicMediaSources(IChannelMediaItem item, CancellationToken cancellationToken) @@ -279,7 +278,6 @@ namespace MediaBrowser.Server.Implementations.Channels var list = SortMediaInfoResults(results) .Select(i => GetMediaSource(item, i)) - .Where(IsValidMediaSource) .ToList(); var cachedVersions = GetCachedChannelItemMediaSources(item); @@ -478,7 +476,7 @@ namespace MediaBrowser.Server.Implementations.Channels public Channel GetChannel(string id) { - return (Channel)_libraryManager.GetItemById(new Guid(id)); + return _libraryManager.GetItemById(new Guid(id)) as Channel; } public IEnumerable<ChannelFeatures> GetAllChannelFeatures() @@ -916,7 +914,7 @@ namespace MediaBrowser.Server.Implementations.Channels if (query.Limit.HasValue && query.Limit.Value > channelInfo.MaxPageSize.Value) { - throw new ArgumentException(string.Format("{0} channel only supports a maximum of {1} records at a time.", channel.Name, channelInfo.MaxPageSize.Value)); + query.Limit = Math.Min(query.Limit.Value, channelInfo.MaxPageSize.Value); } providerLimit = query.Limit; @@ -1401,17 +1399,16 @@ namespace MediaBrowser.Server.Implementations.Channels { var user = string.IsNullOrEmpty(userId) ? null : _userManager.GetUserById(userId); - var folder = await GetInternalChannelFolder(userId, cancellationToken).ConfigureAwait(false); + var folder = await GetInternalChannelFolder(cancellationToken).ConfigureAwait(false); return _dtoService.GetBaseItemDto(folder, new DtoOptions(), user); } - public async Task<Folder> GetInternalChannelFolder(string userId, CancellationToken cancellationToken) + public async Task<Folder> GetInternalChannelFolder(CancellationToken cancellationToken) { var name = _localization.GetLocalizedString("ViewTypeChannels"); - var user = _userManager.GetUserById(userId); - return await _libraryManager.GetNamedView(user, name, "channels", "zz_" + name, cancellationToken).ConfigureAwait(false); + return await _libraryManager.GetNamedView(name, "channels", "zz_" + name, cancellationToken).ConfigureAwait(false); } public async Task DownloadChannelItem(IChannelMediaItem item, string destination, @@ -1424,18 +1421,8 @@ namespace MediaBrowser.Server.Implementations.Channels foreach (var source in list) { - try - { - await TryDownloadChannelItem(source, item, destination, progress, cancellationToken).ConfigureAwait(false); - return; - } - catch (HttpException ex) - { - if (ex.StatusCode.HasValue && ex.StatusCode.Value == HttpStatusCode.NotFound) - { - MarkBadMediaSource(source); - } - } + await TryDownloadChannelItem(source, item, destination, progress, cancellationToken).ConfigureAwait(false); + return; } } @@ -1525,81 +1512,6 @@ namespace MediaBrowser.Server.Implementations.Channels } } - private readonly ReaderWriterLockSlim _mediaSourceHistoryLock = new ReaderWriterLockSlim(); - private bool IsValidMediaSource(MediaSourceInfo source) - { - if (source.Protocol == MediaProtocol.Http) - { - return !GetBadMediaSourceHistory().Contains(source.Path, StringComparer.OrdinalIgnoreCase); - } - return true; - } - - private void MarkBadMediaSource(MediaSourceInfo source) - { - var list = GetBadMediaSourceHistory(); - list.Add(source.Path); - - var path = GetMediaSourceHistoryPath(); - - Directory.CreateDirectory(Path.GetDirectoryName(path)); - - if (_mediaSourceHistoryLock.TryEnterWriteLock(TimeSpan.FromSeconds(5))) - { - try - { - File.WriteAllLines(path, list.ToArray(), Encoding.UTF8); - } - catch (Exception ex) - { - _logger.ErrorException("Error saving file", ex); - } - finally - { - _mediaSourceHistoryLock.ExitWriteLock(); - } - } - } - - private ConcurrentBag<string> _badMediaSources = null; - private ConcurrentBag<string> GetBadMediaSourceHistory() - { - if (_badMediaSources == null) - { - var path = GetMediaSourceHistoryPath(); - - if (_mediaSourceHistoryLock.TryEnterReadLock(TimeSpan.FromSeconds(1))) - { - if (_badMediaSources == null) - { - try - { - _badMediaSources = new ConcurrentBag<string>(File.ReadAllLines(path, Encoding.UTF8)); - } - catch (IOException) - { - _badMediaSources = new ConcurrentBag<string>(); - } - catch (Exception ex) - { - _logger.ErrorException("Error reading file", ex); - _badMediaSources = new ConcurrentBag<string>(); - } - finally - { - _mediaSourceHistoryLock.ExitReadLock(); - } - } - } - } - return _badMediaSources; - } - - private string GetMediaSourceHistoryPath() - { - return Path.Combine(_config.ApplicationPaths.DataPath, "channels", "failures.txt"); - } - private void IncrementDownloadCount(string key, int? limit) { if (!limit.HasValue) diff --git a/MediaBrowser.Server.Implementations/Channels/ChannelPostScanTask.cs b/MediaBrowser.Server.Implementations/Channels/ChannelPostScanTask.cs index d266cca6c..976fa5615 100644 --- a/MediaBrowser.Server.Implementations/Channels/ChannelPostScanTask.cs +++ b/MediaBrowser.Server.Implementations/Channels/ChannelPostScanTask.cs @@ -1,5 +1,6 @@ using MediaBrowser.Common.Progress; using MediaBrowser.Controller.Channels; +using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Model.Channels; using MediaBrowser.Model.Logging; @@ -12,23 +13,25 @@ using System.Threading.Tasks; namespace MediaBrowser.Server.Implementations.Channels { - public class ChannelPostScanTask : ILibraryPostScanTask + public class ChannelPostScanTask { private readonly IChannelManager _channelManager; private readonly IUserManager _userManager; private readonly ILogger _logger; + private readonly ILibraryManager _libraryManager; - public ChannelPostScanTask(IChannelManager channelManager, IUserManager userManager, ILogger logger) + public ChannelPostScanTask(IChannelManager channelManager, IUserManager userManager, ILogger logger, ILibraryManager libraryManager) { _channelManager = channelManager; _userManager = userManager; _logger = logger; + _libraryManager = libraryManager; } public async Task Run(IProgress<double> progress, CancellationToken cancellationToken) { var users = _userManager.Users - .DistinctBy(ChannelDownloadScheduledTask.GetUserDistinctValue) + .DistinctBy(GetUserDistinctValue) .Select(i => i.Id.ToString("N")) .ToList(); @@ -51,9 +54,20 @@ namespace MediaBrowser.Server.Implementations.Channels progress.Report(percent * 100); } + await CleanDatabase(cancellationToken).ConfigureAwait(false); + progress.Report(100); } + public static string GetUserDistinctValue(User user) + { + var channels = user.Policy.EnabledChannels + .OrderBy(i => i) + .ToList(); + + return string.Join("|", channels.ToArray()); + } + private async Task DownloadContent(string user, CancellationToken cancellationToken, IProgress<double> progress) { var channels = await _channelManager.GetChannelsInternal(new ChannelQuery @@ -106,6 +120,59 @@ namespace MediaBrowser.Server.Implementations.Channels progress.Report(100); } + private async Task CleanDatabase(CancellationToken cancellationToken) + { + var allChannels = await _channelManager.GetChannelsInternal(new ChannelQuery { }, cancellationToken); + + var allIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + IncludeItemTypes = new[] { typeof(Channel).Name } + }); + + var invalidIds = allIds + .Except(allChannels.Items.Select(i => i.Id).ToList()) + .ToList(); + + foreach (var id in invalidIds) + { + cancellationToken.ThrowIfCancellationRequested(); + + await CleanChannel(id, cancellationToken).ConfigureAwait(false); + } + } + + private async Task CleanChannel(Guid id, CancellationToken cancellationToken) + { + _logger.Debug("Cleaning channel {0} from database", id); + + // Delete all channel items + var allIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + ChannelIds = new[] { id.ToString("N") } + }); + + foreach (var deleteId in allIds) + { + cancellationToken.ThrowIfCancellationRequested(); + + await DeleteItem(deleteId).ConfigureAwait(false); + } + + // Finally, delete the channel itself + await DeleteItem(id).ConfigureAwait(false); + } + + private Task DeleteItem(Guid id) + { + var item = _libraryManager.GetItemById(id); + + return _libraryManager.DeleteItem(item, new DeleteOptions + { + DeleteFileLocation = false + + }); + } + private async Task GetAllItems(string user, string channelId, string folderId, int currentRefreshLevel, int maxRefreshLevel, IProgress<double> progress, CancellationToken cancellationToken) { var folderItems = new List<string>(); diff --git a/MediaBrowser.Server.Implementations/Channels/RefreshChannelsScheduledTask.cs b/MediaBrowser.Server.Implementations/Channels/RefreshChannelsScheduledTask.cs index c6ecfc250..005bbb852 100644 --- a/MediaBrowser.Server.Implementations/Channels/RefreshChannelsScheduledTask.cs +++ b/MediaBrowser.Server.Implementations/Channels/RefreshChannelsScheduledTask.cs @@ -1,6 +1,7 @@ using MediaBrowser.Common.ScheduledTasks; using MediaBrowser.Controller.Channels; -using MediaBrowser.Model.Tasks; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Logging; using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -9,11 +10,17 @@ namespace MediaBrowser.Server.Implementations.Channels { class RefreshChannelsScheduledTask : IScheduledTask, IConfigurableScheduledTask { - private readonly IChannelManager _manager; + private readonly IChannelManager _channelManager; + private readonly IUserManager _userManager; + private readonly ILogger _logger; + private readonly ILibraryManager _libraryManager; - public RefreshChannelsScheduledTask(IChannelManager manager) + public RefreshChannelsScheduledTask(IChannelManager channelManager, IUserManager userManager, ILogger logger, ILibraryManager libraryManager) { - _manager = manager; + _channelManager = channelManager; + _userManager = userManager; + _logger = logger; + _libraryManager = libraryManager; } public string Name @@ -31,28 +38,27 @@ namespace MediaBrowser.Server.Implementations.Channels get { return "Channels"; } } - public Task Execute(System.Threading.CancellationToken cancellationToken, IProgress<double> progress) + public async Task Execute(System.Threading.CancellationToken cancellationToken, IProgress<double> progress) { - var manager = (ChannelManager)_manager; + var manager = (ChannelManager)_channelManager; - return manager.RefreshChannels(progress, cancellationToken); + await manager.RefreshChannels(new Progress<double>(), cancellationToken).ConfigureAwait(false); + + await new ChannelPostScanTask(_channelManager, _userManager, _logger, _libraryManager).Run(progress, cancellationToken) + .ConfigureAwait(false); } public IEnumerable<ITaskTrigger> GetDefaultTriggers() { return new ITaskTrigger[] { - new StartupTrigger{DelayMs = 10000}, - - new SystemEventTrigger{ SystemEvent = SystemEvent.WakeFromSleep}, - new IntervalTrigger{ Interval = TimeSpan.FromHours(24)} }; } public bool IsHidden { - get { return true; } + get { return false; } } public bool IsEnabled |
