diff options
Diffstat (limited to 'MediaBrowser.Server.Implementations/Sync')
9 files changed, 1291 insertions, 240 deletions
diff --git a/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs b/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs index 6cc5be955..d35ff8fc4 100644 --- a/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs +++ b/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs @@ -17,19 +17,6 @@ namespace MediaBrowser.Server.Implementations.Sync _deviceManager = deviceManager; } - public IEnumerable<SyncTarget> GetSyncTargets() - { - return _deviceManager.GetDevices(new DeviceQuery - { - SupportsSync = true - - }).Items.Select(i => new SyncTarget - { - Id = i.Id, - Name = i.Name - }); - } - public IEnumerable<SyncTarget> GetSyncTargets(string userId) { return _deviceManager.GetDevices(new DeviceQuery diff --git a/MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs b/MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs index da3ecdfa6..59713b138 100644 --- a/MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs +++ b/MediaBrowser.Server.Implementations/Sync/CloudSyncProvider.cs @@ -2,28 +2,26 @@ using MediaBrowser.Controller.Sync; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Sync; +using System; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace MediaBrowser.Server.Implementations.Sync { - public class CloudSyncProvider : ISyncProvider + public class CloudSyncProvider : IServerSyncProvider { - private ICloudSyncProvider[] _providers = {}; + private readonly ICloudSyncProvider[] _providers = {}; public CloudSyncProvider(IApplicationHost appHost) { _providers = appHost.GetExports<ICloudSyncProvider>().ToArray(); } - public IEnumerable<SyncTarget> GetSyncTargets() - { - return new List<SyncTarget>(); - } - public IEnumerable<SyncTarget> GetSyncTargets(string userId) { - return new List<SyncTarget>(); + return _providers.SelectMany(i => i.GetSyncTargets(userId)); } public DeviceProfile GetDeviceProfile(SyncTarget target) @@ -35,5 +33,20 @@ namespace MediaBrowser.Server.Implementations.Sync { get { return "Cloud Sync"; } } + + public Task<List<string>> GetServerItemIds(string serverId, SyncTarget target, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public Task DeleteItem(string serverId, string itemId, SyncTarget target, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public Task TransferItemFile(string serverId, string itemId, string[] pathParts, string name, ItemFileType fileType, SyncTarget target, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } } } diff --git a/MediaBrowser.Server.Implementations/Sync/MediaSync.cs b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs new file mode 100644 index 000000000..efdfbb086 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs @@ -0,0 +1,174 @@ +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Sync; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Sync +{ + public class MediaSync + { + private readonly ISyncManager _syncManager; + private readonly IServerApplicationHost _appHost; + private readonly ILogger _logger; + + public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost) + { + _logger = logger; + _syncManager = syncManager; + _appHost = appHost; + } + + public async Task Sync(IServerSyncProvider provider, + SyncTarget target, + IProgress<double> progress, + CancellationToken cancellationToken) + { + var serverId = _appHost.SystemId; + + await SyncData(provider, serverId, target, cancellationToken).ConfigureAwait(false); + progress.Report(2); + + // Do the data sync twice so the server knows what was removed from the device + await SyncData(provider, serverId, target, cancellationToken).ConfigureAwait(false); + progress.Report(3); + + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(pct => + { + var totalProgress = pct * .97; + totalProgress += 1; + progress.Report(totalProgress); + }); + await GetNewMedia(provider, target, serverId, innerProgress, cancellationToken); + progress.Report(100); + } + + private async Task SyncData(IServerSyncProvider provider, + string serverId, + SyncTarget target, + CancellationToken cancellationToken) + { + var localIds = await provider.GetServerItemIds(serverId, target, cancellationToken).ConfigureAwait(false); + + var result = await _syncManager.SyncData(new SyncDataRequest + { + TargetId = target.Id, + LocalItemIds = localIds + + }).ConfigureAwait(false); + + cancellationToken.ThrowIfCancellationRequested(); + + foreach (var itemIdToRemove in result.ItemIdsToRemove) + { + try + { + await RemoveItem(provider, serverId, itemIdToRemove, target, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting item from sync target. Id: {0}", ex, itemIdToRemove); + } + } + } + + private async Task GetNewMedia(IServerSyncProvider provider, + SyncTarget target, + string serverId, + IProgress<double> progress, + CancellationToken cancellationToken) + { + var jobItems = _syncManager.GetReadySyncItems(target.Id); + + var numComplete = 0; + double startingPercent = 0; + double percentPerItem = 1; + if (jobItems.Count > 0) + { + percentPerItem /= jobItems.Count; + } + + foreach (var jobItem in jobItems) + { + cancellationToken.ThrowIfCancellationRequested(); + + var currentPercent = startingPercent; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(pct => + { + var totalProgress = pct * percentPerItem; + totalProgress += currentPercent; + progress.Report(totalProgress); + }); + + await GetItem(provider, target, serverId, jobItem, innerProgress, cancellationToken).ConfigureAwait(false); + + numComplete++; + startingPercent = numComplete; + startingPercent /= jobItems.Count; + startingPercent *= 100; + progress.Report(startingPercent); + } + } + + private async Task GetItem(IServerSyncProvider provider, + SyncTarget target, + string serverId, + SyncedItem jobItem, + IProgress<double> progress, + CancellationToken cancellationToken) + { + var libraryItem = jobItem.Item; + var internalSyncJobItem = _syncManager.GetJobItem(jobItem.SyncJobItemId); + + var fileTransferProgress = new ActionableProgress<double>(); + fileTransferProgress.RegisterAction(pct => progress.Report(pct * .92)); + + await _syncManager.ReportSyncJobItemTransferBeginning(internalSyncJobItem.Id); + + var transferSuccess = false; + Exception transferException = null; + + try + { + //await provider.TransferItemFile(serverId, libraryItem.Id, internalSyncJobItem.OutputPath, target, cancellationToken) + // .ConfigureAwait(false); + + progress.Report(92); + + transferSuccess = true; + + progress.Report(99); + } + catch (Exception ex) + { + _logger.ErrorException("Error transferring sync job file", ex); + transferException = ex; + } + + if (transferSuccess) + { + await _syncManager.ReportSyncJobItemTransferred(jobItem.SyncJobItemId).ConfigureAwait(false); + } + else + { + await _syncManager.ReportSyncJobItemTransferFailed(jobItem.SyncJobItemId).ConfigureAwait(false); + + throw transferException; + } + } + + private Task RemoveItem(IServerSyncProvider provider, + string serverId, + string itemId, + SyncTarget target, + CancellationToken cancellationToken) + { + return provider.DeleteItem(serverId, itemId, target, cancellationToken); + } + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/SyncConfig.cs b/MediaBrowser.Server.Implementations/Sync/SyncConfig.cs new file mode 100644 index 000000000..52c774330 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/SyncConfig.cs @@ -0,0 +1,29 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Model.Sync; +using System.Collections.Generic; + +namespace MediaBrowser.Server.Implementations.Sync +{ + public class SyncConfigurationFactory : IConfigurationFactory + { + public IEnumerable<ConfigurationStore> GetConfigurations() + { + return new List<ConfigurationStore> + { + new ConfigurationStore + { + ConfigurationType = typeof(SyncOptions), + Key = "sync" + } + }; + } + } + + public static class SyncExtensions + { + public static SyncOptions GetSyncOptions(this IConfigurationManager config) + { + return config.GetConfiguration<SyncOptions>("sync"); + } + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs b/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs index 698806d09..a254d862c 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs @@ -1,6 +1,9 @@ -using MediaBrowser.Common.Progress; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.IO; +using MediaBrowser.Common.Progress; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Sync; @@ -16,6 +19,7 @@ using MediaBrowser.Model.Sync; using MoreLinq; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -26,13 +30,16 @@ namespace MediaBrowser.Server.Implementations.Sync { private readonly ILibraryManager _libraryManager; private readonly ISyncRepository _syncRepo; - private readonly ISyncManager _syncManager; + private readonly SyncManager _syncManager; private readonly ILogger _logger; private readonly IUserManager _userManager; private readonly ITVSeriesManager _tvSeriesManager; private readonly IMediaEncoder _mediaEncoder; + private readonly ISubtitleEncoder _subtitleEncoder; + private readonly IConfigurationManager _config; + private readonly IFileSystem _fileSystem; - public SyncJobProcessor(ILibraryManager libraryManager, ISyncRepository syncRepo, ISyncManager syncManager, ILogger logger, IUserManager userManager, ITVSeriesManager tvSeriesManager, IMediaEncoder mediaEncoder) + public SyncJobProcessor(ILibraryManager libraryManager, ISyncRepository syncRepo, SyncManager syncManager, ILogger logger, IUserManager userManager, ITVSeriesManager tvSeriesManager, IMediaEncoder mediaEncoder, ISubtitleEncoder subtitleEncoder, IConfigurationManager config, IFileSystem fileSystem) { _libraryManager = libraryManager; _syncRepo = syncRepo; @@ -41,6 +48,9 @@ namespace MediaBrowser.Server.Implementations.Sync _userManager = userManager; _tvSeriesManager = tvSeriesManager; _mediaEncoder = mediaEncoder; + _subtitleEncoder = subtitleEncoder; + _config = config; + _fileSystem = fileSystem; } public async Task EnsureJobItems(SyncJob job) @@ -81,6 +91,10 @@ namespace MediaBrowser.Server.Implementations.Sync continue; } + var index = jobItems.Count == 0 ? + 0 : + (jobItems.Select(i => i.JobItemIndex).Max() + 1); + jobItem = new SyncJobItem { Id = Guid.NewGuid().ToString("N"), @@ -88,10 +102,12 @@ namespace MediaBrowser.Server.Implementations.Sync ItemName = GetSyncJobItemName(item), JobId = job.Id, TargetId = job.TargetId, - DateCreated = DateTime.UtcNow + DateCreated = DateTime.UtcNow, + JobItemIndex = index }; await _syncRepo.Create(jobItem).ConfigureAwait(false); + _syncManager.OnSyncJobItemCreated(jobItem); jobItems.Add(jobItem); } @@ -130,7 +146,7 @@ namespace MediaBrowser.Server.Implementations.Sync return UpdateJobStatus(job, result.Items.ToList()); } - private Task UpdateJobStatus(SyncJob job, List<SyncJobItem> jobItems) + private async Task UpdateJobStatus(SyncJob job, List<SyncJobItem> jobItems) { job.ItemCount = jobItems.Count; @@ -138,7 +154,7 @@ namespace MediaBrowser.Server.Implementations.Sync foreach (var item in jobItems) { - if (item.Status == SyncJobItemStatus.Failed || item.Status == SyncJobItemStatus.Synced || item.Status == SyncJobItemStatus.RemovedFromDevice) + if (item.Status == SyncJobItemStatus.Failed || item.Status == SyncJobItemStatus.Synced || item.Status == SyncJobItemStatus.RemovedFromDevice || item.Status == SyncJobItemStatus.Cancelled) { pct += 100; } @@ -158,7 +174,31 @@ namespace MediaBrowser.Server.Implementations.Sync job.Progress = null; } - if (pct >= 100) + if (jobItems.All(i => i.Status == SyncJobItemStatus.Queued)) + { + job.Status = SyncJobStatus.Queued; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.Failed)) + { + job.Status = SyncJobStatus.Failed; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.Cancelled)) + { + job.Status = SyncJobStatus.Cancelled; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.ReadyToTransfer)) + { + job.Status = SyncJobStatus.ReadyToTransfer; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.Transferring)) + { + job.Status = SyncJobStatus.Transferring; + } + else if (jobItems.Any(i => i.Status == SyncJobItemStatus.Converting)) + { + job.Status = SyncJobStatus.Converting; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.Cancelled || i.Status == SyncJobItemStatus.Failed || i.Status == SyncJobItemStatus.Synced || i.Status == SyncJobItemStatus.RemovedFromDevice)) { if (jobItems.Any(i => i.Status == SyncJobItemStatus.Failed)) { @@ -169,24 +209,23 @@ namespace MediaBrowser.Server.Implementations.Sync job.Status = SyncJobStatus.Completed; } } - else if (pct.Equals(0)) - { - job.Status = SyncJobStatus.Queued; - } else { - job.Status = SyncJobStatus.InProgress; + job.Status = SyncJobStatus.Queued; } - return _syncRepo.Update(job); + await _syncRepo.Update(job).ConfigureAwait(false); + + _syncManager.OnSyncJobUpdated(job); } public async Task<IEnumerable<BaseItem>> GetItemsForSync(SyncCategory? category, string parentId, IEnumerable<string> itemIds, User user, bool unwatchedOnly) { var items = category.HasValue ? await GetItemsForSync(category.Value, parentId, user).ConfigureAwait(false) : - itemIds.SelectMany(i => GetItemsForSync(i, user)) - .Where(_syncManager.SupportsSync); + itemIds.SelectMany(i => GetItemsForSync(i, user)); + + items = items.Where(_syncManager.SupportsSync); if (unwatchedOnly) { @@ -266,26 +305,25 @@ namespace MediaBrowser.Server.Implementations.Sync return new List<BaseItem>(); } - return GetItemsForSync(item, user); - } - - private IEnumerable<BaseItem> GetItemsForSync(BaseItem item, User user) - { var itemByName = item as IItemByName; if (itemByName != null) { - var items = user.RootFolder - .GetRecursiveChildren(user); + var itemByNameFilter = itemByName.GetItemFilter(); + + return user.RootFolder + .GetRecursiveChildren(user, i => !i.IsFolder && itemByNameFilter(i)); + } - return itemByName.GetTaggedItems(items); + var series = item as Series; + if (series != null) + { + return series.GetEpisodes(user, false, false); } if (item.IsFolder) { var folder = (Folder)item; - var items = folder.GetRecursiveChildren(user); - - items = items.Where(i => !i.IsFolder); + var items = folder.GetRecursiveChildren(user, i => !i.IsFolder); if (!folder.IsPreSorted) { @@ -298,11 +336,11 @@ namespace MediaBrowser.Server.Implementations.Sync return new[] { item }; } - public async Task EnsureSyncJobs(CancellationToken cancellationToken) + public async Task EnsureSyncJobItems(CancellationToken cancellationToken) { var jobResult = _syncRepo.GetJobs(new SyncJobQuery { - IsCompleted = false + SyncNewContent = true }); foreach (var job in jobResult.Items) @@ -318,7 +356,7 @@ namespace MediaBrowser.Server.Implementations.Sync public async Task Sync(IProgress<double> progress, CancellationToken cancellationToken) { - await EnsureSyncJobs(cancellationToken).ConfigureAwait(false); + await EnsureSyncJobItems(cancellationToken).ConfigureAwait(false); // If it already has a converting status then is must have been aborted during conversion var result = _syncRepo.GetJobItems(new SyncJobItemQuery @@ -326,38 +364,71 @@ namespace MediaBrowser.Server.Implementations.Sync Statuses = new List<SyncJobItemStatus> { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting } }); - var jobItems = result.Items; - var index = 0; + await SyncJobItems(result.Items, true, progress, cancellationToken).ConfigureAwait(false); - foreach (var item in jobItems) + CleanDeadSyncFiles(); + } + + private void CleanDeadSyncFiles() + { + // TODO + // Clean files in sync temp folder that are not linked to any sync jobs + } + + public async Task SyncJobItems(SyncJobItem[] items, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) + { + if (items.Length > 0) { - double percent = index; - percent /= result.TotalRecordCount; + if (!SyncRegistrationInfo.Instance.IsRegistered) + { + _logger.Debug("Cancelling sync job processing. Please obtain a supporter membership."); + return; + } + } - progress.Report(100 * percent); + var numComplete = 0; + foreach (var item in items) + { cancellationToken.ThrowIfCancellationRequested(); + double percentPerItem = 1; + percentPerItem /= items.Length; + var startingPercent = numComplete * percentPerItem * 100; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(p => progress.Report(startingPercent + (percentPerItem * p))); - var job = _syncRepo.GetJob(item.JobId); - await ProcessJobItem(job, item, innerProgress, cancellationToken).ConfigureAwait(false); + // Pull it fresh from the db just to make sure it wasn't deleted or cancelled while another item was converting + var jobItem = enableConversion ? _syncRepo.GetJobItem(item.Id) : item; - job = _syncRepo.GetJob(item.JobId); - await UpdateJobStatus(job).ConfigureAwait(false); + if (jobItem != null) + { + var job = _syncRepo.GetJob(jobItem.JobId); + if (jobItem.Status != SyncJobItemStatus.Cancelled) + { + await ProcessJobItem(job, jobItem, enableConversion, innerProgress, cancellationToken).ConfigureAwait(false); + } + + job = _syncRepo.GetJob(jobItem.JobId); + await UpdateJobStatus(job).ConfigureAwait(false); + } - index++; + numComplete++; + double percent = numComplete; + percent /= items.Length; + progress.Report(100 * percent); } } - private async Task ProcessJobItem(SyncJob job, SyncJobItem jobItem, IProgress<double> progress, CancellationToken cancellationToken) + private async Task ProcessJobItem(SyncJob job, SyncJobItem jobItem, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) { var item = _libraryManager.GetItemById(jobItem.ItemId); if (item == null) { jobItem.Status = SyncJobItemStatus.Failed; _logger.Error("Unable to locate library item for JobItem {0}, ItemId {1}", jobItem.Id, jobItem.ItemId); - await _syncRepo.Update(jobItem).ConfigureAwait(false); + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); return; } @@ -366,24 +437,23 @@ namespace MediaBrowser.Server.Implementations.Sync { jobItem.Status = SyncJobItemStatus.Failed; _logger.Error("Unable to locate SyncTarget for JobItem {0}, SyncTargetId {1}", jobItem.Id, jobItem.TargetId); - await _syncRepo.Update(jobItem).ConfigureAwait(false); + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); return; } jobItem.Progress = 0; - jobItem.Status = SyncJobItemStatus.Converting; var user = _userManager.GetUserById(job.UserId); var video = item as Video; if (video != null) { - await Sync(jobItem, video, user, deviceProfile, progress, cancellationToken).ConfigureAwait(false); + await Sync(jobItem, job, video, user, deviceProfile, enableConversion, progress, cancellationToken).ConfigureAwait(false); } else if (item is Audio) { - await Sync(jobItem, (Audio)item, user, deviceProfile, progress, cancellationToken).ConfigureAwait(false); + await Sync(jobItem, job, (Audio)item, user, deviceProfile, enableConversion, progress, cancellationToken).ConfigureAwait(false); } else if (item is Photo) @@ -393,40 +463,77 @@ namespace MediaBrowser.Server.Implementations.Sync else { - await SyncGeneric(jobItem, item, deviceProfile, cancellationToken).ConfigureAwait(false); + await SyncGeneric(jobItem, item, cancellationToken).ConfigureAwait(false); } } - private async Task Sync(SyncJobItem jobItem, Video item, User user, DeviceProfile profile, IProgress<double> progress, CancellationToken cancellationToken) + private async Task Sync(SyncJobItem jobItem, SyncJob job, Video item, User user, DeviceProfile profile, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) { - var options = new VideoOptions - { - Context = EncodingContext.Static, - ItemId = item.Id.ToString("N"), - DeviceId = jobItem.TargetId, - Profile = profile, - MediaSources = item.GetMediaSources(false, user).ToList() - }; + var options = _syncManager.GetVideoOptions(jobItem, job); + + options.DeviceId = jobItem.TargetId; + options.Context = EncodingContext.Static; + options.Profile = profile; + options.ItemId = item.Id.ToString("N"); + options.MediaSources = item.GetMediaSources(false, user).ToList(); var streamInfo = new StreamBuilder().BuildVideoItem(options); var mediaSource = streamInfo.MediaSource; + // No sense creating external subs if we're already burning one into the video + var externalSubs = streamInfo.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode ? + new List<SubtitleStreamInfo>() : + streamInfo.GetExternalSubtitles("dummy", false); + + // Mark as requiring conversion if transcoding the video, or if any subtitles need to be extracted + var requiresVideoTranscoding = streamInfo.PlayMethod == PlayMethod.Transcode && job.Quality != SyncQuality.Original; + var requiresConversion = requiresVideoTranscoding || externalSubs.Any(i => RequiresExtraction(i, mediaSource)); + + if (requiresConversion && !enableConversion) + { + return; + } + jobItem.MediaSourceId = streamInfo.MediaSourceId; + jobItem.TemporaryPath = GetTemporaryPath(jobItem); - if (streamInfo.PlayMethod == PlayMethod.Transcode) + if (requiresConversion) { jobItem.Status = SyncJobItemStatus.Converting; - jobItem.RequiresConversion = true; - await _syncRepo.Update(jobItem).ConfigureAwait(false); + } + + if (requiresVideoTranscoding) + { + // Save the job item now since conversion could take a while + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(job).ConfigureAwait(false); try { - jobItem.OutputPath = await _mediaEncoder.EncodeVideo(new EncodingJobOptions(streamInfo, profile), progress, - cancellationToken); + var lastJobUpdate = DateTime.MinValue; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(async pct => + { + progress.Report(pct); + + if ((DateTime.UtcNow - lastJobUpdate).TotalSeconds >= DatabaseProgressUpdateIntervalSeconds) + { + jobItem.Progress = pct / 2; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(job).ConfigureAwait(false); + } + }); + + jobItem.OutputPath = await _mediaEncoder.EncodeVideo(new EncodingJobOptions(streamInfo, profile) + { + OutputDirectory = jobItem.TemporaryPath + + }, innerProgress, cancellationToken); } catch (OperationCanceledException) { jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; } catch (Exception ex) { @@ -436,14 +543,14 @@ namespace MediaBrowser.Server.Implementations.Sync if (jobItem.Status == SyncJobItemStatus.Failed || jobItem.Status == SyncJobItemStatus.Queued) { - await _syncRepo.Update(jobItem).ConfigureAwait(false); + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); return; } + + jobItem.MediaSource = await GetEncodedMediaSource(jobItem.OutputPath, user, true).ConfigureAwait(false); } else { - jobItem.RequiresConversion = false; - if (mediaSource.Protocol == MediaProtocol.File) { jobItem.OutputPath = mediaSource.Path; @@ -456,42 +563,154 @@ namespace MediaBrowser.Server.Implementations.Sync { throw new InvalidOperationException(string.Format("Cannot direct stream {0} protocol", mediaSource.Protocol)); } + + jobItem.MediaSource = mediaSource; + } + + if (externalSubs.Count > 0) + { + // Save the job item now since conversion could take a while + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + await ConvertSubtitles(jobItem, externalSubs, streamInfo, cancellationToken).ConfigureAwait(false); } jobItem.Progress = 50; - jobItem.Status = SyncJobItemStatus.Transferring; - await _syncRepo.Update(jobItem).ConfigureAwait(false); + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + } + + private bool RequiresExtraction(SubtitleStreamInfo stream, MediaSourceInfo mediaSource) + { + var originalStream = mediaSource.MediaStreams.FirstOrDefault(i => i.Type == MediaStreamType.Subtitle && i.Index == stream.Index); + + return originalStream != null && !originalStream.IsExternal; } - private async Task Sync(SyncJobItem jobItem, Audio item, User user, DeviceProfile profile, IProgress<double> progress, CancellationToken cancellationToken) + private async Task ConvertSubtitles(SyncJobItem jobItem, + IEnumerable<SubtitleStreamInfo> subtitles, + StreamInfo streamInfo, + CancellationToken cancellationToken) { - var options = new AudioOptions + var files = new List<ItemFileInfo>(); + + var mediaStreams = jobItem.MediaSource.MediaStreams + .Where(i => i.Type != MediaStreamType.Subtitle || !i.IsExternal) + .ToList(); + + var startingIndex = mediaStreams.Count == 0 ? + 0 : + (mediaStreams.Select(i => i.Index).Max() + 1); + + foreach (var subtitle in subtitles) + { + var fileInfo = await ConvertSubtitles(jobItem.TemporaryPath, streamInfo, subtitle, cancellationToken).ConfigureAwait(false); + + // Reset this to a value that will be based on the output media + fileInfo.Index = startingIndex; + files.Add(fileInfo); + + mediaStreams.Add(new MediaStream + { + Index = startingIndex, + Codec = subtitle.Format, + IsForced = subtitle.IsForced, + IsExternal = true, + Language = subtitle.Language, + Path = fileInfo.Path, + SupportsExternalStream = true + }); + + startingIndex++; + } + + jobItem.AdditionalFiles.AddRange(files); + + jobItem.MediaSource.MediaStreams = mediaStreams; + } + + private async Task<ItemFileInfo> ConvertSubtitles(string temporaryPath, StreamInfo streamInfo, SubtitleStreamInfo subtitleStreamInfo, CancellationToken cancellationToken) + { + var subtitleStreamIndex = subtitleStreamInfo.Index; + + var filename = Guid.NewGuid() + "." + subtitleStreamInfo.Format.ToLower(); + + var path = Path.Combine(temporaryPath, filename); + + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + using (var stream = await _subtitleEncoder.GetSubtitles(streamInfo.ItemId, streamInfo.MediaSourceId, subtitleStreamIndex, subtitleStreamInfo.Format, 0, null, cancellationToken).ConfigureAwait(false)) { - Context = EncodingContext.Static, - ItemId = item.Id.ToString("N"), - DeviceId = jobItem.TargetId, - Profile = profile, - MediaSources = item.GetMediaSources(false, user).ToList() + using (var fs = _fileSystem.GetFileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, true)) + { + await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); + } + } + + return new ItemFileInfo + { + Name = Path.GetFileName(path), + Path = path, + Type = ItemFileType.Subtitles, + Index = subtitleStreamIndex }; + } + + private const int DatabaseProgressUpdateIntervalSeconds = 2; + + private async Task Sync(SyncJobItem jobItem, SyncJob job, Audio item, User user, DeviceProfile profile, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) + { + var options = _syncManager.GetAudioOptions(jobItem); + + options.DeviceId = jobItem.TargetId; + options.Context = EncodingContext.Static; + options.Profile = profile; + options.ItemId = item.Id.ToString("N"); + options.MediaSources = item.GetMediaSources(false, user).ToList(); var streamInfo = new StreamBuilder().BuildAudioItem(options); var mediaSource = streamInfo.MediaSource; jobItem.MediaSourceId = streamInfo.MediaSourceId; + jobItem.TemporaryPath = GetTemporaryPath(jobItem); - if (streamInfo.PlayMethod == PlayMethod.Transcode) + if (streamInfo.PlayMethod == PlayMethod.Transcode && job.Quality != SyncQuality.Original) { + if (!enableConversion) + { + return; + } + jobItem.Status = SyncJobItemStatus.Converting; - jobItem.RequiresConversion = true; - await _syncRepo.Update(jobItem).ConfigureAwait(false); + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(job).ConfigureAwait(false); try { - jobItem.OutputPath = await _mediaEncoder.EncodeAudio(new EncodingJobOptions(streamInfo, profile), progress, cancellationToken); + var lastJobUpdate = DateTime.MinValue; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(async pct => + { + progress.Report(pct); + + if ((DateTime.UtcNow - lastJobUpdate).TotalSeconds >= DatabaseProgressUpdateIntervalSeconds) + { + jobItem.Progress = pct / 2; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(job).ConfigureAwait(false); + } + }); + + jobItem.OutputPath = await _mediaEncoder.EncodeAudio(new EncodingJobOptions(streamInfo, profile) + { + OutputDirectory = jobItem.TemporaryPath + + }, innerProgress, cancellationToken); } catch (OperationCanceledException) { jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; } catch (Exception ex) { @@ -501,14 +720,14 @@ namespace MediaBrowser.Server.Implementations.Sync if (jobItem.Status == SyncJobItemStatus.Failed || jobItem.Status == SyncJobItemStatus.Queued) { - await _syncRepo.Update(jobItem).ConfigureAwait(false); + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); return; } + + jobItem.MediaSource = await GetEncodedMediaSource(jobItem.OutputPath, user, false).ConfigureAwait(false); } else { - jobItem.RequiresConversion = false; - if (mediaSource.Protocol == MediaProtocol.File) { jobItem.OutputPath = mediaSource.Path; @@ -521,11 +740,13 @@ namespace MediaBrowser.Server.Implementations.Sync { throw new InvalidOperationException(string.Format("Cannot direct stream {0} protocol", mediaSource.Protocol)); } + + jobItem.MediaSource = mediaSource; } jobItem.Progress = 50; - jobItem.Status = SyncJobItemStatus.Transferring; - await _syncRepo.Update(jobItem).ConfigureAwait(false); + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); } private async Task Sync(SyncJobItem jobItem, Photo item, DeviceProfile profile, CancellationToken cancellationToken) @@ -533,17 +754,17 @@ namespace MediaBrowser.Server.Implementations.Sync jobItem.OutputPath = item.Path; jobItem.Progress = 50; - jobItem.Status = SyncJobItemStatus.Transferring; - await _syncRepo.Update(jobItem).ConfigureAwait(false); + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); } - private async Task SyncGeneric(SyncJobItem jobItem, BaseItem item, DeviceProfile profile, CancellationToken cancellationToken) + private async Task SyncGeneric(SyncJobItem jobItem, BaseItem item, CancellationToken cancellationToken) { jobItem.OutputPath = item.Path; jobItem.Progress = 50; - jobItem.Status = SyncJobItemStatus.Transferring; - await _syncRepo.Update(jobItem).ConfigureAwait(false); + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); } private async Task<string> DownloadFile(SyncJobItem jobItem, MediaSourceInfo mediaSource, CancellationToken cancellationToken) @@ -551,5 +772,75 @@ namespace MediaBrowser.Server.Implementations.Sync // TODO: Download return mediaSource.Path; } + + public string GetTemporaryPath(SyncJob job) + { + return GetTemporaryPath(job.Id); + } + + public string GetTemporaryPath(string jobId) + { + var basePath = _config.GetSyncOptions().TemporaryPath; + + if (string.IsNullOrWhiteSpace(basePath)) + { + basePath = Path.Combine(_config.CommonApplicationPaths.ProgramDataPath, "sync"); + } + + return Path.Combine(basePath, jobId); + } + + public string GetTemporaryPath(SyncJobItem jobItem) + { + return Path.Combine(GetTemporaryPath(jobItem.JobId), jobItem.Id); + } + + private async Task<MediaSourceInfo> GetEncodedMediaSource(string path, User user, bool isVideo) + { + var item = _libraryManager.ResolvePath(new FileInfo(path)); + + await item.RefreshMetadata(CancellationToken.None).ConfigureAwait(false); + + var hasMediaSources = item as IHasMediaSources; + + var mediaSources = hasMediaSources.GetMediaSources(false).ToList(); + + var preferredAudio = string.IsNullOrEmpty(user.Configuration.AudioLanguagePreference) + ? new string[] { } + : new[] { user.Configuration.AudioLanguagePreference }; + + var preferredSubs = string.IsNullOrEmpty(user.Configuration.SubtitleLanguagePreference) + ? new List<string> { } + : new List<string> { user.Configuration.SubtitleLanguagePreference }; + + foreach (var source in mediaSources) + { + if (isVideo) + { + source.DefaultAudioStreamIndex = + MediaStreamSelector.GetDefaultAudioStreamIndex(source.MediaStreams, preferredAudio, user.Configuration.PlayDefaultAudioTrack); + + var defaultAudioIndex = source.DefaultAudioStreamIndex; + var audioLangage = defaultAudioIndex == null + ? null + : source.MediaStreams.Where(i => i.Type == MediaStreamType.Audio && i.Index == defaultAudioIndex).Select(i => i.Language).FirstOrDefault(); + + source.DefaultAudioStreamIndex = + MediaStreamSelector.GetDefaultSubtitleStreamIndex(source.MediaStreams, preferredSubs, user.Configuration.SubtitleMode, audioLangage); + } + else + { + var audio = source.MediaStreams.FirstOrDefault(i => i.Type == MediaStreamType.Audio); + + if (audio != null) + { + source.DefaultAudioStreamIndex = audio.Index; + } + + } + } + + return mediaSources.FirstOrDefault(); + } } } diff --git a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs index b8f199e6b..098e2b095 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs @@ -1,5 +1,8 @@ using MediaBrowser.Common; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Events; using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.IO; using MediaBrowser.Controller.Channels; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Dto; @@ -12,7 +15,9 @@ using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Sync; using MediaBrowser.Controller.TV; using MediaBrowser.Model.Dlna; +using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Events; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Querying; using MediaBrowser.Model.Sync; @@ -22,6 +27,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; namespace MediaBrowser.Server.Implementations.Sync @@ -37,10 +43,19 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly IApplicationHost _appHost; private readonly ITVSeriesManager _tvSeriesManager; private readonly Func<IMediaEncoder> _mediaEncoder; + private readonly IFileSystem _fileSystem; + private readonly Func<ISubtitleEncoder> _subtitleEncoder; + private readonly IConfigurationManager _config; private ISyncProvider[] _providers = { }; - public SyncManager(ILibraryManager libraryManager, ISyncRepository repo, IImageProcessor imageProcessor, ILogger logger, IUserManager userManager, Func<IDtoService> dtoService, IApplicationHost appHost, ITVSeriesManager tvSeriesManager, Func<IMediaEncoder> mediaEncoder) + public event EventHandler<GenericEventArgs<SyncJobCreationResult>> SyncJobCreated; + public event EventHandler<GenericEventArgs<SyncJob>> SyncJobCancelled; + public event EventHandler<GenericEventArgs<SyncJob>> SyncJobUpdated; + public event EventHandler<GenericEventArgs<SyncJobItem>> SyncJobItemUpdated; + public event EventHandler<GenericEventArgs<SyncJobItem>> SyncJobItemCreated; + + public SyncManager(ILibraryManager libraryManager, ISyncRepository repo, IImageProcessor imageProcessor, ILogger logger, IUserManager userManager, Func<IDtoService> dtoService, IApplicationHost appHost, ITVSeriesManager tvSeriesManager, Func<IMediaEncoder> mediaEncoder, IFileSystem fileSystem, Func<ISubtitleEncoder> subtitleEncoder, IConfigurationManager config) { _libraryManager = libraryManager; _repo = repo; @@ -51,6 +66,9 @@ namespace MediaBrowser.Server.Implementations.Sync _appHost = appHost; _tvSeriesManager = tvSeriesManager; _mediaEncoder = mediaEncoder; + _fileSystem = fileSystem; + _subtitleEncoder = subtitleEncoder; + _config = config; } public void AddParts(IEnumerable<ISyncProvider> providers) @@ -60,7 +78,7 @@ namespace MediaBrowser.Server.Implementations.Sync public async Task<SyncJobCreationResult> CreateJob(SyncJobRequest request) { - var processor = new SyncJobProcessor(_libraryManager, _repo, this, _logger, _userManager, _tvSeriesManager, _mediaEncoder()); + var processor = GetSyncJobProcessor(); var user = _userManager.GetUserById(request.UserId); @@ -124,13 +142,41 @@ namespace MediaBrowser.Server.Implementations.Sync await processor.EnsureJobItems(job).ConfigureAwait(false); - return new SyncJobCreationResult + // If it already has a converting status then is must have been aborted during conversion + var jobItemsResult = _repo.GetJobItems(new SyncJobItemQuery + { + Statuses = new List<SyncJobItemStatus> { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + JobId = jobId + }); + + await processor.SyncJobItems(jobItemsResult.Items, false, new Progress<double>(), CancellationToken.None) + .ConfigureAwait(false); + + jobItemsResult = _repo.GetJobItems(new SyncJobItemQuery + { + Statuses = new List<SyncJobItemStatus> { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + JobId = jobId + }); + + var returnResult = new SyncJobCreationResult { - Job = GetJob(jobId) + Job = GetJob(jobId), + JobItems = jobItemsResult.Items.ToList() }; + + if (SyncJobCreated != null) + { + EventHelper.FireEventIfNotNull(SyncJobCreated, this, new GenericEventArgs<SyncJobCreationResult> + { + Argument = returnResult + + }, _logger); + } + + return returnResult; } - public Task UpdateJob(SyncJob job) + public async Task UpdateJob(SyncJob job) { // Get fresh from the db and only update the fields that are supported to be changed. var instance = _repo.GetJob(job.Id); @@ -141,7 +187,47 @@ namespace MediaBrowser.Server.Implementations.Sync instance.SyncNewContent = job.SyncNewContent; instance.ItemLimit = job.ItemLimit; - return _repo.Update(instance); + await _repo.Update(instance).ConfigureAwait(false); + + OnSyncJobUpdated(instance); + } + + internal void OnSyncJobUpdated(SyncJob job) + { + if (SyncJobUpdated != null) + { + EventHelper.FireEventIfNotNull(SyncJobUpdated, this, new GenericEventArgs<SyncJob> + { + Argument = job + + }, _logger); + } + } + + internal async Task UpdateSyncJobItemInternal(SyncJobItem jobItem) + { + await _repo.Update(jobItem).ConfigureAwait(false); + + if (SyncJobUpdated != null) + { + EventHelper.FireEventIfNotNull(SyncJobItemUpdated, this, new GenericEventArgs<SyncJobItem> + { + Argument = jobItem + + }, _logger); + } + } + + internal void OnSyncJobItemCreated(SyncJobItem job) + { + if (SyncJobUpdated != null) + { + EventHelper.FireEventIfNotNull(SyncJobItemCreated, this, new GenericEventArgs<SyncJobItem> + { + Argument = job + + }, _logger); + } } public async Task<QueryResult<SyncJob>> GetJobs(SyncJobQuery query) @@ -158,13 +244,21 @@ namespace MediaBrowser.Server.Implementations.Sync private async Task FillMetadata(SyncJob job) { + var target = GetSyncTargets(job.UserId) + .FirstOrDefault(i => string.Equals(i.Id, job.TargetId, StringComparison.OrdinalIgnoreCase)); + + if (target != null) + { + job.TargetName = target.Name; + } + var item = job.RequestedItemIds .Select(_libraryManager.GetItemById) .FirstOrDefault(i => i != null); if (item == null) { - var processor = new SyncJobProcessor(_libraryManager, _repo, this, _logger, _userManager, _tvSeriesManager, _mediaEncoder()); + var processor = GetSyncJobProcessor(); var user = _userManager.GetUserById(job.UserId); @@ -255,9 +349,40 @@ namespace MediaBrowser.Server.Implementations.Sync } } - public Task CancelJob(string id) + public async Task CancelJob(string id) { - return _repo.DeleteJob(id); + var job = GetJob(id); + + if (job == null) + { + throw new ArgumentException("Job not found."); + } + + await _repo.DeleteJob(id).ConfigureAwait(false); + + var path = GetSyncJobProcessor().GetTemporaryPath(id); + + try + { + _fileSystem.DeleteDirectory(path, true); + } + catch (DirectoryNotFoundException) + { + + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting directory {0}", ex, path); + } + + if (SyncJobCancelled != null) + { + EventHelper.FireEventIfNotNull(SyncJobCancelled, this, new GenericEventArgs<SyncJob> + { + Argument = job + + }, _logger); + } } public SyncJob GetJob(string id) @@ -332,10 +457,30 @@ namespace MediaBrowser.Server.Implementations.Sync return false; } + if (video.VideoType == VideoType.BluRay || video.VideoType == VideoType.Dvd || video.VideoType == VideoType.HdDvd) + { + return false; + } + + if (video.IsPlaceHolder) + { + return false; + } + + if (video.IsArchive) + { + return false; + } + if (video.IsStacked) { return false; } + + if (video.IsShortcut) + { + return false; + } } var game = item as Game; @@ -392,11 +537,14 @@ namespace MediaBrowser.Server.Implementations.Sync jobItem.Status = SyncJobItemStatus.Synced; jobItem.Progress = 100; - if (jobItem.RequiresConversion) + if (!string.IsNullOrWhiteSpace(jobItem.TemporaryPath)) { try { - File.Delete(jobItem.OutputPath); + _fileSystem.DeleteDirectory(jobItem.TemporaryPath, true); + } + catch (DirectoryNotFoundException) + { } catch (Exception ex) { @@ -404,13 +552,18 @@ namespace MediaBrowser.Server.Implementations.Sync } } - await _repo.Update(jobItem).ConfigureAwait(false); + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); - var processor = new SyncJobProcessor(_libraryManager, _repo, this, _logger, _userManager, _tvSeriesManager, _mediaEncoder()); + var processor = GetSyncJobProcessor(); await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); } + private SyncJobProcessor GetSyncJobProcessor() + { + return new SyncJobProcessor(_libraryManager, _repo, this, _logger, _userManager, _tvSeriesManager, _mediaEncoder(), _subtitleEncoder(), _config, _fileSystem); + } + public SyncJobItem GetJobItem(string id) { return _repo.GetJobItem(id); @@ -432,14 +585,34 @@ namespace MediaBrowser.Server.Implementations.Sync { var job = _repo.GetJob(jobItem.JobId); + if (job == null) + { + _logger.Error("GetJobItemInfo job id {0} no longer exists", jobItem.JobId); + return null; + } + var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); + if (libraryItem == null) + { + _logger.Error("GetJobItemInfo library item with id {0} no longer exists", jobItem.ItemId); + return null; + } + var syncedItem = new SyncedItem { SyncJobId = jobItem.JobId, SyncJobItemId = jobItem.Id, ServerId = _appHost.SystemId, - UserId = job.UserId + UserId = job.UserId, + AdditionalFiles = jobItem.AdditionalFiles.Select(i => new ItemFileInfo + { + ImageType = i.ImageType, + Name = i.Name, + Type = i.Type, + Index = i.Index + + }).ToList() }; var dtoOptions = new DtoOptions(); @@ -456,14 +629,11 @@ namespace MediaBrowser.Server.Implementations.Sync syncedItem.Item = _dtoService().GetBaseItemDto(libraryItem, dtoOptions); - // TODO: this should be the media source of the transcoded output - syncedItem.Item.MediaSources = syncedItem.Item.MediaSources - .Where(i => string.Equals(i.Id, jobItem.MediaSourceId)) - .ToList(); - var mediaSource = syncedItem.Item.MediaSources .FirstOrDefault(i => string.Equals(i.Id, jobItem.MediaSourceId)); + syncedItem.Item.MediaSources = new List<MediaSourceInfo>(); + // This will be null for items that are not audio/video if (mediaSource == null) { @@ -472,6 +642,7 @@ namespace MediaBrowser.Server.Implementations.Sync else { syncedItem.OriginalFileName = Path.GetFileName(mediaSource.Path); + syncedItem.Item.MediaSources.Add(mediaSource); } return syncedItem; @@ -487,10 +658,15 @@ namespace MediaBrowser.Server.Implementations.Sync var jobItemResult = GetJobItems(new SyncJobItemQuery { TargetId = targetId, - Statuses = new List<SyncJobItemStatus> { SyncJobItemStatus.Transferring } + Statuses = new List<SyncJobItemStatus> + { + SyncJobItemStatus.ReadyToTransfer + } }); - return jobItemResult.Items.Select(GetJobItemInfo) + return jobItemResult.Items + .Select(GetJobItemInfo) + .Where(i => i != null) .ToList(); } @@ -511,7 +687,12 @@ namespace MediaBrowser.Server.Implementations.Sync var job = _repo.GetJob(jobItem.JobId); var user = _userManager.GetUserById(job.UserId); - if (user == null) + if (jobItem.IsMarkedForRemoval) + { + // Tell the device to remove it since it has been marked for removal + response.ItemIdsToRemove.Add(jobItem.ItemId); + } + else if (user == null) { // Tell the device to remove it since the user is gone now response.ItemIdsToRemove.Add(jobItem.ItemId); @@ -539,7 +720,7 @@ namespace MediaBrowser.Server.Implementations.Sync { // Content is no longer on the device jobItem.Status = SyncJobItemStatus.RemovedFromDevice; - await _repo.Update(jobItem).ConfigureAwait(false); + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); } } @@ -561,9 +742,51 @@ namespace MediaBrowser.Server.Implementations.Sync response.ItemIdsToRemove = response.ItemIdsToRemove.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + var itemsOnDevice = request.LocalItemIds + .Except(response.ItemIdsToRemove) + .ToList(); + + SetUserAccess(request, response, itemsOnDevice); + return response; } + private void SetUserAccess(SyncDataRequest request, SyncDataResponse response, List<string> itemIds) + { + var users = request.OfflineUserIds + .Select(_userManager.GetUserById) + .Where(i => i != null) + .ToList(); + + foreach (var itemId in itemIds) + { + var item = _libraryManager.GetItemById(itemId); + + if (item != null) + { + var usersWithAccess = new List<User>(); + + foreach (var user in users) + { + if (IsUserVisible(item, user)) + { + usersWithAccess.Add(user); + } + } + + response.ItemUserAccess[itemId] = users + .Select(i => i.Id.ToString("N")) + .OrderBy(i => i) + .ToList(); + } + } + } + + private bool IsUserVisible(BaseItem item, User user) + { + return item.IsVisibleStandalone(user); + } + private bool IsLibraryItemAvailable(BaseItem item) { if (item == null) @@ -575,5 +798,161 @@ namespace MediaBrowser.Server.Implementations.Sync return true; } + + public async Task ReEnableJobItem(string id) + { + var jobItem = _repo.GetJobItem(id); + + if (jobItem.Status != SyncJobItemStatus.Failed && jobItem.Status != SyncJobItemStatus.Cancelled) + { + throw new ArgumentException("Operation is not valid for this job item"); + } + + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + jobItem.IsMarkedForRemoval = false; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public async Task CancelJobItem(string id) + { + var jobItem = _repo.GetJobItem(id); + + if (jobItem.Status != SyncJobItemStatus.Queued && jobItem.Status != SyncJobItemStatus.ReadyToTransfer && jobItem.Status != SyncJobItemStatus.Converting) + { + throw new ArgumentException("Operation is not valid for this job item"); + } + + jobItem.Status = SyncJobItemStatus.Cancelled; + jobItem.Progress = 0; + jobItem.IsMarkedForRemoval = true; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + + var path = processor.GetTemporaryPath(jobItem); + + try + { + _fileSystem.DeleteDirectory(path, true); + } + catch (DirectoryNotFoundException) + { + + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting directory {0}", ex, path); + } + } + + public async Task MarkJobItemForRemoval(string id) + { + var jobItem = _repo.GetJobItem(id); + + if (jobItem.Status != SyncJobItemStatus.Synced) + { + throw new ArgumentException("Operation is not valid for this job item"); + } + + jobItem.IsMarkedForRemoval = true; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public async Task UnmarkJobItemForRemoval(string id) + { + var jobItem = _repo.GetJobItem(id); + + if (jobItem.Status != SyncJobItemStatus.Synced) + { + throw new ArgumentException("Operation is not valid for this job item"); + } + + jobItem.IsMarkedForRemoval = false; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public async Task ReportSyncJobItemTransferBeginning(string id) + { + var jobItem = _repo.GetJobItem(id); + + jobItem.Status = SyncJobItemStatus.Transferring; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public async Task ReportSyncJobItemTransferFailed(string id) + { + var jobItem = _repo.GetJobItem(id); + + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public QueryResult<string> GetLibraryItemIds(SyncJobItemQuery query) + { + return _repo.GetLibraryItemIds(query); + } + + public AudioOptions GetAudioOptions(SyncJobItem jobItem) + { + var profile = GetDeviceProfile(jobItem.TargetId); + + return new AudioOptions + { + Profile = profile + }; + } + + public VideoOptions GetVideoOptions(SyncJobItem jobItem, SyncJob job) + { + var profile = GetDeviceProfile(jobItem.TargetId); + var maxBitrate = profile.MaxStaticBitrate; + + if (maxBitrate.HasValue) + { + if (job.Quality == SyncQuality.Medium) + { + maxBitrate = Convert.ToInt32(maxBitrate.Value * .75); + } + else if (job.Quality == SyncQuality.Low) + { + maxBitrate = Convert.ToInt32(maxBitrate.Value * .5); + } + } + + return new VideoOptions + { + Profile = profile, + MaxBitrate = maxBitrate + }; + } } } diff --git a/MediaBrowser.Server.Implementations/Sync/SyncRegistrationInfo.cs b/MediaBrowser.Server.Implementations/Sync/SyncRegistrationInfo.cs new file mode 100644 index 000000000..40b84b1c2 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Sync/SyncRegistrationInfo.cs @@ -0,0 +1,31 @@ +using MediaBrowser.Common.Security; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Sync +{ + public class SyncRegistrationInfo : IRequiresRegistration + { + private readonly ISecurityManager _securityManager; + + public static SyncRegistrationInfo Instance; + + public SyncRegistrationInfo(ISecurityManager securityManager) + { + _securityManager = securityManager; + Instance = this; + } + + private bool _registered; + public bool IsRegistered + { + get { return _registered; } + } + + public async Task LoadRegistrationInfoAsync() + { + var info = await _securityManager.GetRegistrationStatus("sync").ConfigureAwait(false); + + _registered = info.IsValid; + } + } +} diff --git a/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs b/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs index 151860d84..05d804cbb 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs @@ -1,7 +1,9 @@ using MediaBrowser.Controller; using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Dto; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Querying; +using MediaBrowser.Model.Serialization; using MediaBrowser.Model.Sync; using MediaBrowser.Server.Implementations.Persistence; using System; @@ -23,20 +25,26 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly IServerApplicationPaths _appPaths; private readonly CultureInfo _usCulture = new CultureInfo("en-US"); + private IDbCommand _insertJobCommand; + private IDbCommand _updateJobCommand; private IDbCommand _deleteJobCommand; + private IDbCommand _deleteJobItemsCommand; - private IDbCommand _saveJobCommand; - private IDbCommand _saveJobItemCommand; + private IDbCommand _insertJobItemCommand; + private IDbCommand _updateJobItemCommand; + + private readonly IJsonSerializer _json; - public SyncRepository(ILogger logger, IServerApplicationPaths appPaths) + public SyncRepository(ILogger logger, IServerApplicationPaths appPaths, IJsonSerializer json) { _logger = logger; _appPaths = appPaths; + _json = json; } public async Task Initialize() { - var dbFile = Path.Combine(_appPaths.DataPath, "sync11.db"); + var dbFile = Path.Combine(_appPaths.DataPath, "sync14.db"); _connection = await SqliteExtensions.ConnectToDb(dbFile, _logger).ConfigureAwait(false); @@ -45,7 +53,7 @@ namespace MediaBrowser.Server.Implementations.Sync "create table if not exists SyncJobs (Id GUID PRIMARY KEY, TargetId TEXT NOT NULL, Name TEXT NOT NULL, Quality TEXT NOT NULL, Status TEXT NOT NULL, Progress FLOAT, UserId TEXT NOT NULL, ItemIds TEXT NOT NULL, Category TEXT, ParentId TEXT, UnwatchedOnly BIT, ItemLimit INT, SyncNewContent BIT, DateCreated DateTime, DateLastModified DateTime, ItemCount int)", "create index if not exists idx_SyncJobs on SyncJobs(Id)", - "create table if not exists SyncJobItems (Id GUID PRIMARY KEY, ItemId TEXT, ItemName TEXT, MediaSourceId TEXT, JobId TEXT, RequiresConversion BIT, OutputPath TEXT, Status TEXT, TargetId TEXT, DateCreated DateTime, Progress FLOAT)", + "create table if not exists SyncJobItems (Id GUID PRIMARY KEY, ItemId TEXT, ItemName TEXT, MediaSourceId TEXT, JobId TEXT, TemporaryPath TEXT, OutputPath TEXT, Status TEXT, TargetId TEXT, DateCreated DateTime, Progress FLOAT, AdditionalFiles TEXT, MediaSource TEXT, IsMarkedForRemoval BIT, JobItemIndex INT)", "create index if not exists idx_SyncJobItems on SyncJobs(Id)", //pragmas @@ -61,52 +69,101 @@ namespace MediaBrowser.Server.Implementations.Sync private void PrepareStatements() { + // _deleteJobCommand _deleteJobCommand = _connection.CreateCommand(); _deleteJobCommand.CommandText = "delete from SyncJobs where Id=@Id"; _deleteJobCommand.Parameters.Add(_deleteJobCommand, "@Id"); + // _deleteJobItemsCommand _deleteJobItemsCommand = _connection.CreateCommand(); _deleteJobItemsCommand.CommandText = "delete from SyncJobItems where JobId=@JobId"; _deleteJobItemsCommand.Parameters.Add(_deleteJobItemsCommand, "@JobId"); - _saveJobCommand = _connection.CreateCommand(); - _saveJobCommand.CommandText = "replace into SyncJobs (Id, TargetId, Name, Quality, Status, Progress, UserId, ItemIds, Category, ParentId, UnwatchedOnly, ItemLimit, SyncNewContent, DateCreated, DateLastModified, ItemCount) values (@Id, @TargetId, @Name, @Quality, @Status, @Progress, @UserId, @ItemIds, @Category, @ParentId, @UnwatchedOnly, @ItemLimit, @SyncNewContent, @DateCreated, @DateLastModified, @ItemCount)"; - - _saveJobCommand.Parameters.Add(_saveJobCommand, "@Id"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@TargetId"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@Name"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@Quality"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@Status"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@Progress"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@UserId"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@ItemIds"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@Category"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@ParentId"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@UnwatchedOnly"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@ItemLimit"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@SyncNewContent"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@DateCreated"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@DateLastModified"); - _saveJobCommand.Parameters.Add(_saveJobCommand, "@ItemCount"); - - _saveJobItemCommand = _connection.CreateCommand(); - _saveJobItemCommand.CommandText = "replace into SyncJobItems (Id, ItemId, ItemName, MediaSourceId, JobId, RequiresConversion, OutputPath, Status, TargetId, DateCreated, Progress) values (@Id, @ItemId, @ItemName, @MediaSourceId, @JobId, @RequiresConversion, @OutputPath, @Status, @TargetId, @DateCreated, @Progress)"; - - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@Id"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@ItemId"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@ItemName"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@MediaSourceId"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@JobId"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@RequiresConversion"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@OutputPath"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@Status"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@TargetId"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@DateCreated"); - _saveJobItemCommand.Parameters.Add(_saveJobItemCommand, "@Progress"); + // _insertJobCommand + _insertJobCommand = _connection.CreateCommand(); + _insertJobCommand.CommandText = "insert into SyncJobs (Id, TargetId, Name, Quality, Status, Progress, UserId, ItemIds, Category, ParentId, UnwatchedOnly, ItemLimit, SyncNewContent, DateCreated, DateLastModified, ItemCount) values (@Id, @TargetId, @Name, @Quality, @Status, @Progress, @UserId, @ItemIds, @Category, @ParentId, @UnwatchedOnly, @ItemLimit, @SyncNewContent, @DateCreated, @DateLastModified, @ItemCount)"; + + _insertJobCommand.Parameters.Add(_insertJobCommand, "@Id"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@TargetId"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@Name"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@Quality"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@Status"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@Progress"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@UserId"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@ItemIds"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@Category"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@ParentId"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@UnwatchedOnly"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@ItemLimit"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@SyncNewContent"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@DateCreated"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@DateLastModified"); + _insertJobCommand.Parameters.Add(_insertJobCommand, "@ItemCount"); + + // _updateJobCommand + _updateJobCommand = _connection.CreateCommand(); + _updateJobCommand.CommandText = "update SyncJobs set TargetId=@TargetId,Name=@Name,Quality=@Quality,Status=@Status,Progress=@Progress,UserId=@UserId,ItemIds=@ItemIds,Category=@Category,ParentId=@ParentId,UnwatchedOnly=@UnwatchedOnly,ItemLimit=@ItemLimit,SyncNewContent=@SyncNewContent,DateCreated=@DateCreated,DateLastModified=@DateLastModified,ItemCount=@ItemCount where Id=@ID"; + + _updateJobCommand.Parameters.Add(_updateJobCommand, "@Id"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@TargetId"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@Name"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@Quality"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@Status"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@Progress"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@UserId"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@ItemIds"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@Category"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@ParentId"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@UnwatchedOnly"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@ItemLimit"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@SyncNewContent"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@DateCreated"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@DateLastModified"); + _updateJobCommand.Parameters.Add(_updateJobCommand, "@ItemCount"); + + // _insertJobItemCommand + _insertJobItemCommand = _connection.CreateCommand(); + _insertJobItemCommand.CommandText = "insert into SyncJobItems (Id, ItemId, ItemName, MediaSourceId, JobId, TemporaryPath, OutputPath, Status, TargetId, DateCreated, Progress, AdditionalFiles, MediaSource, IsMarkedForRemoval, JobItemIndex) values (@Id, @ItemId, @ItemName, @MediaSourceId, @JobId, @TemporaryPath, @OutputPath, @Status, @TargetId, @DateCreated, @Progress, @AdditionalFiles, @MediaSource, @IsMarkedForRemoval, @JobItemIndex)"; + + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@Id"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@ItemId"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@ItemName"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@MediaSourceId"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@JobId"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@TemporaryPath"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@OutputPath"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@Status"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@TargetId"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@DateCreated"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@Progress"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@AdditionalFiles"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@MediaSource"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@IsMarkedForRemoval"); + _insertJobItemCommand.Parameters.Add(_insertJobItemCommand, "@JobItemIndex"); + + // _updateJobItemCommand + _updateJobItemCommand = _connection.CreateCommand(); + _updateJobItemCommand.CommandText = "update SyncJobItems set ItemId=@ItemId,ItemName=@ItemName,MediaSourceId=@MediaSourceId,JobId=@JobId,TemporaryPath=@TemporaryPath,OutputPath=@OutputPath,Status=@Status,TargetId=@TargetId,DateCreated=@DateCreated,Progress=@Progress,AdditionalFiles=@AdditionalFiles,MediaSource=@MediaSource,IsMarkedForRemoval=@IsMarkedForRemoval,JobItemIndex=@JobItemIndex where Id=@Id"; + + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@Id"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@ItemId"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@ItemName"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@MediaSourceId"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@JobId"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@TemporaryPath"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@OutputPath"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@Status"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@TargetId"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@DateCreated"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@Progress"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@AdditionalFiles"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@MediaSource"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@IsMarkedForRemoval"); + _updateJobItemCommand.Parameters.Add(_updateJobItemCommand, "@JobItemIndex"); } private const string BaseJobSelectText = "select Id, TargetId, Name, Quality, Status, Progress, UserId, ItemIds, Category, ParentId, UnwatchedOnly, ItemLimit, SyncNewContent, DateCreated, DateLastModified, ItemCount from SyncJobs"; - private const string BaseJobItemSelectText = "select Id, ItemId, ItemName, MediaSourceId, JobId, RequiresConversion, OutputPath, Status, TargetId, DateCreated, Progress from SyncJobItems"; + private const string BaseJobItemSelectText = "select Id, ItemId, ItemName, MediaSourceId, JobId, TemporaryPath, OutputPath, Status, TargetId, DateCreated, Progress, AdditionalFiles, MediaSource, IsMarkedForRemoval, JobItemIndex from SyncJobItems"; public SyncJob GetJob(string id) { @@ -115,6 +172,8 @@ namespace MediaBrowser.Server.Implementations.Sync throw new ArgumentNullException("id"); } + CheckDisposed(); + var guid = new Guid(id); if (guid == Guid.Empty) @@ -205,16 +264,23 @@ namespace MediaBrowser.Server.Implementations.Sync public Task Create(SyncJob job) { - return Update(job); + return InsertOrUpdate(job, _insertJobCommand); } - public async Task Update(SyncJob job) + public Task Update(SyncJob job) + { + return InsertOrUpdate(job, _updateJobCommand); + } + + private async Task InsertOrUpdate(SyncJob job, IDbCommand cmd) { if (job == null) { throw new ArgumentNullException("job"); } + CheckDisposed(); + await _writeLock.WaitAsync().ConfigureAwait(false); IDbTransaction transaction = null; @@ -225,26 +291,26 @@ namespace MediaBrowser.Server.Implementations.Sync var index = 0; - _saveJobCommand.GetParameter(index++).Value = new Guid(job.Id); - _saveJobCommand.GetParameter(index++).Value = job.TargetId; - _saveJobCommand.GetParameter(index++).Value = job.Name; - _saveJobCommand.GetParameter(index++).Value = job.Quality; - _saveJobCommand.GetParameter(index++).Value = job.Status.ToString(); - _saveJobCommand.GetParameter(index++).Value = job.Progress; - _saveJobCommand.GetParameter(index++).Value = job.UserId; - _saveJobCommand.GetParameter(index++).Value = string.Join(",", job.RequestedItemIds.ToArray()); - _saveJobCommand.GetParameter(index++).Value = job.Category; - _saveJobCommand.GetParameter(index++).Value = job.ParentId; - _saveJobCommand.GetParameter(index++).Value = job.UnwatchedOnly; - _saveJobCommand.GetParameter(index++).Value = job.ItemLimit; - _saveJobCommand.GetParameter(index++).Value = job.SyncNewContent; - _saveJobCommand.GetParameter(index++).Value = job.DateCreated; - _saveJobCommand.GetParameter(index++).Value = job.DateLastModified; - _saveJobCommand.GetParameter(index++).Value = job.ItemCount; - - _saveJobCommand.Transaction = transaction; - - _saveJobCommand.ExecuteNonQuery(); + cmd.GetParameter(index++).Value = new Guid(job.Id); + cmd.GetParameter(index++).Value = job.TargetId; + cmd.GetParameter(index++).Value = job.Name; + cmd.GetParameter(index++).Value = job.Quality; + cmd.GetParameter(index++).Value = job.Status.ToString(); + cmd.GetParameter(index++).Value = job.Progress; + cmd.GetParameter(index++).Value = job.UserId; + cmd.GetParameter(index++).Value = string.Join(",", job.RequestedItemIds.ToArray()); + cmd.GetParameter(index++).Value = job.Category; + cmd.GetParameter(index++).Value = job.ParentId; + cmd.GetParameter(index++).Value = job.UnwatchedOnly; + cmd.GetParameter(index++).Value = job.ItemLimit; + cmd.GetParameter(index++).Value = job.SyncNewContent; + cmd.GetParameter(index++).Value = job.DateCreated; + cmd.GetParameter(index++).Value = job.DateLastModified; + cmd.GetParameter(index++).Value = job.ItemCount; + + cmd.Transaction = transaction; + + cmd.ExecuteNonQuery(); transaction.Commit(); } @@ -286,6 +352,8 @@ namespace MediaBrowser.Server.Implementations.Sync throw new ArgumentNullException("id"); } + CheckDisposed(); + await _writeLock.WaitAsync().ConfigureAwait(false); IDbTransaction transaction = null; @@ -345,34 +413,34 @@ namespace MediaBrowser.Server.Implementations.Sync throw new ArgumentNullException("query"); } + CheckDisposed(); + using (var cmd = _connection.CreateCommand()) { cmd.CommandText = BaseJobSelectText; var whereClauses = new List<string>(); - if (query.IsCompleted.HasValue) + if (query.Statuses.Count > 0) { - if (query.IsCompleted.Value) - { - whereClauses.Add("Status=@Status"); - } - else - { - whereClauses.Add("Status<>@Status"); - } - cmd.Parameters.Add(cmd, "@Status", DbType.String).Value = SyncJobStatus.Completed.ToString(); + var statuses = string.Join(",", query.Statuses.Select(i => "'" + i.ToString() + "'").ToArray()); + + whereClauses.Add(string.Format("Status in ({0})", statuses)); } if (!string.IsNullOrWhiteSpace(query.TargetId)) { whereClauses.Add("TargetId=@TargetId"); - cmd.Parameters.Add(cmd, "@TargetId", DbType.String).Value = query.TargetId; } if (!string.IsNullOrWhiteSpace(query.UserId)) { whereClauses.Add("UserId=@UserId"); cmd.Parameters.Add(cmd, "@UserId", DbType.String).Value = query.UserId; } + if (query.SyncNewContent.HasValue) + { + whereClauses.Add("SyncNewContent=@SyncNewContent"); + cmd.Parameters.Add(cmd, "@SyncNewContent", DbType.Boolean).Value = query.SyncNewContent.Value; + } var whereTextWithoutPaging = whereClauses.Count == 0 ? string.Empty : @@ -381,7 +449,7 @@ namespace MediaBrowser.Server.Implementations.Sync var startIndex = query.StartIndex ?? 0; if (startIndex > 0) { - whereClauses.Add(string.Format("Id NOT IN (SELECT Id FROM SyncJobs ORDER BY DateLastModified DESC LIMIT {0})", + whereClauses.Add(string.Format("Id NOT IN (SELECT Id FROM SyncJobs ORDER BY (Select Max(DateLastModified) from SyncJobs where TargetId=@TargetId) DESC, DateLastModified DESC LIMIT {0})", startIndex.ToString(_usCulture))); } @@ -390,7 +458,8 @@ namespace MediaBrowser.Server.Implementations.Sync cmd.CommandText += " where " + string.Join(" AND ", whereClauses.ToArray()); } - cmd.CommandText += " ORDER BY DateLastModified DESC"; + cmd.CommandText += " ORDER BY (Select Max(DateLastModified) from SyncJobs where TargetId=@TargetId) DESC, DateLastModified DESC"; + cmd.Parameters.Add(cmd, "@TargetId", DbType.String).Value = query.TargetId; if (query.Limit.HasValue) { @@ -430,6 +499,8 @@ namespace MediaBrowser.Server.Implementations.Sync throw new ArgumentNullException("id"); } + CheckDisposed(); + var guid = new Guid(id); using (var cmd = _connection.CreateCommand()) @@ -450,7 +521,7 @@ namespace MediaBrowser.Server.Implementations.Sync return null; } - public QueryResult<SyncJobItem> GetJobItems(SyncJobItemQuery query) + private QueryResult<T> GetJobItemReader<T>(SyncJobItemQuery query, string baseSelectText, Func<IDataReader, T> itemFactory) { if (query == null) { @@ -459,7 +530,7 @@ namespace MediaBrowser.Server.Implementations.Sync using (var cmd = _connection.CreateCommand()) { - cmd.CommandText = BaseJobItemSelectText; + cmd.CommandText = baseSelectText; var whereClauses = new List<string>(); @@ -488,7 +559,7 @@ namespace MediaBrowser.Server.Implementations.Sync var startIndex = query.StartIndex ?? 0; if (startIndex > 0) { - whereClauses.Add(string.Format("Id NOT IN (SELECT Id FROM SyncJobItems ORDER BY DateCreated LIMIT {0})", + whereClauses.Add(string.Format("Id NOT IN (SELECT Id FROM SyncJobItems ORDER BY JobItemIndex, DateCreated LIMIT {0})", startIndex.ToString(_usCulture))); } @@ -497,7 +568,7 @@ namespace MediaBrowser.Server.Implementations.Sync cmd.CommandText += " where " + string.Join(" AND ", whereClauses.ToArray()); } - cmd.CommandText += " ORDER BY DateCreated"; + cmd.CommandText += " ORDER BY JobItemIndex, DateCreated"; if (query.Limit.HasValue) { @@ -506,14 +577,14 @@ namespace MediaBrowser.Server.Implementations.Sync cmd.CommandText += "; select count (Id) from SyncJobItems" + whereTextWithoutPaging; - var list = new List<SyncJobItem>(); + var list = new List<T>(); var count = 0; using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess)) { while (reader.Read()) { - list.Add(GetJobItem(reader)); + list.Add(itemFactory(reader)); } if (reader.NextResult() && reader.Read()) @@ -522,7 +593,7 @@ namespace MediaBrowser.Server.Implementations.Sync } } - return new QueryResult<SyncJobItem>() + return new QueryResult<T>() { Items = list.ToArray(), TotalRecordCount = count @@ -530,18 +601,35 @@ namespace MediaBrowser.Server.Implementations.Sync } } + public QueryResult<string> GetLibraryItemIds(SyncJobItemQuery query) + { + return GetJobItemReader(query, "select ItemId from SyncJobItems", GetItemId); + } + + public QueryResult<SyncJobItem> GetJobItems(SyncJobItemQuery query) + { + return GetJobItemReader(query, BaseJobItemSelectText, GetJobItem); + } + public Task Create(SyncJobItem jobItem) { - return Update(jobItem); + return InsertOrUpdate(jobItem, _insertJobItemCommand); } - public async Task Update(SyncJobItem jobItem) + public Task Update(SyncJobItem jobItem) + { + return InsertOrUpdate(jobItem, _updateJobItemCommand); + } + + private async Task InsertOrUpdate(SyncJobItem jobItem, IDbCommand cmd) { if (jobItem == null) { throw new ArgumentNullException("jobItem"); } + CheckDisposed(); + await _writeLock.WaitAsync().ConfigureAwait(false); IDbTransaction transaction = null; @@ -552,21 +640,25 @@ namespace MediaBrowser.Server.Implementations.Sync var index = 0; - _saveJobItemCommand.GetParameter(index++).Value = new Guid(jobItem.Id); - _saveJobItemCommand.GetParameter(index++).Value = jobItem.ItemId; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.ItemName; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.MediaSourceId; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.JobId; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.RequiresConversion; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.OutputPath; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.Status.ToString(); - _saveJobItemCommand.GetParameter(index++).Value = jobItem.TargetId; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.DateCreated; - _saveJobItemCommand.GetParameter(index++).Value = jobItem.Progress; - - _saveJobItemCommand.Transaction = transaction; - - _saveJobItemCommand.ExecuteNonQuery(); + cmd.GetParameter(index++).Value = new Guid(jobItem.Id); + cmd.GetParameter(index++).Value = jobItem.ItemId; + cmd.GetParameter(index++).Value = jobItem.ItemName; + cmd.GetParameter(index++).Value = jobItem.MediaSourceId; + cmd.GetParameter(index++).Value = jobItem.JobId; + cmd.GetParameter(index++).Value = jobItem.TemporaryPath; + cmd.GetParameter(index++).Value = jobItem.OutputPath; + cmd.GetParameter(index++).Value = jobItem.Status.ToString(); + cmd.GetParameter(index++).Value = jobItem.TargetId; + cmd.GetParameter(index++).Value = jobItem.DateCreated; + cmd.GetParameter(index++).Value = jobItem.Progress; + cmd.GetParameter(index++).Value = _json.SerializeToString(jobItem.AdditionalFiles); + cmd.GetParameter(index++).Value = jobItem.MediaSource == null ? null : _json.SerializeToString(jobItem.MediaSource); + cmd.GetParameter(index++).Value = jobItem.IsMarkedForRemoval; + cmd.GetParameter(index++).Value = jobItem.JobItemIndex; + + cmd.Transaction = transaction; + + cmd.ExecuteNonQuery(); transaction.Commit(); } @@ -620,8 +712,11 @@ namespace MediaBrowser.Server.Implementations.Sync } info.JobId = reader.GetString(4); - info.RequiresConversion = reader.GetBoolean(5); + if (!reader.IsDBNull(5)) + { + info.TemporaryPath = reader.GetString(5); + } if (!reader.IsDBNull(6)) { info.OutputPath = reader.GetString(6); @@ -634,16 +729,44 @@ namespace MediaBrowser.Server.Implementations.Sync info.TargetId = reader.GetString(8); - info.DateCreated = reader.GetDateTime(9); + info.DateCreated = reader.GetDateTime(9).ToUniversalTime(); if (!reader.IsDBNull(10)) { info.Progress = reader.GetDouble(10); } + if (!reader.IsDBNull(11)) + { + var json = reader.GetString(11); + + if (!string.IsNullOrWhiteSpace(json)) + { + info.AdditionalFiles = _json.DeserializeFromString<List<ItemFileInfo>>(json); + } + } + + if (!reader.IsDBNull(12)) + { + var json = reader.GetString(12); + + if (!string.IsNullOrWhiteSpace(json)) + { + info.MediaSource = _json.DeserializeFromString<MediaSourceInfo>(json); + } + } + + info.IsMarkedForRemoval = reader.GetBoolean(13); + info.JobItemIndex = reader.GetInt32(14); + return info; } + private string GetItemId(IDataReader reader) + { + return reader.GetString(0); + } + /// <summary> /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// </summary> @@ -653,6 +776,15 @@ namespace MediaBrowser.Server.Implementations.Sync GC.SuppressFinalize(this); } + private bool _disposed; + private void CheckDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(GetType().Name + " has been disposed and cannot be accessed."); + } + } + private readonly object _disposeLock = new object(); /// <summary> @@ -663,6 +795,8 @@ namespace MediaBrowser.Server.Implementations.Sync { if (dispose) { + _disposed = true; + try { lock (_disposeLock) diff --git a/MediaBrowser.Server.Implementations/Sync/SyncScheduledTask.cs b/MediaBrowser.Server.Implementations/Sync/SyncScheduledTask.cs index c2925551b..ccc9508e8 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncScheduledTask.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncScheduledTask.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.ScheduledTasks; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.IO; +using MediaBrowser.Common.ScheduledTasks; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Sync; @@ -11,7 +13,7 @@ using System.Threading.Tasks; namespace MediaBrowser.Server.Implementations.Sync { - public class SyncScheduledTask : IScheduledTask, IConfigurableScheduledTask + public class SyncScheduledTask : IScheduledTask, IConfigurableScheduledTask, IHasKey { private readonly ILibraryManager _libraryManager; private readonly ISyncRepository _syncRepo; @@ -20,8 +22,11 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly IUserManager _userManager; private readonly ITVSeriesManager _tvSeriesManager; private readonly IMediaEncoder _mediaEncoder; + private readonly ISubtitleEncoder _subtitleEncoder; + private readonly IConfigurationManager _config; + private readonly IFileSystem _fileSystem; - public SyncScheduledTask(ILibraryManager libraryManager, ISyncRepository syncRepo, ISyncManager syncManager, ILogger logger, IUserManager userManager, ITVSeriesManager tvSeriesManager, IMediaEncoder mediaEncoder) + public SyncScheduledTask(ILibraryManager libraryManager, ISyncRepository syncRepo, ISyncManager syncManager, ILogger logger, IUserManager userManager, ITVSeriesManager tvSeriesManager, IMediaEncoder mediaEncoder, ISubtitleEncoder subtitleEncoder, IConfigurationManager config, IFileSystem fileSystem) { _libraryManager = libraryManager; _syncRepo = syncRepo; @@ -30,11 +35,14 @@ namespace MediaBrowser.Server.Implementations.Sync _userManager = userManager; _tvSeriesManager = tvSeriesManager; _mediaEncoder = mediaEncoder; + _subtitleEncoder = subtitleEncoder; + _config = config; + _fileSystem = fileSystem; } public string Name { - get { return "Sync preparation"; } + get { return "Convert media"; } } public string Description @@ -46,14 +54,14 @@ namespace MediaBrowser.Server.Implementations.Sync { get { - return "Library"; + return "Sync"; } } public Task Execute(CancellationToken cancellationToken, IProgress<double> progress) { - return new SyncJobProcessor(_libraryManager, _syncRepo, _syncManager, _logger, _userManager, _tvSeriesManager, _mediaEncoder).Sync(progress, - cancellationToken); + return new SyncJobProcessor(_libraryManager, _syncRepo, (SyncManager)_syncManager, _logger, _userManager, _tvSeriesManager, _mediaEncoder, _subtitleEncoder, _config, _fileSystem) + .Sync(progress, cancellationToken); } public IEnumerable<ITaskTrigger> GetDefaultTriggers() @@ -67,12 +75,17 @@ namespace MediaBrowser.Server.Implementations.Sync public bool IsHidden { - get { return true; } + get { return false; } } public bool IsEnabled { get { return true; } } + + public string Key + { + get { return "SyncPrepare"; } + } } } |