diff options
Diffstat (limited to 'MediaBrowser.Server.Implementations/Sync')
9 files changed, 328 insertions, 255 deletions
diff --git a/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs b/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs index 99d758233..7b1fa4dec 100644 --- a/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs +++ b/MediaBrowser.Server.Implementations/Sync/AppSyncProvider.cs @@ -44,7 +44,7 @@ namespace MediaBrowser.Server.Implementations.Sync public string Name { - get { return "App Sync"; } + get { return "Mobile Sync"; } } public IEnumerable<SyncTarget> GetAllSyncTargets() diff --git a/MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs b/MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs index 73400f834..992f1d16c 100644 --- a/MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs +++ b/MediaBrowser.Server.Implementations/Sync/CloudSyncProfile.cs @@ -195,17 +195,39 @@ namespace MediaBrowser.Server.Implementations.Sync } }; - var maxAudioChannels = supportsAc3 || supportsDca ? "5" : "2"; codecProfiles.Add(new CodecProfile { Type = CodecType.VideoAudio, + Codec = "ac3", Conditions = new[] { new ProfileCondition { Condition = ProfileConditionType.LessThanEqual, Property = ProfileConditionValue.AudioChannels, - Value = maxAudioChannels, + Value = "6", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.Equals, + Property = ProfileConditionValue.IsSecondaryAudio, + Value = "false", + IsRequired = false + } + } + }); + codecProfiles.Add(new CodecProfile + { + Type = CodecType.VideoAudio, + Codec = "aac,mp3", + Conditions = new[] + { + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.AudioChannels, + Value = "2", IsRequired = true }, new ProfileCondition diff --git a/MediaBrowser.Server.Implementations/Sync/MediaSync.cs b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs index 5bc8b8088..96e996ff1 100644 --- a/MediaBrowser.Server.Implementations/Sync/MediaSync.cs +++ b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs @@ -1,7 +1,8 @@ -using System.Globalization; +using MediaBrowser.Common.Configuration; using MediaBrowser.Common.IO; using MediaBrowser.Common.Progress; using MediaBrowser.Controller; +using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Sync; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; @@ -10,12 +11,14 @@ using MediaBrowser.Model.MediaInfo; using MediaBrowser.Model.Sync; using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; +using Interfaces.IO; namespace MediaBrowser.Server.Implementations.Sync { @@ -25,13 +28,18 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly IServerApplicationHost _appHost; private readonly ILogger _logger; private readonly IFileSystem _fileSystem; + private readonly IConfigurationManager _config; - public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost, IFileSystem fileSystem) + public const string PathSeparatorString = "/"; + public const char PathSeparatorChar = '/'; + + public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost, IFileSystem fileSystem, IConfigurationManager config) { _logger = logger; _syncManager = syncManager; _appHost = appHost; _fileSystem = fileSystem; + _config = config; } public async Task Sync(IServerSyncProvider provider, @@ -67,7 +75,24 @@ namespace MediaBrowser.Server.Implementations.Sync SyncTarget target, CancellationToken cancellationToken) { - var jobItemIds = await dataProvider.GetSyncJobItemIds(target, serverId).ConfigureAwait(false); + var localItems = await dataProvider.GetLocalItems(target, serverId).ConfigureAwait(false); + var remoteFiles = await provider.GetFiles(new FileQuery(), target, cancellationToken).ConfigureAwait(false); + var remoteIds = remoteFiles.Items.Select(i => i.Id).ToList(); + + var jobItemIds = new List<string>(); + + foreach (var localItem in localItems) + { + // TODO: Remove this after a while + if (string.IsNullOrWhiteSpace(localItem.FileId)) + { + jobItemIds.Add(localItem.SyncJobItemId); + } + else if (remoteIds.Contains(localItem.FileId, StringComparer.OrdinalIgnoreCase)) + { + jobItemIds.Add(localItem.SyncJobItemId); + } + } var result = await _syncManager.SyncData(new SyncDataRequest { @@ -152,12 +177,14 @@ namespace MediaBrowser.Server.Implementations.Sync var transferSuccess = false; Exception transferException = null; + var options = _config.GetSyncOptions(); + try { var fileTransferProgress = new ActionableProgress<double>(); fileTransferProgress.RegisterAction(pct => progress.Report(pct * .92)); - var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath, target, fileTransferProgress, cancellationToken).ConfigureAwait(false); + var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath.Split(PathSeparatorChar), target, options, fileTransferProgress, cancellationToken).ConfigureAwait(false); if (localItem.Item.MediaSources != null) { @@ -171,6 +198,8 @@ namespace MediaBrowser.Server.Implementations.Sync } } + localItem.FileId = sendFileResult.Id; + // Create db record await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false); @@ -179,7 +208,7 @@ namespace MediaBrowser.Server.Implementations.Sync var mediaSource = localItem.Item.MediaSources.FirstOrDefault(); if (mediaSource != null) { - await SendSubtitles(localItem, mediaSource, provider, dataProvider, target, cancellationToken).ConfigureAwait(false); + await SendSubtitles(localItem, mediaSource, provider, dataProvider, target, options, cancellationToken).ConfigureAwait(false); } } @@ -207,7 +236,7 @@ namespace MediaBrowser.Server.Implementations.Sync } } - private async Task SendSubtitles(LocalItem localItem, MediaSourceInfo mediaSource, IServerSyncProvider provider, ISyncDataProvider dataProvider, SyncTarget target, CancellationToken cancellationToken) + private async Task SendSubtitles(LocalItem localItem, MediaSourceInfo mediaSource, IServerSyncProvider provider, ISyncDataProvider dataProvider, SyncTarget target, SyncOptions options, CancellationToken cancellationToken) { var failedSubtitles = new List<MediaStream>(); var requiresSave = false; @@ -219,13 +248,13 @@ namespace MediaBrowser.Server.Implementations.Sync try { var remotePath = GetRemoteSubtitlePath(localItem, mediaStream, provider, target); - var sendFileResult = await SendFile(provider, mediaStream.Path, remotePath, target, new Progress<double>(), cancellationToken).ConfigureAwait(false); + var sendFileResult = await SendFile(provider, mediaStream.Path, remotePath, target, options, new Progress<double>(), cancellationToken).ConfigureAwait(false); // This is the path that will be used when talking to the provider - mediaStream.ExternalId = remotePath; + mediaStream.ExternalId = sendFileResult.Id; // Keep track of all additional files for cleanup later. - localItem.AdditionalFiles.Add(remotePath); + localItem.AdditionalFiles.Add(sendFileResult.Id); // This is the public path clients will use mediaStream.Path = sendFileResult.Path; @@ -250,17 +279,15 @@ namespace MediaBrowser.Server.Implementations.Sync } } - private string GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target) + private string[] GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target) { - var path = item.LocalPath; - var filename = GetSubtitleSaveFileName(item, stream.Language, stream.IsForced) + "." + stream.Codec.ToLower(); - var parentPath = provider.GetParentDirectoryPath(path, target); - - path = Path.Combine(parentPath, filename); + var pathParts = item.LocalPath.Split(PathSeparatorChar); + var list = pathParts.Take(pathParts.Length - 1).ToList(); + list.Add(filename); - return path; + return list.ToArray(); } private string GetSubtitleSaveFileName(LocalItem item, string language, bool isForced) @@ -289,17 +316,21 @@ namespace MediaBrowser.Server.Implementations.Sync SyncTarget target, CancellationToken cancellationToken) { - var localItems = await dataProvider.GetCachedItemsBySyncJobItemId(target, serverId, syncJobItemId); + var localItems = await dataProvider.GetItemsBySyncJobItemId(target, serverId, syncJobItemId); foreach (var localItem in localItems) { var files = localItem.AdditionalFiles.ToList(); - files.Insert(0, localItem.LocalPath); + + // TODO: Remove this. Have to check it for now since this is a new property + if (!string.IsNullOrWhiteSpace(localItem.FileId)) + { + files.Insert(0, localItem.FileId); + } foreach (var file in files) { _logger.Debug("Removing {0} from {1}.", file, target.Name); - await provider.DeleteFile(file, target, cancellationToken).ConfigureAwait(false); } @@ -307,12 +338,19 @@ namespace MediaBrowser.Server.Implementations.Sync } } - private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string remotePath, SyncTarget target, IProgress<double> progress, CancellationToken cancellationToken) + private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string[] pathParts, SyncTarget target, SyncOptions options, IProgress<double> progress, CancellationToken cancellationToken) { - _logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, remotePath); - using (var stream = _fileSystem.GetFileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read, true)) + _logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, string.Join("/", pathParts)); + using (var fileStream = _fileSystem.GetFileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read, true)) { - return await provider.SendFile(stream, remotePath, target, progress, cancellationToken).ConfigureAwait(false); + Stream stream = fileStream; + + if (options.UploadSpeedLimitBytes > 0 && provider is IRemoteSyncProvider) + { + stream = new ThrottledStream(stream, options.UploadSpeedLimitBytes); + } + + return await provider.SendFile(stream, pathParts, target, progress, cancellationToken).ConfigureAwait(false); } } @@ -336,7 +374,7 @@ namespace MediaBrowser.Server.Implementations.Sync var path = GetDirectoryPath(provider, job, syncedItem, libraryItem, serverName); path.Add(GetLocalFileName(provider, libraryItem, originalFileName)); - var localPath = provider.GetFullPath(path, target); + var localPath = string.Join(PathSeparatorString, path.ToArray()); foreach (var mediaSource in libraryItem.MediaSources) { diff --git a/MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs b/MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs index a8bc24c2a..6f09e96f0 100644 --- a/MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs +++ b/MediaBrowser.Server.Implementations/Sync/MultiProviderSync.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Common.IO; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.IO; using MediaBrowser.Common.Progress; using MediaBrowser.Controller; using MediaBrowser.Controller.Sync; @@ -18,13 +19,15 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly IServerApplicationHost _appHost; private readonly ILogger _logger; private readonly IFileSystem _fileSystem; + private readonly IConfigurationManager _config; - public MultiProviderSync(SyncManager syncManager, IServerApplicationHost appHost, ILogger logger, IFileSystem fileSystem) + public MultiProviderSync(SyncManager syncManager, IServerApplicationHost appHost, ILogger logger, IFileSystem fileSystem, IConfigurationManager config) { _syncManager = syncManager; _appHost = appHost; _logger = logger; _fileSystem = fileSystem; + _config = config; } public async Task Sync(IEnumerable<IServerSyncProvider> providers, IProgress<double> progress, CancellationToken cancellationToken) @@ -56,7 +59,7 @@ namespace MediaBrowser.Server.Implementations.Sync var dataProvider = _syncManager.GetDataProvider(target.Item1, target.Item2); - await new MediaSync(_logger, _syncManager, _appHost, _fileSystem) + await new MediaSync(_logger, _syncManager, _appHost, _fileSystem, _config) .Sync(target.Item1, dataProvider, target.Item2, innerProgress, cancellationToken) .ConfigureAwait(false); diff --git a/MediaBrowser.Server.Implementations/Sync/ServerSyncScheduledTask.cs b/MediaBrowser.Server.Implementations/Sync/ServerSyncScheduledTask.cs index 148602bd4..9477a23f1 100644 --- a/MediaBrowser.Server.Implementations/Sync/ServerSyncScheduledTask.cs +++ b/MediaBrowser.Server.Implementations/Sync/ServerSyncScheduledTask.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Common.IO; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.IO; using MediaBrowser.Common.ScheduledTasks; using MediaBrowser.Controller; using MediaBrowser.Controller.Sync; @@ -17,13 +18,15 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly ILogger _logger; private readonly IFileSystem _fileSystem; private readonly IServerApplicationHost _appHost; + private readonly IConfigurationManager _config; - public ServerSyncScheduledTask(ISyncManager syncManager, ILogger logger, IFileSystem fileSystem, IServerApplicationHost appHost) + public ServerSyncScheduledTask(ISyncManager syncManager, ILogger logger, IFileSystem fileSystem, IServerApplicationHost appHost, IConfigurationManager config) { _syncManager = syncManager; _logger = logger; _fileSystem = fileSystem; _appHost = appHost; + _config = config; } public string Name @@ -46,7 +49,7 @@ namespace MediaBrowser.Server.Implementations.Sync public Task Execute(CancellationToken cancellationToken, IProgress<double> progress) { - return new MultiProviderSync((SyncManager)_syncManager, _appHost, _logger, _fileSystem) + return new MultiProviderSync((SyncManager)_syncManager, _appHost, _logger, _fileSystem, _config) .Sync(ServerSyncProviders, progress, cancellationToken); } diff --git a/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs b/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs index 271b2bb39..fd4092974 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs @@ -67,9 +67,10 @@ namespace MediaBrowser.Server.Implementations.Sync var items = (await GetItemsForSync(job.Category, job.ParentId, job.RequestedItemIds, user, job.UnwatchedOnly).ConfigureAwait(false)) .ToList(); - var jobItems = _syncRepo.GetJobItems(new SyncJobItemQuery + var jobItems = _syncManager.GetJobItems(new SyncJobItemQuery { - JobId = job.Id + JobId = job.Id, + AddMetadata = false }).Items.ToList(); @@ -140,9 +141,10 @@ namespace MediaBrowser.Server.Implementations.Sync throw new ArgumentNullException("job"); } - var result = _syncRepo.GetJobItems(new SyncJobItemQuery + var result = _syncManager.GetJobItems(new SyncJobItemQuery { - JobId = job.Id + JobId = job.Id, + AddMetadata = false }); return UpdateJobStatus(job, result.Items.ToList()); @@ -362,9 +364,10 @@ namespace MediaBrowser.Server.Implementations.Sync await EnsureSyncJobItems(null, cancellationToken).ConfigureAwait(false); // If it already has a converting status then is must have been aborted during conversion - var result = _syncRepo.GetJobItems(new SyncJobItemQuery + var result = _syncManager.GetJobItems(new SyncJobItemQuery { - Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting } + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + AddMetadata = false }); await SyncJobItems(result.Items, true, progress, cancellationToken).ConfigureAwait(false); @@ -384,13 +387,14 @@ namespace MediaBrowser.Server.Implementations.Sync await EnsureSyncJobItems(targetId, cancellationToken).ConfigureAwait(false); // If it already has a converting status then is must have been aborted during conversion - var result = _syncRepo.GetJobItems(new SyncJobItemQuery + var result = _syncManager.GetJobItems(new SyncJobItemQuery { - Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, - TargetId = targetId + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + TargetId = targetId, + AddMetadata = false }); - await SyncJobItems(result.Items, true, progress, cancellationToken).ConfigureAwait(false); + await SyncJobItems(result.Items, enableConversion, progress, cancellationToken).ConfigureAwait(false); } public async Task SyncJobItems(SyncJobItem[] items, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) @@ -452,17 +456,18 @@ namespace MediaBrowser.Server.Implementations.Sync jobItem.Progress = 0; + var syncOptions = _config.GetSyncOptions(); var user = _userManager.GetUserById(job.UserId); var video = item as Video; if (video != null) { - await Sync(jobItem, job, video, user, enableConversion, progress, cancellationToken).ConfigureAwait(false); + await Sync(jobItem, job, video, user, enableConversion, syncOptions, progress, cancellationToken).ConfigureAwait(false); } else if (item is Audio) { - await Sync(jobItem, job, (Audio)item, user, enableConversion, progress, cancellationToken).ConfigureAwait(false); + await Sync(jobItem, job, (Audio)item, user, enableConversion, syncOptions, progress, cancellationToken).ConfigureAwait(false); } else if (item is Photo) @@ -476,7 +481,7 @@ namespace MediaBrowser.Server.Implementations.Sync } } - private async Task Sync(SyncJobItem jobItem, SyncJob job, Video item, User user, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) + private async Task Sync(SyncJobItem jobItem, SyncJob job, Video item, User user, bool enableConversion, SyncOptions syncOptions, IProgress<double> progress, CancellationToken cancellationToken) { var jobOptions = _syncManager.GetVideoOptions(jobItem, job); var conversionOptions = new VideoOptions @@ -489,7 +494,7 @@ namespace MediaBrowser.Server.Implementations.Sync conversionOptions.ItemId = item.Id.ToString("N"); conversionOptions.MediaSources = _mediaSourceManager.GetStaticMediaSources(item, false, user).ToList(); - var streamInfo = new StreamBuilder().BuildVideoItem(conversionOptions); + var streamInfo = new StreamBuilder(_logger).BuildVideoItem(conversionOptions); var mediaSource = streamInfo.MediaSource; // No sense creating external subs if we're already burning one into the video @@ -538,7 +543,9 @@ namespace MediaBrowser.Server.Implementations.Sync jobItem.OutputPath = await _mediaEncoder.EncodeVideo(new EncodingJobOptions(streamInfo, conversionOptions.Profile) { - OutputDirectory = jobItem.TemporaryPath + OutputDirectory = jobItem.TemporaryPath, + CpuCoreLimit = syncOptions.TranscodingCpuCoreLimit, + ReadInputAtNativeFramerate = !syncOptions.EnableFullSpeedTranscoding }, innerProgress, cancellationToken); } @@ -673,7 +680,7 @@ namespace MediaBrowser.Server.Implementations.Sync private const int DatabaseProgressUpdateIntervalSeconds = 2; - private async Task Sync(SyncJobItem jobItem, SyncJob job, Audio item, User user, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) + private async Task Sync(SyncJobItem jobItem, SyncJob job, Audio item, User user, bool enableConversion, SyncOptions syncOptions, IProgress<double> progress, CancellationToken cancellationToken) { var jobOptions = _syncManager.GetAudioOptions(jobItem, job); var conversionOptions = new AudioOptions @@ -686,7 +693,7 @@ namespace MediaBrowser.Server.Implementations.Sync conversionOptions.ItemId = item.Id.ToString("N"); conversionOptions.MediaSources = _mediaSourceManager.GetStaticMediaSources(item, false, user).ToList(); - var streamInfo = new StreamBuilder().BuildAudioItem(conversionOptions); + var streamInfo = new StreamBuilder(_logger).BuildAudioItem(conversionOptions); var mediaSource = streamInfo.MediaSource; jobItem.MediaSourceId = streamInfo.MediaSourceId; @@ -721,7 +728,8 @@ namespace MediaBrowser.Server.Implementations.Sync jobItem.OutputPath = await _mediaEncoder.EncodeAudio(new EncodingJobOptions(streamInfo, conversionOptions.Profile) { - OutputDirectory = jobItem.TemporaryPath + OutputDirectory = jobItem.TemporaryPath, + CpuCoreLimit = syncOptions.TranscodingCpuCoreLimit }, innerProgress, cancellationToken); } diff --git a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs index 102218979..0e4a3bcf1 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs @@ -182,19 +182,21 @@ namespace MediaBrowser.Server.Implementations.Sync await processor.EnsureJobItems(job).ConfigureAwait(false); // If it already has a converting status then is must have been aborted during conversion - var jobItemsResult = _repo.GetJobItems(new SyncJobItemQuery + var jobItemsResult = GetJobItems(new SyncJobItemQuery { - Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, - JobId = jobId + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + JobId = jobId, + AddMetadata = false }); await processor.SyncJobItems(jobItemsResult.Items, false, new Progress<double>(), CancellationToken.None) .ConfigureAwait(false); - jobItemsResult = _repo.GetJobItems(new SyncJobItemQuery + jobItemsResult = GetJobItems(new SyncJobItemQuery { - Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, - JobId = jobId + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + JobId = jobId, + AddMetadata = false }); var returnResult = new SyncJobCreationResult @@ -510,12 +512,7 @@ namespace MediaBrowser.Server.Implementations.Sync var video = item as Video; if (video != null) { - if (video.VideoType == VideoType.Iso) - { - return false; - } - - if (video.VideoType == VideoType.BluRay || video.VideoType == VideoType.Dvd || video.VideoType == VideoType.HdDvd) + if (video.VideoType == VideoType.Iso || video.VideoType == VideoType.HdDvd) { return false; } @@ -550,7 +547,7 @@ namespace MediaBrowser.Server.Implementations.Sync } } - if (item is LiveTvChannel || item is IChannelItem || item is ILiveTvRecording) + if (item is LiveTvChannel || item is IChannelItem) { return false; } @@ -564,7 +561,7 @@ namespace MediaBrowser.Server.Implementations.Sync return true; } - return item.LocationType == LocationType.FileSystem || item is Season || item is ILiveTvRecording; + return item.LocationType == LocationType.FileSystem || item is Season; } private string GetDefaultName(BaseItem item) @@ -726,14 +723,19 @@ namespace MediaBrowser.Server.Implementations.Sync TargetId = targetId, Statuses = new[] { - SyncJobItemStatus.ReadyToTransfer + SyncJobItemStatus.ReadyToTransfer, + SyncJobItemStatus.Transferring } }); - return jobItemResult.Items + var readyItems = jobItemResult.Items .Select(GetJobItemInfo) .Where(i => i != null) .ToList(); + + _logger.Debug("Returning {0} ready sync items for targetId {1}", readyItems.Count, targetId); + + return readyItems; } public async Task<SyncDataResponse> SyncData(SyncDataRequest request) @@ -753,6 +755,11 @@ namespace MediaBrowser.Server.Implementations.Sync foreach (var jobItem in jobItemResult.Items) { + var requiresSaving = false; + var removeFromDevice = false; + + var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); + if (request.LocalItemIds.Contains(jobItem.ItemId, StringComparer.OrdinalIgnoreCase)) { var job = _repo.GetJob(jobItem.JobId); @@ -761,36 +768,55 @@ namespace MediaBrowser.Server.Implementations.Sync if (jobItem.IsMarkedForRemoval) { // Tell the device to remove it since it has been marked for removal - response.ItemIdsToRemove.Add(jobItem.ItemId); + _logger.Debug("Adding ItemIdsToRemove {0} because IsMarkedForRemoval is set.", jobItem.ItemId); + removeFromDevice = true; } else if (user == null) { // Tell the device to remove it since the user is gone now - response.ItemIdsToRemove.Add(jobItem.ItemId); + _logger.Debug("Adding ItemIdsToRemove {0} because the user is no longer valid.", jobItem.ItemId); + removeFromDevice = true; + } + else if (!IsLibraryItemAvailable(libraryItem)) + { + // Tell the device to remove it since it's no longer available + _logger.Debug("Adding ItemIdsToRemove {0} because it is no longer available.", jobItem.ItemId); + removeFromDevice = true; } else if (job.UnwatchedOnly) { - var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); - - if (IsLibraryItemAvailable(libraryItem)) - { - if (libraryItem.IsPlayed(user) && libraryItem is Video) - { - // Tell the device to remove it since it has been played - response.ItemIdsToRemove.Add(jobItem.ItemId); - } - } - else + if (libraryItem.IsPlayed(user) && libraryItem is Video) { - // Tell the device to remove it since it's no longer available - response.ItemIdsToRemove.Add(jobItem.ItemId); + // Tell the device to remove it since it has been played + _logger.Debug("Adding ItemIdsToRemove {0} because it has been marked played.", jobItem.ItemId); + removeFromDevice = true; } } } else { // Content is no longer on the device - jobItem.Status = SyncJobItemStatus.RemovedFromDevice; + if (jobItem.IsMarkedForRemoval) + { + jobItem.Status = SyncJobItemStatus.RemovedFromDevice; + } + else + { + _logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.ItemId); + jobItem.Status = SyncJobItemStatus.Queued; + } + requiresSaving = true; + } + + if (removeFromDevice) + { + response.ItemIdsToRemove.Add(jobItem.ItemId); + jobItem.IsMarkedForRemoval = true; + requiresSaving = true; + } + + if (requiresSaving) + { await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); } } @@ -834,6 +860,11 @@ namespace MediaBrowser.Server.Implementations.Sync foreach (var jobItem in jobItemResult.Items) { + var requiresSaving = false; + var removeFromDevice = false; + + var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); + if (request.SyncJobItemIds.Contains(jobItem.Id, StringComparer.OrdinalIgnoreCase)) { var job = _repo.GetJob(jobItem.JobId); @@ -842,36 +873,55 @@ namespace MediaBrowser.Server.Implementations.Sync if (jobItem.IsMarkedForRemoval) { // Tell the device to remove it since it has been marked for removal - response.ItemIdsToRemove.Add(jobItem.Id); + _logger.Debug("Adding ItemIdsToRemove {0} because IsMarkedForRemoval is set.", jobItem.Id); + removeFromDevice = true; } else if (user == null) { // Tell the device to remove it since the user is gone now - response.ItemIdsToRemove.Add(jobItem.Id); + _logger.Debug("Adding ItemIdsToRemove {0} because the user is no longer valid.", jobItem.Id); + removeFromDevice = true; + } + else if (!IsLibraryItemAvailable(libraryItem)) + { + // Tell the device to remove it since it's no longer available + _logger.Debug("Adding ItemIdsToRemove {0} because it is no longer available.", jobItem.Id); + removeFromDevice = true; } else if (job.UnwatchedOnly) { - var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); - - if (IsLibraryItemAvailable(libraryItem)) - { - if (libraryItem.IsPlayed(user) && libraryItem is Video) - { - // Tell the device to remove it since it has been played - response.ItemIdsToRemove.Add(jobItem.Id); - } - } - else + if (libraryItem.IsPlayed(user) && libraryItem is Video) { - // Tell the device to remove it since it's no longer available - response.ItemIdsToRemove.Add(jobItem.Id); + // Tell the device to remove it since it has been played + _logger.Debug("Adding ItemIdsToRemove {0} because it has been marked played.", jobItem.Id); + removeFromDevice = true; } } } else { // Content is no longer on the device - jobItem.Status = SyncJobItemStatus.RemovedFromDevice; + if (jobItem.IsMarkedForRemoval) + { + jobItem.Status = SyncJobItemStatus.RemovedFromDevice; + } + else + { + _logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.Id); + jobItem.Status = SyncJobItemStatus.Queued; + } + requiresSaving = true; + } + + if (removeFromDevice) + { + response.ItemIdsToRemove.Add(jobItem.Id); + jobItem.IsMarkedForRemoval = true; + requiresSaving = true; + } + + if (requiresSaving) + { await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); } } @@ -894,12 +944,6 @@ namespace MediaBrowser.Server.Implementations.Sync response.ItemIdsToRemove = response.ItemIdsToRemove.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); - var itemsOnDevice = request.LocalItemIds - .Except(response.ItemIdsToRemove) - .ToList(); - - SetUserAccess(request, response, itemsOnDevice); - return response; } @@ -962,16 +1006,39 @@ namespace MediaBrowser.Server.Implementations.Sync await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); } + public async Task CancelItems(string targetId, IEnumerable<string> itemIds) + { + foreach (var item in itemIds) + { + var syncJobItemResult = GetJobItems(new SyncJobItemQuery + { + AddMetadata = false, + ItemId = item, + TargetId = targetId, + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Converting, SyncJobItemStatus.Synced, SyncJobItemStatus.Failed } + }); + + foreach (var jobItem in syncJobItemResult.Items) + { + await CancelJobItem(jobItem.Id).ConfigureAwait(false); + } + } + } + public async Task CancelJobItem(string id) { var jobItem = _repo.GetJobItem(id); - if (jobItem.Status != SyncJobItemStatus.Queued && jobItem.Status != SyncJobItemStatus.ReadyToTransfer && jobItem.Status != SyncJobItemStatus.Converting) + if (jobItem.Status != SyncJobItemStatus.Queued && jobItem.Status != SyncJobItemStatus.ReadyToTransfer && jobItem.Status != SyncJobItemStatus.Converting && jobItem.Status != SyncJobItemStatus.Failed && jobItem.Status != SyncJobItemStatus.Synced) { throw new ArgumentException("Operation is not valid for this job item"); } - jobItem.Status = SyncJobItemStatus.Cancelled; + if (jobItem.Status != SyncJobItemStatus.Synced) + { + jobItem.Status = SyncJobItemStatus.Cancelled; + } + jobItem.Progress = 0; jobItem.IsMarkedForRemoval = true; @@ -995,24 +1062,24 @@ namespace MediaBrowser.Server.Implementations.Sync { _logger.ErrorException("Error deleting directory {0}", ex, path); } + + //var jobItemsResult = GetJobItems(new SyncJobItemQuery + //{ + // AddMetadata = false, + // JobId = jobItem.JobId, + // Limit = 0, + // Statuses = new[] { SyncJobItemStatus.Converting, SyncJobItemStatus.Failed, SyncJobItemStatus.Queued, SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Synced, SyncJobItemStatus.Transferring } + //}); + + //if (jobItemsResult.TotalRecordCount == 0) + //{ + // await CancelJob(jobItem.JobId).ConfigureAwait(false); + //} } - public async Task MarkJobItemForRemoval(string id) + public Task MarkJobItemForRemoval(string id) { - var jobItem = _repo.GetJobItem(id); - - if (jobItem.Status != SyncJobItemStatus.Synced) - { - throw new ArgumentException("Operation is not valid for this job item"); - } - - jobItem.IsMarkedForRemoval = true; - - await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); - - var processor = GetSyncJobProcessor(); - - await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + return CancelJobItem(id); } public async Task UnmarkJobItemForRemoval(string id) @@ -1137,13 +1204,18 @@ namespace MediaBrowser.Server.Implementations.Sync public IEnumerable<SyncQualityOption> GetQualityOptions(string targetId) { + return GetQualityOptions(targetId, null); + } + + public IEnumerable<SyncQualityOption> GetQualityOptions(string targetId, User user) + { foreach (var provider in _providers) { foreach (var target in GetSyncTargets(provider)) { if (string.Equals(target.Id, targetId, StringComparison.OrdinalIgnoreCase)) { - return GetQualityOptions(provider, target); + return GetQualityOptions(provider, target, user); } } } @@ -1151,12 +1223,19 @@ namespace MediaBrowser.Server.Implementations.Sync return new List<SyncQualityOption>(); } - private IEnumerable<SyncQualityOption> GetQualityOptions(ISyncProvider provider, SyncTarget target) + private IEnumerable<SyncQualityOption> GetQualityOptions(ISyncProvider provider, SyncTarget target, User user) { var hasQuality = provider as IHasSyncQuality; if (hasQuality != null) { - return hasQuality.GetQualityOptions(target); + var options = hasQuality.GetQualityOptions(target); + + if (user != null && !user.Policy.EnableSyncTranscoding) + { + options = options.Where(i => i.IsOriginalQuality); + } + + return options; } // Default options for providers that don't override @@ -1186,7 +1265,7 @@ namespace MediaBrowser.Server.Implementations.Sync }; } - public IEnumerable<SyncProfileOption> GetProfileOptions(string targetId) + public IEnumerable<SyncProfileOption> GetProfileOptions(string targetId, User user) { foreach (var provider in _providers) { @@ -1194,7 +1273,7 @@ namespace MediaBrowser.Server.Implementations.Sync { if (string.Equals(target.Id, targetId, StringComparison.OrdinalIgnoreCase)) { - return GetProfileOptions(provider, target); + return GetProfileOptions(provider, target, user); } } } @@ -1202,7 +1281,12 @@ namespace MediaBrowser.Server.Implementations.Sync return new List<SyncProfileOption>(); } - private IEnumerable<SyncProfileOption> GetProfileOptions(ISyncProvider provider, SyncTarget target) + public IEnumerable<SyncProfileOption> GetProfileOptions(string targetId) + { + return GetProfileOptions(targetId, null); + } + + private IEnumerable<SyncProfileOption> GetProfileOptions(ISyncProvider provider, SyncTarget target, User user) { var hasQuality = provider as IHasSyncQuality; if (hasQuality != null) @@ -1220,20 +1304,23 @@ namespace MediaBrowser.Server.Implementations.Sync EnableQualityOptions = false }); - list.Add(new SyncProfileOption + if (user == null || user.Policy.EnableSyncTranscoding) { - Name = "Baseline", - Id = "baseline", - Description = "Designed for compatibility with all devices, including web browsers. Targets H264/AAC video and MP3 audio." - }); + list.Add(new SyncProfileOption + { + Name = "Baseline", + Id = "baseline", + Description = "Designed for compatibility with all devices, including web browsers. Targets H264/AAC video and MP3 audio." + }); - list.Add(new SyncProfileOption - { - Name = "General", - Id = "general", - Description = "Designed for compatibility with Chromecast, Roku, Smart TV's, and other similar devices. Targets H264/AAC/AC3 video and MP3 audio.", - IsDefault = true - }); + list.Add(new SyncProfileOption + { + Name = "General", + Id = "general", + Description = "Designed for compatibility with Chromecast, Roku, Smart TV's, and other similar devices. Targets H264/AAC/AC3 video and MP3 audio.", + IsDefault = true + }); + } return list; } diff --git a/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs b/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs index d1ef523e1..f7f320741 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs @@ -56,7 +56,7 @@ namespace MediaBrowser.Server.Implementations.Sync var syncProvider = targetTuple.Item1; var dataProvider = _syncManager.GetDataProvider(targetTuple.Item1, syncTarget); - var localItems = await dataProvider.GetCachedItems(syncTarget, serverId, item.Id.ToString("N")).ConfigureAwait(false); + var localItems = await dataProvider.GetItems(syncTarget, serverId, item.Id.ToString("N")).ConfigureAwait(false); foreach (var localItem in localItems) { @@ -109,8 +109,13 @@ namespace MediaBrowser.Server.Implementations.Sync var dataProvider = _syncManager.GetDataProvider(provider, target); var localItem = await dataProvider.Get(target, openKeys[2]).ConfigureAwait(false); + var fileId = localItem.FileId; + if (string.IsNullOrWhiteSpace(fileId)) + { + } + var requiresDynamicAccess = (IHasDynamicAccess)provider; - var dynamicInfo = await requiresDynamicAccess.GetSyncedFileInfo(localItem.LocalPath, target, cancellationToken).ConfigureAwait(false); + var dynamicInfo = await requiresDynamicAccess.GetSyncedFileInfo(fileId, target, cancellationToken).ConfigureAwait(false); var mediaSource = localItem.Item.MediaSources.First(); mediaSource.LiveStreamId = Guid.NewGuid().ToString(); diff --git a/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs b/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs index dea868848..676adad34 100644 --- a/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs +++ b/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs @@ -1,5 +1,4 @@ using MediaBrowser.Common.Configuration; -using MediaBrowser.Common.Extensions; using MediaBrowser.Common.IO; using MediaBrowser.Controller; using MediaBrowser.Controller.Sync; @@ -12,6 +11,7 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Interfaces.IO; namespace MediaBrowser.Server.Implementations.Sync { @@ -29,8 +29,6 @@ namespace MediaBrowser.Server.Implementations.Sync private readonly IApplicationPaths _appPaths; private readonly IServerApplicationHost _appHost; - private readonly SemaphoreSlim _cacheFileLock = new SemaphoreSlim(1, 1); - public TargetDataProvider(IServerSyncProvider provider, SyncTarget target, IServerApplicationHost appHost, ILogger logger, IJsonSerializer json, IFileSystem fileSystem, IApplicationPaths appPaths) { _logger = logger; @@ -42,12 +40,7 @@ namespace MediaBrowser.Server.Implementations.Sync _appHost = appHost; } - private string GetCachePath() - { - return Path.Combine(_appPaths.DataPath, "sync", _target.Id.GetMD5().ToString("N") + ".json"); - } - - private string GetRemotePath() + private string[] GetRemotePath() { var parts = new List<string> { @@ -57,7 +50,7 @@ namespace MediaBrowser.Server.Implementations.Sync parts = parts.Select(i => GetValidFilename(_provider, i)).ToList(); - return _provider.GetFullPath(parts, _target); + return parts.ToArray(); } private string GetValidFilename(IServerSyncProvider provider, string filename) @@ -66,58 +59,29 @@ namespace MediaBrowser.Server.Implementations.Sync return _fileSystem.GetValidFilename(filename); } - private async Task CacheData(Stream stream) - { - var cachePath = GetCachePath(); - - await _cacheFileLock.WaitAsync().ConfigureAwait(false); - - try - { - Directory.CreateDirectory(Path.GetDirectoryName(cachePath)); - using (var fileStream = _fileSystem.GetFileStream(cachePath, FileMode.Create, FileAccess.Write, FileShare.Read, true)) - { - await stream.CopyToAsync(fileStream).ConfigureAwait(false); - } - } - catch (Exception ex) - { - _logger.ErrorException("Error saving sync data to {0}", ex, cachePath); - } - finally - { - _cacheFileLock.Release(); - } - } - private async Task EnsureData(CancellationToken cancellationToken) { if (_items == null) { - try + _logger.Debug("Getting {0} from {1}", string.Join(MediaSync.PathSeparatorString, GetRemotePath().ToArray()), _provider.Name); + + var fileResult = await _provider.GetFiles(new FileQuery + { + FullPath = GetRemotePath().ToArray() + + }, _target, cancellationToken).ConfigureAwait(false); + + if (fileResult.Items.Length > 0) { - using (var stream = await _provider.GetFile(GetRemotePath(), _target, new Progress<double>(), cancellationToken)) + using (var stream = await _provider.GetFile(fileResult.Items[0].Id, _target, new Progress<double>(), cancellationToken)) { _items = _json.DeserializeFromStream<List<LocalItem>>(stream); } } - catch (FileNotFoundException) - { - _items = new List<LocalItem>(); - } - catch (DirectoryNotFoundException) + else { _items = new List<LocalItem>(); } - - using (var memoryStream = new MemoryStream()) - { - _json.SerializeToStream(_items, memoryStream); - - // Now cache it - memoryStream.Position = 0; - await CacheData(memoryStream).ConfigureAwait(false); - } } } @@ -130,10 +94,6 @@ namespace MediaBrowser.Server.Implementations.Sync // Save to sync provider stream.Position = 0; await _provider.SendFile(stream, GetRemotePath(), _target, new Progress<double>(), cancellationToken).ConfigureAwait(false); - - // Now cache it - stream.Position = 0; - await CacheData(stream).ConfigureAwait(false); } } @@ -171,14 +131,9 @@ namespace MediaBrowser.Server.Implementations.Sync } } - public Task<List<string>> GetServerItemIds(SyncTarget target, string serverId) + public Task<List<LocalItem>> GetLocalItems(SyncTarget target, string serverId) { - return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).Select(i => i.ItemId).ToList()); - } - - public Task<List<string>> GetSyncJobItemIds(SyncTarget target, string serverId) - { - return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).Select(i => i.SyncJobItemId).Where(i => !string.IsNullOrWhiteSpace(i)).ToList()); + return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).ToList()); } public Task AddOrUpdate(SyncTarget target, LocalItem item) @@ -204,62 +159,14 @@ namespace MediaBrowser.Server.Implementations.Sync return GetData(items => items.FirstOrDefault(i => string.Equals(i.Id, id, StringComparison.OrdinalIgnoreCase))); } - private async Task<List<LocalItem>> GetCachedData() - { - if (_items == null) - { - await _cacheFileLock.WaitAsync().ConfigureAwait(false); - - try - { - if (_items == null) - { - try - { - _items = _json.DeserializeFromFile<List<LocalItem>>(GetCachePath()); - } - catch (FileNotFoundException) - { - _items = new List<LocalItem>(); - } - catch (DirectoryNotFoundException) - { - _items = new List<LocalItem>(); - } - } - } - finally - { - _cacheFileLock.Release(); - } - } - - return _items.ToList(); - } - - public async Task<List<string>> GetCachedServerItemIds(SyncTarget target, string serverId) - { - var items = await GetCachedData().ConfigureAwait(false); - - return items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)) - .Select(i => i.ItemId) - .ToList(); - } - - public async Task<List<LocalItem>> GetCachedItems(SyncTarget target, string serverId, string itemId) + public Task<List<LocalItem>> GetItems(SyncTarget target, string serverId, string itemId) { - var items = await GetCachedData().ConfigureAwait(false); - - return items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.ItemId, itemId, StringComparison.OrdinalIgnoreCase)) - .ToList(); + return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.ItemId, itemId, StringComparison.OrdinalIgnoreCase)).ToList()); } - public async Task<List<LocalItem>> GetCachedItemsBySyncJobItemId(SyncTarget target, string serverId, string syncJobItemId) + public Task<List<LocalItem>> GetItemsBySyncJobItemId(SyncTarget target, string serverId, string syncJobItemId) { - var items = await GetCachedData().ConfigureAwait(false); - - return items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.SyncJobItemId, syncJobItemId, StringComparison.OrdinalIgnoreCase)) - .ToList(); + return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.SyncJobItemId, syncJobItemId, StringComparison.OrdinalIgnoreCase)).ToList()); } } } |