diff options
| author | Luke <luke.pulverenti@gmail.com> | 2016-11-03 19:59:50 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2016-11-03 19:59:50 -0400 |
| commit | c53745548ac2130f4cfbbe0d7a2804c36c8ae4eb (patch) | |
| tree | 6ee298ebb5470c4f3bcbef8d814a0354901469c4 /Emby.Server.Implementations/Sync | |
| parent | 338b04a0c58729ec70aed89924ea6bd12422872b (diff) | |
| parent | 405a5f69c5967b4d919b5fe91396f12cb83e8aa8 (diff) | |
Merge pull request #2267 from MediaBrowser/dev
Dev
Diffstat (limited to 'Emby.Server.Implementations/Sync')
16 files changed, 4060 insertions, 0 deletions
diff --git a/Emby.Server.Implementations/Sync/AppSyncProvider.cs b/Emby.Server.Implementations/Sync/AppSyncProvider.cs new file mode 100644 index 000000000..d405a0ff9 --- /dev/null +++ b/Emby.Server.Implementations/Sync/AppSyncProvider.cs @@ -0,0 +1,118 @@ +using MediaBrowser.Controller.Devices; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Devices; +using MediaBrowser.Model.Dlna; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Emby.Server.Implementations.Sync +{ + public class AppSyncProvider : ISyncProvider, IHasUniqueTargetIds, IHasSyncQuality, IHasDuplicateCheck + { + private readonly IDeviceManager _deviceManager; + + public AppSyncProvider(IDeviceManager deviceManager) + { + _deviceManager = deviceManager; + } + + public IEnumerable<SyncTarget> GetSyncTargets(string userId) + { + return _deviceManager.GetDevices(new DeviceQuery + { + SupportsSync = true, + UserId = userId + + }).Items.Select(i => new SyncTarget + { + Id = i.Id, + Name = i.Name + }); + } + + public DeviceProfile GetDeviceProfile(SyncTarget target, string profile, string quality) + { + var caps = _deviceManager.GetCapabilities(target.Id); + + var deviceProfile = caps == null || caps.DeviceProfile == null ? new DeviceProfile() : caps.DeviceProfile; + deviceProfile.MaxStaticBitrate = SyncHelper.AdjustBitrate(deviceProfile.MaxStaticBitrate, quality); + + return deviceProfile; + } + + public string Name + { + get { return "Mobile Sync"; } + } + + public IEnumerable<SyncTarget> GetAllSyncTargets() + { + return _deviceManager.GetDevices(new DeviceQuery + { + SupportsSync = true + + }).Items.Select(i => new SyncTarget + { + Id = i.Id, + Name = i.Name + }); + } + + public IEnumerable<SyncQualityOption> GetQualityOptions(SyncTarget target) + { + return new List<SyncQualityOption> + { + new SyncQualityOption + { + Name = "Original", + Id = "original", + Description = "Syncs original files as-is, regardless of whether the device is capable of playing them or not." + }, + new SyncQualityOption + { + Name = "High", + Id = "high", + IsDefault = true + }, + new SyncQualityOption + { + Name = "Medium", + Id = "medium" + }, + new SyncQualityOption + { + Name = "Low", + Id = "low" + }, + new SyncQualityOption + { + Name = "Custom", + Id = "custom" + } + }; + } + + public IEnumerable<SyncProfileOption> GetProfileOptions(SyncTarget target) + { + return new List<SyncProfileOption>(); + } + + public SyncJobOptions GetSyncJobOptions(SyncTarget target, string profile, string quality) + { + var isConverting = !string.Equals(quality, "original", StringComparison.OrdinalIgnoreCase); + + return new SyncJobOptions + { + DeviceProfile = GetDeviceProfile(target, profile, quality), + IsConverting = isConverting + }; + } + + public bool AllowDuplicateJobItem(SyncJobItem original, SyncJobItem duplicate) + { + return false; + } + } +} diff --git a/Emby.Server.Implementations/Sync/CloudSyncProfile.cs b/Emby.Server.Implementations/Sync/CloudSyncProfile.cs new file mode 100644 index 000000000..1a78c8ae6 --- /dev/null +++ b/Emby.Server.Implementations/Sync/CloudSyncProfile.cs @@ -0,0 +1,302 @@ +using MediaBrowser.Model.Dlna; +using System.Collections.Generic; + +namespace Emby.Server.Implementations.Sync +{ + public class CloudSyncProfile : DeviceProfile + { + public CloudSyncProfile(bool supportsAc3, bool supportsDca) + { + Name = "Cloud Sync"; + + MaxStreamingBitrate = 20000000; + MaxStaticBitrate = 20000000; + + var mkvAudio = "aac,mp3"; + var mp4Audio = "aac"; + + if (supportsAc3) + { + mkvAudio += ",ac3"; + mp4Audio += ",ac3"; + } + + if (supportsDca) + { + mkvAudio += ",dca,dts"; + } + + var videoProfile = "high|main|baseline|constrained baseline"; + var videoLevel = "40"; + + DirectPlayProfiles = new[] + { + //new DirectPlayProfile + //{ + // Container = "mkv", + // VideoCodec = "h264,mpeg4", + // AudioCodec = mkvAudio, + // Type = DlnaProfileType.Video + //}, + new DirectPlayProfile + { + Container = "mp4,mov,m4v", + VideoCodec = "h264,mpeg4", + AudioCodec = mp4Audio, + Type = DlnaProfileType.Video + }, + new DirectPlayProfile + { + Container = "mp3", + Type = DlnaProfileType.Audio + } + }; + + ContainerProfiles = new[] + { + new ContainerProfile + { + Type = DlnaProfileType.Video, + Conditions = new [] + { + new ProfileCondition + { + Condition = ProfileConditionType.NotEquals, + Property = ProfileConditionValue.NumAudioStreams, + Value = "0", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.EqualsAny, + Property = ProfileConditionValue.NumVideoStreams, + Value = "1", + IsRequired = false + } + } + } + }; + + var codecProfiles = new List<CodecProfile> + { + new CodecProfile + { + Type = CodecType.Video, + Codec = "h264", + Conditions = new [] + { + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.VideoBitDepth, + Value = "8", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.Width, + Value = "1920", + IsRequired = true + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.Height, + Value = "1080", + IsRequired = true + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.RefFrames, + Value = "4", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.VideoFramerate, + Value = "30", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.Equals, + Property = ProfileConditionValue.IsAnamorphic, + Value = "false", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.VideoLevel, + Value = videoLevel, + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.EqualsAny, + Property = ProfileConditionValue.VideoProfile, + Value = videoProfile, + IsRequired = false + } + } + }, + new CodecProfile + { + Type = CodecType.Video, + Codec = "mpeg4", + Conditions = new [] + { + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.VideoBitDepth, + Value = "8", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.Width, + Value = "1920", + IsRequired = true + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.Height, + Value = "1080", + IsRequired = true + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.RefFrames, + Value = "4", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.VideoFramerate, + Value = "30", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.Equals, + Property = ProfileConditionValue.IsAnamorphic, + Value = "false", + IsRequired = false + } + } + } + }; + + codecProfiles.Add(new CodecProfile + { + Type = CodecType.VideoAudio, + Codec = "ac3", + Conditions = new[] + { + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.AudioChannels, + Value = "6", + IsRequired = false + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.AudioBitrate, + Value = "320000", + IsRequired = true + }, + new ProfileCondition + { + Condition = ProfileConditionType.Equals, + Property = ProfileConditionValue.IsSecondaryAudio, + Value = "false", + IsRequired = false + } + } + }); + codecProfiles.Add(new CodecProfile + { + Type = CodecType.VideoAudio, + Codec = "aac,mp3", + Conditions = new[] + { + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.AudioChannels, + Value = "2", + IsRequired = true + }, + new ProfileCondition + { + Condition = ProfileConditionType.LessThanEqual, + Property = ProfileConditionValue.AudioBitrate, + Value = "320000", + IsRequired = true + }, + new ProfileCondition + { + Condition = ProfileConditionType.Equals, + Property = ProfileConditionValue.IsSecondaryAudio, + Value = "false", + IsRequired = false + } + } + }); + + CodecProfiles = codecProfiles.ToArray(); + + SubtitleProfiles = new[] + { + new SubtitleProfile + { + Format = "srt", + Method = SubtitleDeliveryMethod.External + }, + new SubtitleProfile + { + Format = "vtt", + Method = SubtitleDeliveryMethod.External + } + }; + + TranscodingProfiles = new[] + { + new TranscodingProfile + { + Container = "mp3", + AudioCodec = "mp3", + Type = DlnaProfileType.Audio, + Context = EncodingContext.Static + }, + + new TranscodingProfile + { + Container = "mp4", + Type = DlnaProfileType.Video, + AudioCodec = "aac", + VideoCodec = "h264", + Context = EncodingContext.Static + }, + + new TranscodingProfile + { + Container = "jpeg", + Type = DlnaProfileType.Photo, + Context = EncodingContext.Static + } + }; + + } + } +} diff --git a/Emby.Server.Implementations/Sync/IHasSyncQuality.cs b/Emby.Server.Implementations/Sync/IHasSyncQuality.cs new file mode 100644 index 000000000..bec8b37a7 --- /dev/null +++ b/Emby.Server.Implementations/Sync/IHasSyncQuality.cs @@ -0,0 +1,31 @@ +using MediaBrowser.Model.Sync; +using System.Collections.Generic; + +namespace Emby.Server.Implementations.Sync +{ + public interface IHasSyncQuality + { + /// <summary> + /// Gets the device profile. + /// </summary> + /// <param name="target">The target.</param> + /// <param name="profile">The profile.</param> + /// <param name="quality">The quality.</param> + /// <returns>DeviceProfile.</returns> + SyncJobOptions GetSyncJobOptions(SyncTarget target, string profile, string quality); + + /// <summary> + /// Gets the quality options. + /// </summary> + /// <param name="target">The target.</param> + /// <returns>IEnumerable<SyncQualityOption>.</returns> + IEnumerable<SyncQualityOption> GetQualityOptions(SyncTarget target); + + /// <summary> + /// Gets the profile options. + /// </summary> + /// <param name="target">The target.</param> + /// <returns>IEnumerable<SyncQualityOption>.</returns> + IEnumerable<SyncProfileOption> GetProfileOptions(SyncTarget target); + } +} diff --git a/Emby.Server.Implementations/Sync/MediaSync.cs b/Emby.Server.Implementations/Sync/MediaSync.cs new file mode 100644 index 000000000..b420a3df4 --- /dev/null +++ b/Emby.Server.Implementations/Sync/MediaSync.cs @@ -0,0 +1,500 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Dto; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.MediaInfo; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations.IO; +using MediaBrowser.Model.Cryptography; +using MediaBrowser.Model.IO; + +namespace Emby.Server.Implementations.Sync +{ + public class MediaSync + { + private readonly ISyncManager _syncManager; + private readonly IServerApplicationHost _appHost; + private readonly ILogger _logger; + private readonly IFileSystem _fileSystem; + private readonly IConfigurationManager _config; + private readonly ICryptographyProvider _cryptographyProvider; + + public const string PathSeparatorString = "/"; + public const char PathSeparatorChar = '/'; + + public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost, IFileSystem fileSystem, IConfigurationManager config, ICryptographyProvider cryptographyProvider) + { + _logger = logger; + _syncManager = syncManager; + _appHost = appHost; + _fileSystem = fileSystem; + _config = config; + _cryptographyProvider = cryptographyProvider; + } + + public async Task Sync(IServerSyncProvider provider, + ISyncDataProvider dataProvider, + SyncTarget target, + IProgress<double> progress, + CancellationToken cancellationToken) + { + var serverId = _appHost.SystemId; + var serverName = _appHost.FriendlyName; + + await SyncData(provider, dataProvider, serverId, target, cancellationToken).ConfigureAwait(false); + progress.Report(3); + + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(pct => + { + var totalProgress = pct * .97; + totalProgress += 1; + progress.Report(totalProgress); + }); + await GetNewMedia(provider, dataProvider, target, serverId, serverName, innerProgress, cancellationToken); + + // Do the data sync twice so the server knows what was removed from the device + await SyncData(provider, dataProvider, serverId, target, cancellationToken).ConfigureAwait(false); + + progress.Report(100); + } + + private async Task SyncData(IServerSyncProvider provider, + ISyncDataProvider dataProvider, + string serverId, + SyncTarget target, + CancellationToken cancellationToken) + { + var localItems = await dataProvider.GetLocalItems(target, serverId).ConfigureAwait(false); + var remoteFiles = await provider.GetFiles(target, cancellationToken).ConfigureAwait(false); + var remoteIds = remoteFiles.Items.Select(i => i.FullName).ToList(); + + var jobItemIds = new List<string>(); + + foreach (var localItem in localItems) + { + if (remoteIds.Contains(localItem.FileId, StringComparer.OrdinalIgnoreCase)) + { + jobItemIds.Add(localItem.SyncJobItemId); + } + } + + var result = await _syncManager.SyncData(new SyncDataRequest + { + TargetId = target.Id, + SyncJobItemIds = jobItemIds + + }).ConfigureAwait(false); + + cancellationToken.ThrowIfCancellationRequested(); + + foreach (var itemIdToRemove in result.ItemIdsToRemove) + { + try + { + await RemoveItem(provider, dataProvider, serverId, itemIdToRemove, target, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting item from device. Id: {0}", ex, itemIdToRemove); + } + } + } + + private async Task GetNewMedia(IServerSyncProvider provider, + ISyncDataProvider dataProvider, + SyncTarget target, + string serverId, + string serverName, + IProgress<double> progress, + CancellationToken cancellationToken) + { + var jobItems = await _syncManager.GetReadySyncItems(target.Id).ConfigureAwait(false); + + var numComplete = 0; + double startingPercent = 0; + double percentPerItem = 1; + if (jobItems.Count > 0) + { + percentPerItem /= jobItems.Count; + } + + foreach (var jobItem in jobItems) + { + cancellationToken.ThrowIfCancellationRequested(); + + var currentPercent = startingPercent; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(pct => + { + var totalProgress = pct * percentPerItem; + totalProgress += currentPercent; + progress.Report(totalProgress); + }); + + try + { + await GetItem(provider, dataProvider, target, serverId, serverName, jobItem, innerProgress, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.ErrorException("Error syncing item", ex); + } + + numComplete++; + startingPercent = numComplete; + startingPercent /= jobItems.Count; + startingPercent *= 100; + progress.Report(startingPercent); + } + } + + private async Task GetItem(IServerSyncProvider provider, + ISyncDataProvider dataProvider, + SyncTarget target, + string serverId, + string serverName, + SyncedItem jobItem, + IProgress<double> progress, + CancellationToken cancellationToken) + { + var libraryItem = jobItem.Item; + var internalSyncJobItem = _syncManager.GetJobItem(jobItem.SyncJobItemId); + var internalSyncJob = _syncManager.GetJob(jobItem.SyncJobId); + + var localItem = CreateLocalItem(provider, jobItem, internalSyncJob, target, libraryItem, serverId, serverName, jobItem.OriginalFileName); + + await _syncManager.ReportSyncJobItemTransferBeginning(internalSyncJobItem.Id); + + var transferSuccess = false; + Exception transferException = null; + + var options = _config.GetSyncOptions(); + + try + { + var fileTransferProgress = new ActionableProgress<double>(); + fileTransferProgress.RegisterAction(pct => progress.Report(pct * .92)); + + var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath.Split(PathSeparatorChar), target, options, fileTransferProgress, cancellationToken).ConfigureAwait(false); + + if (localItem.Item.MediaSources != null) + { + var mediaSource = localItem.Item.MediaSources.FirstOrDefault(); + if (mediaSource != null) + { + mediaSource.Path = sendFileResult.Path; + mediaSource.Protocol = sendFileResult.Protocol; + mediaSource.RequiredHttpHeaders = sendFileResult.RequiredHttpHeaders; + mediaSource.SupportsTranscoding = false; + } + } + + localItem.FileId = sendFileResult.Id; + + // Create db record + await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false); + + if (localItem.Item.MediaSources != null) + { + var mediaSource = localItem.Item.MediaSources.FirstOrDefault(); + if (mediaSource != null) + { + await SendSubtitles(localItem, mediaSource, provider, dataProvider, target, options, cancellationToken).ConfigureAwait(false); + } + } + + progress.Report(92); + + transferSuccess = true; + + progress.Report(99); + } + catch (Exception ex) + { + _logger.ErrorException("Error transferring sync job file", ex); + transferException = ex; + } + + if (transferSuccess) + { + await _syncManager.ReportSyncJobItemTransferred(jobItem.SyncJobItemId).ConfigureAwait(false); + } + else + { + await _syncManager.ReportSyncJobItemTransferFailed(jobItem.SyncJobItemId).ConfigureAwait(false); + + throw transferException; + } + } + + private async Task SendSubtitles(LocalItem localItem, MediaSourceInfo mediaSource, IServerSyncProvider provider, ISyncDataProvider dataProvider, SyncTarget target, SyncOptions options, CancellationToken cancellationToken) + { + var failedSubtitles = new List<MediaStream>(); + var requiresSave = false; + + foreach (var mediaStream in mediaSource.MediaStreams + .Where(i => i.Type == MediaStreamType.Subtitle && i.IsExternal) + .ToList()) + { + try + { + var remotePath = GetRemoteSubtitlePath(localItem, mediaStream, provider, target); + var sendFileResult = await SendFile(provider, mediaStream.Path, remotePath, target, options, new Progress<double>(), cancellationToken).ConfigureAwait(false); + + // This is the path that will be used when talking to the provider + mediaStream.ExternalId = sendFileResult.Id; + + // Keep track of all additional files for cleanup later. + localItem.AdditionalFiles.Add(sendFileResult.Id); + + // This is the public path clients will use + mediaStream.Path = sendFileResult.Path; + requiresSave = true; + } + catch (Exception ex) + { + _logger.ErrorException("Error sending subtitle stream", ex); + failedSubtitles.Add(mediaStream); + } + } + + if (failedSubtitles.Count > 0) + { + mediaSource.MediaStreams = mediaSource.MediaStreams.Except(failedSubtitles).ToList(); + requiresSave = true; + } + + if (requiresSave) + { + await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false); + } + } + + private string[] GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target) + { + var filename = GetSubtitleSaveFileName(item, stream.Language, stream.IsForced) + "." + stream.Codec.ToLower(); + + var pathParts = item.LocalPath.Split(PathSeparatorChar); + var list = pathParts.Take(pathParts.Length - 1).ToList(); + list.Add(filename); + + return list.ToArray(); + } + + private string GetSubtitleSaveFileName(LocalItem item, string language, bool isForced) + { + var path = item.LocalPath; + + var name = Path.GetFileNameWithoutExtension(path); + + if (!string.IsNullOrWhiteSpace(language)) + { + name += "." + language.ToLower(); + } + + if (isForced) + { + name += ".foreign"; + } + + return name; + } + + private async Task RemoveItem(IServerSyncProvider provider, + ISyncDataProvider dataProvider, + string serverId, + string syncJobItemId, + SyncTarget target, + CancellationToken cancellationToken) + { + var localItems = await dataProvider.GetItemsBySyncJobItemId(target, serverId, syncJobItemId); + + foreach (var localItem in localItems) + { + var files = localItem.AdditionalFiles.ToList(); + + foreach (var file in files) + { + _logger.Debug("Removing {0} from {1}.", file, target.Name); + await provider.DeleteFile(file, target, cancellationToken).ConfigureAwait(false); + } + + _logger.Debug("Removing {0} from {1}.", localItem.FileId, target.Name); + await provider.DeleteFile(localItem.FileId, target, cancellationToken).ConfigureAwait(false); + + await dataProvider.Delete(target, localItem.Id).ConfigureAwait(false); + } + } + + private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string[] pathParts, SyncTarget target, SyncOptions options, IProgress<double> progress, CancellationToken cancellationToken) + { + _logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, string.Join("/", pathParts)); + var supportsDirectCopy = provider as ISupportsDirectCopy; + if (supportsDirectCopy != null) + { + return await supportsDirectCopy.SendFile(inputPath, pathParts, target, progress, cancellationToken).ConfigureAwait(false); + } + + using (var fileStream = _fileSystem.GetFileStream(inputPath, FileOpenMode.Open, FileAccessMode.Read, FileShareMode.Read, true)) + { + Stream stream = fileStream; + + if (options.UploadSpeedLimitBytes > 0 && provider is IRemoteSyncProvider) + { + stream = new ThrottledStream(stream, options.UploadSpeedLimitBytes); + } + + return await provider.SendFile(stream, pathParts, target, progress, cancellationToken).ConfigureAwait(false); + } + } + + private string GetLocalId(string jobItemId, string itemId) + { + var bytes = Encoding.UTF8.GetBytes(jobItemId + itemId); + bytes = CreateMd5(bytes); + return BitConverter.ToString(bytes, 0, bytes.Length).Replace("-", string.Empty); + } + + private byte[] CreateMd5(byte[] value) + { + return _cryptographyProvider.GetMD5Bytes(value); + } + + public LocalItem CreateLocalItem(IServerSyncProvider provider, SyncedItem syncedItem, SyncJob job, SyncTarget target, BaseItemDto libraryItem, string serverId, string serverName, string originalFileName) + { + var path = GetDirectoryPath(provider, job, syncedItem, libraryItem, serverName); + path.Add(GetLocalFileName(provider, libraryItem, originalFileName)); + + var localPath = string.Join(PathSeparatorString, path.ToArray()); + + foreach (var mediaSource in libraryItem.MediaSources) + { + mediaSource.Path = localPath; + mediaSource.Protocol = MediaProtocol.File; + } + + return new LocalItem + { + Item = libraryItem, + ItemId = libraryItem.Id, + ServerId = serverId, + LocalPath = localPath, + Id = GetLocalId(syncedItem.SyncJobItemId, libraryItem.Id), + SyncJobItemId = syncedItem.SyncJobItemId + }; + } + + private List<string> GetDirectoryPath(IServerSyncProvider provider, SyncJob job, SyncedItem syncedItem, BaseItemDto item, string serverName) + { + var parts = new List<string> + { + serverName + }; + + var profileOption = _syncManager.GetProfileOptions(job.TargetId) + .FirstOrDefault(i => string.Equals(i.Id, job.Profile, StringComparison.OrdinalIgnoreCase)); + + string name; + + if (profileOption != null && !string.IsNullOrWhiteSpace(profileOption.Name)) + { + name = profileOption.Name; + + if (job.Bitrate.HasValue) + { + name += "-" + job.Bitrate.Value.ToString(CultureInfo.InvariantCulture); + } + else + { + var qualityOption = _syncManager.GetQualityOptions(job.TargetId) + .FirstOrDefault(i => string.Equals(i.Id, job.Quality, StringComparison.OrdinalIgnoreCase)); + + if (qualityOption != null && !string.IsNullOrWhiteSpace(qualityOption.Name)) + { + name += "-" + qualityOption.Name; + } + } + } + else + { + name = syncedItem.SyncJobName + "-" + syncedItem.SyncJobDateCreated + .ToLocalTime() + .ToString("g") + .Replace(" ", "-"); + } + + name = GetValidFilename(provider, name); + parts.Add(name); + + if (item.IsType("episode")) + { + parts.Add("TV"); + if (!string.IsNullOrWhiteSpace(item.SeriesName)) + { + parts.Add(item.SeriesName); + } + } + else if (item.IsVideo) + { + parts.Add("Videos"); + parts.Add(item.Name); + } + else if (item.IsAudio) + { + parts.Add("Music"); + + if (!string.IsNullOrWhiteSpace(item.AlbumArtist)) + { + parts.Add(item.AlbumArtist); + } + + if (!string.IsNullOrWhiteSpace(item.Album)) + { + parts.Add(item.Album); + } + } + else if (string.Equals(item.MediaType, MediaType.Photo, StringComparison.OrdinalIgnoreCase)) + { + parts.Add("Photos"); + + if (!string.IsNullOrWhiteSpace(item.Album)) + { + parts.Add(item.Album); + } + } + + return parts.Select(i => GetValidFilename(provider, i)).ToList(); + } + + private string GetLocalFileName(IServerSyncProvider provider, BaseItemDto item, string originalFileName) + { + var filename = originalFileName; + + if (string.IsNullOrWhiteSpace(filename)) + { + filename = item.Name; + } + + return GetValidFilename(provider, filename); + } + + private string GetValidFilename(IServerSyncProvider provider, string filename) + { + // We can always add this method to the sync provider if it's really needed + return _fileSystem.GetValidFilename(filename); + } + } +} diff --git a/Emby.Server.Implementations/Sync/MultiProviderSync.cs b/Emby.Server.Implementations/Sync/MultiProviderSync.cs new file mode 100644 index 000000000..db6cfcbd6 --- /dev/null +++ b/Emby.Server.Implementations/Sync/MultiProviderSync.cs @@ -0,0 +1,79 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.IO; +using MediaBrowser.Controller.IO; +using MediaBrowser.Model.Cryptography; +using MediaBrowser.Model.IO; + +namespace Emby.Server.Implementations.Sync +{ + public class MultiProviderSync + { + private readonly SyncManager _syncManager; + private readonly IServerApplicationHost _appHost; + private readonly ILogger _logger; + private readonly IFileSystem _fileSystem; + private readonly IConfigurationManager _config; + private readonly ICryptographyProvider _cryptographyProvider; + + public MultiProviderSync(SyncManager syncManager, IServerApplicationHost appHost, ILogger logger, IFileSystem fileSystem, IConfigurationManager config, ICryptographyProvider cryptographyProvider) + { + _syncManager = syncManager; + _appHost = appHost; + _logger = logger; + _fileSystem = fileSystem; + _config = config; + _cryptographyProvider = cryptographyProvider; + } + + public async Task Sync(IEnumerable<IServerSyncProvider> providers, IProgress<double> progress, CancellationToken cancellationToken) + { + var targets = providers + .SelectMany(i => i.GetAllSyncTargets().Select(t => new Tuple<IServerSyncProvider, SyncTarget>(i, t))) + .ToList(); + + var numComplete = 0; + double startingPercent = 0; + double percentPerItem = 1; + if (targets.Count > 0) + { + percentPerItem /= targets.Count; + } + + foreach (var target in targets) + { + cancellationToken.ThrowIfCancellationRequested(); + + var currentPercent = startingPercent; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(pct => + { + var totalProgress = pct * percentPerItem; + totalProgress += currentPercent; + progress.Report(totalProgress); + }); + + var dataProvider = _syncManager.GetDataProvider(target.Item1, target.Item2); + + await new MediaSync(_logger, _syncManager, _appHost, _fileSystem, _config, _cryptographyProvider) + .Sync(target.Item1, dataProvider, target.Item2, innerProgress, cancellationToken) + .ConfigureAwait(false); + + numComplete++; + startingPercent = numComplete; + startingPercent /= targets.Count; + startingPercent *= 100; + progress.Report(startingPercent); + } + } + } +} diff --git a/Emby.Server.Implementations/Sync/ServerSyncScheduledTask.cs b/Emby.Server.Implementations/Sync/ServerSyncScheduledTask.cs new file mode 100644 index 000000000..17171633e --- /dev/null +++ b/Emby.Server.Implementations/Sync/ServerSyncScheduledTask.cs @@ -0,0 +1,95 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Model.Cryptography; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.Tasks; + +namespace Emby.Server.Implementations.Sync +{ + class ServerSyncScheduledTask : IScheduledTask, IConfigurableScheduledTask + { + private readonly ISyncManager _syncManager; + private readonly ILogger _logger; + private readonly IFileSystem _fileSystem; + private readonly IServerApplicationHost _appHost; + private readonly IConfigurationManager _config; + private readonly ICryptographyProvider _cryptographyProvider; + + public ServerSyncScheduledTask(ISyncManager syncManager, ILogger logger, IFileSystem fileSystem, IServerApplicationHost appHost, IConfigurationManager config, ICryptographyProvider cryptographyProvider) + { + _syncManager = syncManager; + _logger = logger; + _fileSystem = fileSystem; + _appHost = appHost; + _config = config; + _cryptographyProvider = cryptographyProvider; + } + + public string Name + { + get { return "Cloud & Folder Sync"; } + } + + public string Description + { + get { return "Sync media to the cloud"; } + } + + public string Category + { + get + { + return "Sync"; + } + } + + public Task Execute(CancellationToken cancellationToken, IProgress<double> progress) + { + return new MultiProviderSync((SyncManager)_syncManager, _appHost, _logger, _fileSystem, _config, _cryptographyProvider) + .Sync(ServerSyncProviders, progress, cancellationToken); + } + + public IEnumerable<IServerSyncProvider> ServerSyncProviders + { + get { return ((SyncManager)_syncManager).ServerSyncProviders; } + } + + /// <summary> + /// Creates the triggers that define when the task will run + /// </summary> + public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() + { + return new[] { + + // Every so often + new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(3).Ticks} + }; + } + public bool IsHidden + { + get { return !IsEnabled; } + } + + public bool IsEnabled + { + get { return ServerSyncProviders.Any(); } + } + + public bool IsLogged + { + get { return true; } + } + + public string Key + { + get { return "ServerSync"; } + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncConfig.cs b/Emby.Server.Implementations/Sync/SyncConfig.cs new file mode 100644 index 000000000..8a97326bd --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncConfig.cs @@ -0,0 +1,29 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Model.Sync; +using System.Collections.Generic; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncConfigurationFactory : IConfigurationFactory + { + public IEnumerable<ConfigurationStore> GetConfigurations() + { + return new List<ConfigurationStore> + { + new ConfigurationStore + { + ConfigurationType = typeof(SyncOptions), + Key = "sync" + } + }; + } + } + + public static class SyncExtensions + { + public static SyncOptions GetSyncOptions(this IConfigurationManager config) + { + return config.GetConfiguration<SyncOptions>("sync"); + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncConvertScheduledTask.cs b/Emby.Server.Implementations/Sync/SyncConvertScheduledTask.cs new file mode 100644 index 000000000..8dafac7e1 --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncConvertScheduledTask.cs @@ -0,0 +1,89 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.MediaEncoding; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Controller.TV; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.IO; +using MediaBrowser.Controller.IO; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.Tasks; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncConvertScheduledTask : IScheduledTask + { + private readonly ILibraryManager _libraryManager; + private readonly ISyncRepository _syncRepo; + private readonly ISyncManager _syncManager; + private readonly ILogger _logger; + private readonly IUserManager _userManager; + private readonly ITVSeriesManager _tvSeriesManager; + private readonly IMediaEncoder _mediaEncoder; + private readonly ISubtitleEncoder _subtitleEncoder; + private readonly IConfigurationManager _config; + private readonly IFileSystem _fileSystem; + private readonly IMediaSourceManager _mediaSourceManager; + + public SyncConvertScheduledTask(ILibraryManager libraryManager, ISyncRepository syncRepo, ISyncManager syncManager, ILogger logger, IUserManager userManager, ITVSeriesManager tvSeriesManager, IMediaEncoder mediaEncoder, ISubtitleEncoder subtitleEncoder, IConfigurationManager config, IFileSystem fileSystem, IMediaSourceManager mediaSourceManager) + { + _libraryManager = libraryManager; + _syncRepo = syncRepo; + _syncManager = syncManager; + _logger = logger; + _userManager = userManager; + _tvSeriesManager = tvSeriesManager; + _mediaEncoder = mediaEncoder; + _subtitleEncoder = subtitleEncoder; + _config = config; + _fileSystem = fileSystem; + _mediaSourceManager = mediaSourceManager; + } + + public string Name + { + get { return "Convert media"; } + } + + public string Description + { + get { return "Runs scheduled sync jobs"; } + } + + public string Category + { + get + { + return "Sync"; + } + } + + public Task Execute(CancellationToken cancellationToken, IProgress<double> progress) + { + return new SyncJobProcessor(_libraryManager, _syncRepo, (SyncManager)_syncManager, _logger, _userManager, _tvSeriesManager, _mediaEncoder, _subtitleEncoder, _config, _fileSystem, _mediaSourceManager) + .Sync(progress, cancellationToken); + } + + /// <summary> + /// Creates the triggers that define when the task will run + /// </summary> + /// <returns>IEnumerable{BaseTaskTrigger}.</returns> + public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() + { + return new[] { + + // Every so often + new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(3).Ticks} + }; + } + + public string Key + { + get { return "SyncPrepare"; } + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncHelper.cs b/Emby.Server.Implementations/Sync/SyncHelper.cs new file mode 100644 index 000000000..da475f003 --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncHelper.cs @@ -0,0 +1,24 @@ +using System; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncHelper + { + public static int? AdjustBitrate(int? profileBitrate, string quality) + { + if (profileBitrate.HasValue) + { + if (string.Equals(quality, "medium", StringComparison.OrdinalIgnoreCase)) + { + profileBitrate = Math.Min(profileBitrate.Value, 4000000); + } + else if (string.Equals(quality, "low", StringComparison.OrdinalIgnoreCase)) + { + profileBitrate = Math.Min(profileBitrate.Value, 1500000); + } + } + + return profileBitrate; + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncJobOptions.cs b/Emby.Server.Implementations/Sync/SyncJobOptions.cs new file mode 100644 index 000000000..8e4d8e2ed --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncJobOptions.cs @@ -0,0 +1,18 @@ +using MediaBrowser.Model.Dlna; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncJobOptions + { + /// <summary> + /// Gets or sets the conversion options. + /// </summary> + /// <value>The conversion options.</value> + public DeviceProfile DeviceProfile { get; set; } + /// <summary> + /// Gets or sets a value indicating whether this instance is converting. + /// </summary> + /// <value><c>true</c> if this instance is converting; otherwise, <c>false</c>.</value> + public bool IsConverting { get; set; } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncJobProcessor.cs b/Emby.Server.Implementations/Sync/SyncJobProcessor.cs new file mode 100644 index 000000000..415757609 --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncJobProcessor.cs @@ -0,0 +1,988 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.IO; +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.MediaEncoding; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Controller.TV; +using MediaBrowser.Model.Dlna; +using MediaBrowser.Model.Dto; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.MediaInfo; +using MediaBrowser.Model.Querying; +using MediaBrowser.Model.Session; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.IO; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.Extensions; +using MediaBrowser.Model.IO; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncJobProcessor + { + private readonly ILibraryManager _libraryManager; + private readonly ISyncRepository _syncRepo; + private readonly SyncManager _syncManager; + private readonly ILogger _logger; + private readonly IUserManager _userManager; + private readonly ITVSeriesManager _tvSeriesManager; + private readonly IMediaEncoder _mediaEncoder; + private readonly ISubtitleEncoder _subtitleEncoder; + private readonly IConfigurationManager _config; + private readonly IFileSystem _fileSystem; + private readonly IMediaSourceManager _mediaSourceManager; + + public SyncJobProcessor(ILibraryManager libraryManager, ISyncRepository syncRepo, SyncManager syncManager, ILogger logger, IUserManager userManager, ITVSeriesManager tvSeriesManager, IMediaEncoder mediaEncoder, ISubtitleEncoder subtitleEncoder, IConfigurationManager config, IFileSystem fileSystem, IMediaSourceManager mediaSourceManager) + { + _libraryManager = libraryManager; + _syncRepo = syncRepo; + _syncManager = syncManager; + _logger = logger; + _userManager = userManager; + _tvSeriesManager = tvSeriesManager; + _mediaEncoder = mediaEncoder; + _subtitleEncoder = subtitleEncoder; + _config = config; + _fileSystem = fileSystem; + _mediaSourceManager = mediaSourceManager; + } + + public async Task EnsureJobItems(SyncJob job) + { + var user = _userManager.GetUserById(job.UserId); + + if (user == null) + { + throw new InvalidOperationException("Cannot proceed with sync because user no longer exists."); + } + + var items = (await GetItemsForSync(job.Category, job.ParentId, job.RequestedItemIds, user, job.UnwatchedOnly).ConfigureAwait(false)) + .ToList(); + + var jobItems = _syncManager.GetJobItems(new SyncJobItemQuery + { + JobId = job.Id, + AddMetadata = false + + }).Items.ToList(); + + foreach (var item in items) + { + // Respect ItemLimit, if set + if (job.ItemLimit.HasValue) + { + if (jobItems.Count(j => j.Status != SyncJobItemStatus.RemovedFromDevice && j.Status != SyncJobItemStatus.Failed) >= job.ItemLimit.Value) + { + break; + } + } + + var itemId = item.Id.ToString("N"); + + var jobItem = jobItems.FirstOrDefault(i => string.Equals(i.ItemId, itemId, StringComparison.OrdinalIgnoreCase)); + + if (jobItem != null) + { + continue; + } + + var index = jobItems.Count == 0 ? + 0 : + jobItems.Select(i => i.JobItemIndex).Max() + 1; + + jobItem = new SyncJobItem + { + Id = Guid.NewGuid().ToString("N"), + ItemId = itemId, + ItemName = GetSyncJobItemName(item), + JobId = job.Id, + TargetId = job.TargetId, + DateCreated = DateTime.UtcNow, + JobItemIndex = index + }; + + await _syncRepo.Create(jobItem).ConfigureAwait(false); + _syncManager.OnSyncJobItemCreated(jobItem); + + jobItems.Add(jobItem); + } + + jobItems = jobItems + .OrderBy(i => i.DateCreated) + .ToList(); + + await UpdateJobStatus(job, jobItems).ConfigureAwait(false); + } + + private string GetSyncJobItemName(BaseItem item) + { + var name = item.Name; + var episode = item as Episode; + + if (episode != null) + { + if (episode.IndexNumber.HasValue) + { + name = "E" + episode.IndexNumber.Value.ToString(CultureInfo.InvariantCulture) + " - " + name; + } + + if (episode.ParentIndexNumber.HasValue) + { + name = "S" + episode.ParentIndexNumber.Value.ToString(CultureInfo.InvariantCulture) + ", " + name; + } + } + + return name; + } + + public Task UpdateJobStatus(string id) + { + var job = _syncRepo.GetJob(id); + + if (job == null) + { + return Task.FromResult(true); + } + + var result = _syncManager.GetJobItems(new SyncJobItemQuery + { + JobId = job.Id, + AddMetadata = false + }); + + return UpdateJobStatus(job, result.Items.ToList()); + } + + private async Task UpdateJobStatus(SyncJob job, List<SyncJobItem> jobItems) + { + job.ItemCount = jobItems.Count; + + double pct = 0; + + foreach (var item in jobItems) + { + if (item.Status == SyncJobItemStatus.Failed || item.Status == SyncJobItemStatus.Synced || item.Status == SyncJobItemStatus.RemovedFromDevice || item.Status == SyncJobItemStatus.Cancelled) + { + pct += 100; + } + else + { + pct += item.Progress ?? 0; + } + } + + if (job.ItemCount > 0) + { + pct /= job.ItemCount; + job.Progress = pct; + } + else + { + job.Progress = null; + } + + if (jobItems.Any(i => i.Status == SyncJobItemStatus.Transferring)) + { + job.Status = SyncJobStatus.Transferring; + } + else if (jobItems.Any(i => i.Status == SyncJobItemStatus.Converting)) + { + job.Status = SyncJobStatus.Converting; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.Failed)) + { + job.Status = SyncJobStatus.Failed; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.Cancelled)) + { + job.Status = SyncJobStatus.Cancelled; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.ReadyToTransfer)) + { + job.Status = SyncJobStatus.ReadyToTransfer; + } + else if (jobItems.All(i => i.Status == SyncJobItemStatus.Cancelled || i.Status == SyncJobItemStatus.Failed || i.Status == SyncJobItemStatus.Synced || i.Status == SyncJobItemStatus.RemovedFromDevice)) + { + if (jobItems.Any(i => i.Status == SyncJobItemStatus.Failed)) + { + job.Status = SyncJobStatus.CompletedWithError; + } + else + { + job.Status = SyncJobStatus.Completed; + } + } + else + { + job.Status = SyncJobStatus.Queued; + } + + await _syncRepo.Update(job).ConfigureAwait(false); + + _syncManager.OnSyncJobUpdated(job); + } + + public async Task<IEnumerable<BaseItem>> GetItemsForSync(SyncCategory? category, string parentId, IEnumerable<string> itemIds, User user, bool unwatchedOnly) + { + var list = new List<BaseItem>(); + + if (category.HasValue) + { + list = (await GetItemsForSync(category.Value, parentId, user).ConfigureAwait(false)).ToList(); + } + else + { + foreach (var itemId in itemIds) + { + var subList = await GetItemsForSync(itemId, user).ConfigureAwait(false); + list.AddRange(subList); + } + } + + IEnumerable<BaseItem> items = list; + items = items.Where(_syncManager.SupportsSync); + + if (unwatchedOnly) + { + // Avoid implicitly captured closure + var currentUser = user; + + items = items.Where(i => + { + var video = i as Video; + + if (video != null) + { + return !video.IsPlayed(currentUser); + } + + return true; + }); + } + + return items.DistinctBy(i => i.Id); + } + + private async Task<IEnumerable<BaseItem>> GetItemsForSync(SyncCategory category, string parentId, User user) + { + var parent = string.IsNullOrWhiteSpace(parentId) + ? user.RootFolder + : (Folder)_libraryManager.GetItemById(parentId); + + InternalItemsQuery query; + + switch (category) + { + case SyncCategory.Latest: + query = new InternalItemsQuery + { + IsFolder = false, + SortBy = new[] { ItemSortBy.DateCreated, ItemSortBy.SortName }, + SortOrder = SortOrder.Descending, + Recursive = true + }; + break; + case SyncCategory.Resume: + query = new InternalItemsQuery + { + IsFolder = false, + SortBy = new[] { ItemSortBy.DatePlayed, ItemSortBy.SortName }, + SortOrder = SortOrder.Descending, + Recursive = true, + IsResumable = true, + MediaTypes = new[] { MediaType.Video } + }; + break; + + case SyncCategory.NextUp: + return _tvSeriesManager.GetNextUp(new NextUpQuery + { + ParentId = parentId, + UserId = user.Id.ToString("N") + }).Items; + + default: + throw new ArgumentException("Unrecognized category: " + category); + } + + if (parent == null) + { + return new List<BaseItem>(); + } + + query.User = user; + + var result = await parent.GetItems(query).ConfigureAwait(false); + return result.Items; + } + + private async Task<List<BaseItem>> GetItemsForSync(string id, User user) + { + var item = _libraryManager.GetItemById(id); + + if (item == null) + { + return new List<BaseItem>(); + } + + var itemByName = item as IItemByName; + if (itemByName != null) + { + return itemByName.GetTaggedItems(new InternalItemsQuery(user) + { + IsFolder = false, + Recursive = true + }).ToList(); + } + + if (item.IsFolder) + { + var folder = (Folder)item; + var itemsResult = await folder.GetItems(new InternalItemsQuery(user) + { + Recursive = true, + IsFolder = false + + }).ConfigureAwait(false); + + var items = itemsResult.Items; + + if (!folder.IsPreSorted) + { + items = _libraryManager.Sort(items, user, new[] { ItemSortBy.SortName }, SortOrder.Ascending) + .ToArray(); + } + + return items.ToList(); + } + + return new List<BaseItem> { item }; + } + + private async Task EnsureSyncJobItems(string targetId, CancellationToken cancellationToken) + { + var jobResult = _syncRepo.GetJobs(new SyncJobQuery + { + SyncNewContent = true, + TargetId = targetId + }); + + foreach (var job in jobResult.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (job.SyncNewContent) + { + await EnsureJobItems(job).ConfigureAwait(false); + } + } + } + + public async Task Sync(IProgress<double> progress, CancellationToken cancellationToken) + { + await EnsureSyncJobItems(null, cancellationToken).ConfigureAwait(false); + + // Look job items that are supposedly transfering, but need to be requeued because the synced files have been deleted somehow + await HandleDeletedSyncFiles(cancellationToken).ConfigureAwait(false); + + // If it already has a converting status then is must have been aborted during conversion + var result = _syncManager.GetJobItems(new SyncJobItemQuery + { + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + AddMetadata = false + }); + + await SyncJobItems(result.Items, true, progress, cancellationToken).ConfigureAwait(false); + + CleanDeadSyncFiles(); + } + + private async Task HandleDeletedSyncFiles(CancellationToken cancellationToken) + { + // Look job items that are supposedly transfering, but need to be requeued because the synced files have been deleted somehow + var result = _syncManager.GetJobItems(new SyncJobItemQuery + { + Statuses = new[] { SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Transferring }, + AddMetadata = false + }); + + foreach (var item in result.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(item.OutputPath) || !_fileSystem.FileExists(item.OutputPath)) + { + item.Status = SyncJobItemStatus.Queued; + await _syncManager.UpdateSyncJobItemInternal(item).ConfigureAwait(false); + await UpdateJobStatus(item.JobId).ConfigureAwait(false); + } + } + } + + private void CleanDeadSyncFiles() + { + // TODO + // Clean files in sync temp folder that are not linked to any sync jobs + } + + public async Task SyncJobItems(string targetId, bool enableConversion, IProgress<double> progress, + CancellationToken cancellationToken) + { + await EnsureSyncJobItems(targetId, cancellationToken).ConfigureAwait(false); + + // If it already has a converting status then is must have been aborted during conversion + var result = _syncManager.GetJobItems(new SyncJobItemQuery + { + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + TargetId = targetId, + AddMetadata = false + }); + + await SyncJobItems(result.Items, enableConversion, progress, cancellationToken).ConfigureAwait(false); + } + + public async Task SyncJobItems(SyncJobItem[] items, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) + { + if (items.Length > 0) + { + if (!SyncRegistrationInfo.Instance.IsRegistered) + { + _logger.Debug("Cancelling sync job processing. Please obtain a supporter membership."); + return; + } + } + + var numComplete = 0; + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + double percentPerItem = 1; + percentPerItem /= items.Length; + var startingPercent = numComplete * percentPerItem * 100; + + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(p => progress.Report(startingPercent + percentPerItem * p)); + + // Pull it fresh from the db just to make sure it wasn't deleted or cancelled while another item was converting + var jobItem = enableConversion ? _syncRepo.GetJobItem(item.Id) : item; + + if (jobItem != null) + { + if (jobItem.Status != SyncJobItemStatus.Cancelled) + { + await ProcessJobItem(jobItem, enableConversion, innerProgress, cancellationToken).ConfigureAwait(false); + } + + await UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + numComplete++; + double percent = numComplete; + percent /= items.Length; + progress.Report(100 * percent); + } + } + + private async Task ProcessJobItem(SyncJobItem jobItem, bool enableConversion, IProgress<double> progress, CancellationToken cancellationToken) + { + if (jobItem == null) + { + throw new ArgumentNullException("jobItem"); + } + + var item = _libraryManager.GetItemById(jobItem.ItemId); + if (item == null) + { + jobItem.Status = SyncJobItemStatus.Failed; + _logger.Error("Unable to locate library item for JobItem {0}, ItemId {1}", jobItem.Id, jobItem.ItemId); + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + return; + } + + jobItem.Progress = 0; + + var syncOptions = _config.GetSyncOptions(); + var job = _syncManager.GetJob(jobItem.JobId); + var user = _userManager.GetUserById(job.UserId); + if (user == null) + { + jobItem.Status = SyncJobItemStatus.Failed; + _logger.Error("User not found. Cannot complete the sync job."); + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + return; + } + + // See if there's already another active job item for the same target + var existingJobItems = _syncManager.GetJobItems(new SyncJobItemQuery + { + AddMetadata = false, + ItemId = jobItem.ItemId, + TargetId = jobItem.TargetId, + Statuses = new[] { SyncJobItemStatus.Converting, SyncJobItemStatus.Queued, SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Synced, SyncJobItemStatus.Transferring } + }); + + var duplicateJobItems = existingJobItems.Items + .Where(i => !string.Equals(i.Id, jobItem.Id, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (duplicateJobItems.Count > 0) + { + var syncProvider = _syncManager.GetSyncProvider(jobItem) as IHasDuplicateCheck; + + if (!duplicateJobItems.Any(i => AllowDuplicateJobItem(syncProvider, i, jobItem))) + { + _logger.Debug("Cancelling sync job item because there is already another active job for the same target."); + jobItem.Status = SyncJobItemStatus.Cancelled; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + return; + } + } + + var video = item as Video; + if (video != null) + { + await Sync(jobItem, video, user, enableConversion, syncOptions, progress, cancellationToken).ConfigureAwait(false); + } + + else if (item is Audio) + { + await Sync(jobItem, (Audio)item, user, enableConversion, syncOptions, progress, cancellationToken).ConfigureAwait(false); + } + + else if (item is Photo) + { + await Sync(jobItem, (Photo)item, cancellationToken).ConfigureAwait(false); + } + + else + { + await SyncGeneric(jobItem, item, cancellationToken).ConfigureAwait(false); + } + } + + private bool AllowDuplicateJobItem(IHasDuplicateCheck provider, SyncJobItem original, SyncJobItem duplicate) + { + if (provider != null) + { + return provider.AllowDuplicateJobItem(original, duplicate); + } + + return true; + } + + private async Task Sync(SyncJobItem jobItem, Video item, User user, bool enableConversion, SyncOptions syncOptions, IProgress<double> progress, CancellationToken cancellationToken) + { + var job = _syncManager.GetJob(jobItem.JobId); + var jobOptions = _syncManager.GetVideoOptions(jobItem, job); + var conversionOptions = new VideoOptions + { + Profile = jobOptions.DeviceProfile + }; + + conversionOptions.DeviceId = jobItem.TargetId; + conversionOptions.Context = EncodingContext.Static; + conversionOptions.ItemId = item.Id.ToString("N"); + conversionOptions.MediaSources = _mediaSourceManager.GetStaticMediaSources(item, false, user).ToList(); + + var streamInfo = new StreamBuilder(_mediaEncoder, _logger).BuildVideoItem(conversionOptions); + var mediaSource = streamInfo.MediaSource; + + // No sense creating external subs if we're already burning one into the video + var externalSubs = streamInfo.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode ? + new List<SubtitleStreamInfo>() : + streamInfo.GetExternalSubtitles(false, true, null, null); + + // Mark as requiring conversion if transcoding the video, or if any subtitles need to be extracted + var requiresVideoTranscoding = streamInfo.PlayMethod == PlayMethod.Transcode && jobOptions.IsConverting; + var requiresConversion = requiresVideoTranscoding || externalSubs.Any(i => RequiresExtraction(i, mediaSource)); + + if (requiresConversion && !enableConversion) + { + return; + } + + jobItem.MediaSourceId = streamInfo.MediaSourceId; + jobItem.TemporaryPath = GetTemporaryPath(jobItem); + + if (requiresConversion) + { + jobItem.Status = SyncJobItemStatus.Converting; + } + + if (requiresVideoTranscoding) + { + // Save the job item now since conversion could take a while + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + + try + { + var lastJobUpdate = DateTime.MinValue; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(async pct => + { + progress.Report(pct); + + if ((DateTime.UtcNow - lastJobUpdate).TotalSeconds >= DatabaseProgressUpdateIntervalSeconds) + { + jobItem.Progress = pct / 2; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + }); + + jobItem.OutputPath = await _mediaEncoder.EncodeVideo(new EncodingJobOptions(streamInfo, conversionOptions.Profile) + { + OutputDirectory = jobItem.TemporaryPath, + CpuCoreLimit = syncOptions.TranscodingCpuCoreLimit, + ReadInputAtNativeFramerate = !syncOptions.EnableFullSpeedTranscoding + + }, innerProgress, cancellationToken); + + jobItem.ItemDateModifiedTicks = item.DateModified.Ticks; + _syncManager.OnConversionComplete(jobItem); + } + catch (OperationCanceledException) + { + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + } + catch (Exception ex) + { + jobItem.Status = SyncJobItemStatus.Failed; + _logger.ErrorException("Error during sync transcoding", ex); + } + + if (jobItem.Status == SyncJobItemStatus.Failed || jobItem.Status == SyncJobItemStatus.Queued) + { + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + return; + } + + jobItem.MediaSource = await GetEncodedMediaSource(jobItem.OutputPath, user, true).ConfigureAwait(false); + } + else + { + if (mediaSource.Protocol == MediaProtocol.File) + { + jobItem.OutputPath = mediaSource.Path; + } + else if (mediaSource.Protocol == MediaProtocol.Http) + { + jobItem.OutputPath = await DownloadFile(jobItem, mediaSource, cancellationToken).ConfigureAwait(false); + } + else + { + throw new InvalidOperationException(string.Format("Cannot direct stream {0} protocol", mediaSource.Protocol)); + } + + jobItem.ItemDateModifiedTicks = item.DateModified.Ticks; + jobItem.MediaSource = mediaSource; + } + + jobItem.MediaSource.SupportsTranscoding = false; + + if (externalSubs.Count > 0) + { + // Save the job item now since conversion could take a while + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + await ConvertSubtitles(jobItem, externalSubs, streamInfo, cancellationToken).ConfigureAwait(false); + } + + jobItem.Progress = 50; + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + } + + private bool RequiresExtraction(SubtitleStreamInfo stream, MediaSourceInfo mediaSource) + { + var originalStream = mediaSource.MediaStreams.FirstOrDefault(i => i.Type == MediaStreamType.Subtitle && i.Index == stream.Index); + + return originalStream != null && !originalStream.IsExternal; + } + + private async Task ConvertSubtitles(SyncJobItem jobItem, + IEnumerable<SubtitleStreamInfo> subtitles, + StreamInfo streamInfo, + CancellationToken cancellationToken) + { + var files = new List<ItemFileInfo>(); + + var mediaStreams = jobItem.MediaSource.MediaStreams + .Where(i => i.Type != MediaStreamType.Subtitle || !i.IsExternal) + .ToList(); + + var startingIndex = mediaStreams.Count == 0 ? + 0 : + mediaStreams.Select(i => i.Index).Max() + 1; + + foreach (var subtitle in subtitles) + { + var fileInfo = await ConvertSubtitles(jobItem.TemporaryPath, streamInfo, subtitle, cancellationToken).ConfigureAwait(false); + + // Reset this to a value that will be based on the output media + fileInfo.Index = startingIndex; + files.Add(fileInfo); + + mediaStreams.Add(new MediaStream + { + Index = startingIndex, + Codec = subtitle.Format, + IsForced = subtitle.IsForced, + IsExternal = true, + Language = subtitle.Language, + Path = fileInfo.Path, + SupportsExternalStream = true, + Type = MediaStreamType.Subtitle + }); + + startingIndex++; + } + + jobItem.AdditionalFiles.AddRange(files); + + jobItem.MediaSource.MediaStreams = mediaStreams; + } + + private async Task<ItemFileInfo> ConvertSubtitles(string temporaryPath, StreamInfo streamInfo, SubtitleStreamInfo subtitleStreamInfo, CancellationToken cancellationToken) + { + var subtitleStreamIndex = subtitleStreamInfo.Index; + + var filename = Guid.NewGuid() + "." + subtitleStreamInfo.Format.ToLower(); + + var path = Path.Combine(temporaryPath, filename); + + _fileSystem.CreateDirectory(Path.GetDirectoryName(path)); + + using (var stream = await _subtitleEncoder.GetSubtitles(streamInfo.ItemId, streamInfo.MediaSourceId, subtitleStreamIndex, subtitleStreamInfo.Format, 0, null, false, cancellationToken).ConfigureAwait(false)) + { + using (var fs = _fileSystem.GetFileStream(path, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, true)) + { + await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); + } + } + + return new ItemFileInfo + { + Name = Path.GetFileName(path), + Path = path, + Type = ItemFileType.Subtitles, + Index = subtitleStreamIndex + }; + } + + private const int DatabaseProgressUpdateIntervalSeconds = 2; + + private async Task Sync(SyncJobItem jobItem, Audio item, User user, bool enableConversion, SyncOptions syncOptions, IProgress<double> progress, CancellationToken cancellationToken) + { + var job = _syncManager.GetJob(jobItem.JobId); + var jobOptions = _syncManager.GetAudioOptions(jobItem, job); + var conversionOptions = new AudioOptions + { + Profile = jobOptions.DeviceProfile + }; + + conversionOptions.DeviceId = jobItem.TargetId; + conversionOptions.Context = EncodingContext.Static; + conversionOptions.ItemId = item.Id.ToString("N"); + conversionOptions.MediaSources = _mediaSourceManager.GetStaticMediaSources(item, false, user).ToList(); + + var streamInfo = new StreamBuilder(_mediaEncoder, _logger).BuildAudioItem(conversionOptions); + var mediaSource = streamInfo.MediaSource; + + jobItem.MediaSourceId = streamInfo.MediaSourceId; + jobItem.TemporaryPath = GetTemporaryPath(jobItem); + + if (streamInfo.PlayMethod == PlayMethod.Transcode && jobOptions.IsConverting) + { + if (!enableConversion) + { + return; + } + + jobItem.Status = SyncJobItemStatus.Converting; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + + try + { + var lastJobUpdate = DateTime.MinValue; + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(async pct => + { + progress.Report(pct); + + if ((DateTime.UtcNow - lastJobUpdate).TotalSeconds >= DatabaseProgressUpdateIntervalSeconds) + { + jobItem.Progress = pct / 2; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + await UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + }); + + jobItem.OutputPath = await _mediaEncoder.EncodeAudio(new EncodingJobOptions(streamInfo, conversionOptions.Profile) + { + OutputDirectory = jobItem.TemporaryPath, + CpuCoreLimit = syncOptions.TranscodingCpuCoreLimit + + }, innerProgress, cancellationToken); + + jobItem.ItemDateModifiedTicks = item.DateModified.Ticks; + _syncManager.OnConversionComplete(jobItem); + } + catch (OperationCanceledException) + { + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + } + catch (Exception ex) + { + jobItem.Status = SyncJobItemStatus.Failed; + _logger.ErrorException("Error during sync transcoding", ex); + } + + if (jobItem.Status == SyncJobItemStatus.Failed || jobItem.Status == SyncJobItemStatus.Queued) + { + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + return; + } + + jobItem.MediaSource = await GetEncodedMediaSource(jobItem.OutputPath, user, false).ConfigureAwait(false); + } + else + { + if (mediaSource.Protocol == MediaProtocol.File) + { + jobItem.OutputPath = mediaSource.Path; + } + else if (mediaSource.Protocol == MediaProtocol.Http) + { + jobItem.OutputPath = await DownloadFile(jobItem, mediaSource, cancellationToken).ConfigureAwait(false); + } + else + { + throw new InvalidOperationException(string.Format("Cannot direct stream {0} protocol", mediaSource.Protocol)); + } + + jobItem.ItemDateModifiedTicks = item.DateModified.Ticks; + jobItem.MediaSource = mediaSource; + } + + jobItem.MediaSource.SupportsTranscoding = false; + + jobItem.Progress = 50; + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + } + + private async Task Sync(SyncJobItem jobItem, Photo item, CancellationToken cancellationToken) + { + jobItem.OutputPath = item.Path; + + jobItem.Progress = 50; + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + jobItem.ItemDateModifiedTicks = item.DateModified.Ticks; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + } + + private async Task SyncGeneric(SyncJobItem jobItem, BaseItem item, CancellationToken cancellationToken) + { + jobItem.OutputPath = item.Path; + + jobItem.Progress = 50; + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + jobItem.ItemDateModifiedTicks = item.DateModified.Ticks; + await _syncManager.UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + } + + private async Task<string> DownloadFile(SyncJobItem jobItem, MediaSourceInfo mediaSource, CancellationToken cancellationToken) + { + // TODO: Download + return mediaSource.Path; + } + + public string GetTemporaryPath(SyncJob job) + { + return GetTemporaryPath(job.Id); + } + + public string GetTemporaryPath(string jobId) + { + var basePath = _config.GetSyncOptions().TemporaryPath; + + if (string.IsNullOrWhiteSpace(basePath)) + { + basePath = Path.Combine(_config.CommonApplicationPaths.ProgramDataPath, "sync"); + } + + return Path.Combine(basePath, jobId); + } + + public string GetTemporaryPath(SyncJobItem jobItem) + { + return Path.Combine(GetTemporaryPath(jobItem.JobId), jobItem.Id); + } + + private async Task<MediaSourceInfo> GetEncodedMediaSource(string path, User user, bool isVideo) + { + var item = _libraryManager.ResolvePath(_fileSystem.GetFileSystemInfo(path)); + + await item.RefreshMetadata(CancellationToken.None).ConfigureAwait(false); + + var hasMediaSources = item as IHasMediaSources; + + var mediaSources = _mediaSourceManager.GetStaticMediaSources(hasMediaSources, false).ToList(); + + var preferredAudio = string.IsNullOrEmpty(user.Configuration.AudioLanguagePreference) + ? new string[] { } + : new[] { user.Configuration.AudioLanguagePreference }; + + var preferredSubs = string.IsNullOrEmpty(user.Configuration.SubtitleLanguagePreference) + ? new List<string>() : new List<string> { user.Configuration.SubtitleLanguagePreference }; + + foreach (var source in mediaSources) + { + if (isVideo) + { + source.DefaultAudioStreamIndex = + MediaStreamSelector.GetDefaultAudioStreamIndex(source.MediaStreams, preferredAudio, user.Configuration.PlayDefaultAudioTrack); + + var defaultAudioIndex = source.DefaultAudioStreamIndex; + var audioLangage = defaultAudioIndex == null + ? null + : source.MediaStreams.Where(i => i.Type == MediaStreamType.Audio && i.Index == defaultAudioIndex).Select(i => i.Language).FirstOrDefault(); + + source.DefaultAudioStreamIndex = + MediaStreamSelector.GetDefaultSubtitleStreamIndex(source.MediaStreams, preferredSubs, user.Configuration.SubtitleMode, audioLangage); + } + else + { + var audio = source.MediaStreams.FirstOrDefault(i => i.Type == MediaStreamType.Audio); + + if (audio != null) + { + source.DefaultAudioStreamIndex = audio.Index; + } + + } + } + + return mediaSources.FirstOrDefault(); + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncManager.cs b/Emby.Server.Implementations/Sync/SyncManager.cs new file mode 100644 index 000000000..d06ed49fd --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncManager.cs @@ -0,0 +1,1362 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Events; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Drawing; +using MediaBrowser.Controller.Dto; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.MediaEncoding; +using MediaBrowser.Controller.Playlists; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Controller.TV; +using MediaBrowser.Model.Dto; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Events; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Querying; +using MediaBrowser.Model.Serialization; +using MediaBrowser.Model.Sync; +using MediaBrowser.Model.Users; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.Tasks; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncManager : ISyncManager + { + private readonly ILibraryManager _libraryManager; + private readonly ISyncRepository _repo; + private readonly IImageProcessor _imageProcessor; + private readonly ILogger _logger; + private readonly IUserManager _userManager; + private readonly Func<IDtoService> _dtoService; + private readonly IServerApplicationHost _appHost; + private readonly ITVSeriesManager _tvSeriesManager; + private readonly Func<IMediaEncoder> _mediaEncoder; + private readonly IFileSystem _fileSystem; + private readonly Func<ISubtitleEncoder> _subtitleEncoder; + private readonly IConfigurationManager _config; + private readonly IUserDataManager _userDataManager; + private readonly Func<IMediaSourceManager> _mediaSourceManager; + private readonly IJsonSerializer _json; + private readonly ITaskManager _taskManager; + private readonly IMemoryStreamProvider _memoryStreamProvider; + + private ISyncProvider[] _providers = { }; + + public event EventHandler<GenericEventArgs<SyncJobCreationResult>> SyncJobCreated; + public event EventHandler<GenericEventArgs<SyncJob>> SyncJobCancelled; + public event EventHandler<GenericEventArgs<SyncJob>> SyncJobUpdated; + public event EventHandler<GenericEventArgs<SyncJobItem>> SyncJobItemUpdated; + public event EventHandler<GenericEventArgs<SyncJobItem>> SyncJobItemCreated; + + public SyncManager(ILibraryManager libraryManager, ISyncRepository repo, IImageProcessor imageProcessor, ILogger logger, IUserManager userManager, Func<IDtoService> dtoService, IServerApplicationHost appHost, ITVSeriesManager tvSeriesManager, Func<IMediaEncoder> mediaEncoder, IFileSystem fileSystem, Func<ISubtitleEncoder> subtitleEncoder, IConfigurationManager config, IUserDataManager userDataManager, Func<IMediaSourceManager> mediaSourceManager, IJsonSerializer json, ITaskManager taskManager, IMemoryStreamProvider memoryStreamProvider) + { + _libraryManager = libraryManager; + _repo = repo; + _imageProcessor = imageProcessor; + _logger = logger; + _userManager = userManager; + _dtoService = dtoService; + _appHost = appHost; + _tvSeriesManager = tvSeriesManager; + _mediaEncoder = mediaEncoder; + _fileSystem = fileSystem; + _subtitleEncoder = subtitleEncoder; + _config = config; + _userDataManager = userDataManager; + _mediaSourceManager = mediaSourceManager; + _json = json; + _taskManager = taskManager; + _memoryStreamProvider = memoryStreamProvider; + } + + public void AddParts(IEnumerable<ISyncProvider> providers) + { + _providers = providers.ToArray(); + } + + public IEnumerable<IServerSyncProvider> ServerSyncProviders + { + get { return _providers.OfType<IServerSyncProvider>(); } + } + + private readonly ConcurrentDictionary<string, ISyncDataProvider> _dataProviders = + new ConcurrentDictionary<string, ISyncDataProvider>(StringComparer.OrdinalIgnoreCase); + + public ISyncDataProvider GetDataProvider(IServerSyncProvider provider, SyncTarget target) + { + return _dataProviders.GetOrAdd(target.Id, key => new TargetDataProvider(provider, target, _appHost, _logger, _json, _fileSystem, _config.CommonApplicationPaths, _memoryStreamProvider)); + } + + public async Task<SyncJobCreationResult> CreateJob(SyncJobRequest request) + { + var processor = GetSyncJobProcessor(); + + var user = _userManager.GetUserById(request.UserId); + + var items = (await processor + .GetItemsForSync(request.Category, request.ParentId, request.ItemIds, user, request.UnwatchedOnly).ConfigureAwait(false)) + .ToList(); + + if (items.Any(i => !SupportsSync(i))) + { + throw new ArgumentException("Item does not support sync."); + } + + if (string.IsNullOrWhiteSpace(request.Name)) + { + if (request.ItemIds.Count == 1) + { + request.Name = GetDefaultName(_libraryManager.GetItemById(request.ItemIds[0])); + } + } + + if (string.IsNullOrWhiteSpace(request.Name)) + { + request.Name = DateTime.Now.ToString("f1", CultureInfo.CurrentCulture); + } + + var target = GetSyncTargets(request.UserId) + .FirstOrDefault(i => string.Equals(request.TargetId, i.Id)); + + if (target == null) + { + throw new ArgumentException("Sync target not found."); + } + + var jobId = Guid.NewGuid().ToString("N"); + + if (string.IsNullOrWhiteSpace(request.Quality)) + { + request.Quality = GetQualityOptions(request.TargetId) + .Where(i => i.IsDefault) + .Select(i => i.Id) + .FirstOrDefault(i => !string.IsNullOrWhiteSpace(i)); + } + + var job = new SyncJob + { + Id = jobId, + Name = request.Name, + TargetId = target.Id, + UserId = request.UserId, + UnwatchedOnly = request.UnwatchedOnly, + ItemLimit = request.ItemLimit, + RequestedItemIds = request.ItemIds ?? new List<string>(), + DateCreated = DateTime.UtcNow, + DateLastModified = DateTime.UtcNow, + SyncNewContent = request.SyncNewContent, + ItemCount = items.Count, + Category = request.Category, + ParentId = request.ParentId, + Quality = request.Quality, + Profile = request.Profile, + Bitrate = request.Bitrate + }; + + if (!request.Category.HasValue && request.ItemIds != null) + { + var requestedItems = request.ItemIds + .Select(_libraryManager.GetItemById) + .Where(i => i != null); + + // It's just a static list + if (!requestedItems.Any(i => i.IsFolder || i is IItemByName)) + { + job.SyncNewContent = false; + } + } + + await _repo.Create(job).ConfigureAwait(false); + + await processor.EnsureJobItems(job).ConfigureAwait(false); + + // If it already has a converting status then is must have been aborted during conversion + var jobItemsResult = GetJobItems(new SyncJobItemQuery + { + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + JobId = jobId, + AddMetadata = false + }); + + await processor.SyncJobItems(jobItemsResult.Items, false, new Progress<double>(), CancellationToken.None) + .ConfigureAwait(false); + + jobItemsResult = GetJobItems(new SyncJobItemQuery + { + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }, + JobId = jobId, + AddMetadata = false + }); + + var returnResult = new SyncJobCreationResult + { + Job = GetJob(jobId), + JobItems = jobItemsResult.Items.ToList() + }; + + if (SyncJobCreated != null) + { + EventHelper.FireEventIfNotNull(SyncJobCreated, this, new GenericEventArgs<SyncJobCreationResult> + { + Argument = returnResult + + }, _logger); + } + + if (returnResult.JobItems.Any(i => i.Status == SyncJobItemStatus.Queued || i.Status == SyncJobItemStatus.Converting)) + { + _taskManager.QueueScheduledTask<SyncConvertScheduledTask>(); + } + + return returnResult; + } + + public async Task UpdateJob(SyncJob job) + { + // Get fresh from the db and only update the fields that are supported to be changed. + var instance = _repo.GetJob(job.Id); + + instance.Name = job.Name; + instance.Quality = job.Quality; + instance.Profile = job.Profile; + instance.UnwatchedOnly = job.UnwatchedOnly; + instance.SyncNewContent = job.SyncNewContent; + instance.ItemLimit = job.ItemLimit; + + await _repo.Update(instance).ConfigureAwait(false); + + OnSyncJobUpdated(instance); + } + + internal void OnSyncJobUpdated(SyncJob job) + { + if (SyncJobUpdated != null) + { + EventHelper.FireEventIfNotNull(SyncJobUpdated, this, new GenericEventArgs<SyncJob> + { + Argument = job + + }, _logger); + } + } + + internal async Task UpdateSyncJobItemInternal(SyncJobItem jobItem) + { + await _repo.Update(jobItem).ConfigureAwait(false); + + if (SyncJobUpdated != null) + { + EventHelper.FireEventIfNotNull(SyncJobItemUpdated, this, new GenericEventArgs<SyncJobItem> + { + Argument = jobItem + + }, _logger); + } + } + + internal void OnSyncJobItemCreated(SyncJobItem job) + { + if (SyncJobUpdated != null) + { + EventHelper.FireEventIfNotNull(SyncJobItemCreated, this, new GenericEventArgs<SyncJobItem> + { + Argument = job + + }, _logger); + } + } + + public async Task<QueryResult<SyncJob>> GetJobs(SyncJobQuery query) + { + var result = _repo.GetJobs(query); + + foreach (var item in result.Items) + { + await FillMetadata(item).ConfigureAwait(false); + } + + return result; + } + + private async Task FillMetadata(SyncJob job) + { + var user = _userManager.GetUserById(job.UserId); + + if (user == null) + { + return; + } + + var target = GetSyncTargets(job.UserId) + .FirstOrDefault(i => string.Equals(i.Id, job.TargetId, StringComparison.OrdinalIgnoreCase)); + + if (target != null) + { + job.TargetName = target.Name; + } + + var item = job.RequestedItemIds + .Select(_libraryManager.GetItemById) + .FirstOrDefault(i => i != null); + + if (item == null) + { + var processor = GetSyncJobProcessor(); + + item = (await processor + .GetItemsForSync(job.Category, job.ParentId, job.RequestedItemIds, user, job.UnwatchedOnly).ConfigureAwait(false)) + .FirstOrDefault(); + } + + if (item != null) + { + var hasSeries = item as IHasSeries; + if (hasSeries != null) + { + job.ParentName = hasSeries.SeriesName; + } + + var hasAlbumArtist = item as IHasAlbumArtist; + if (hasAlbumArtist != null) + { + job.ParentName = hasAlbumArtist.AlbumArtists.FirstOrDefault(); + } + + var primaryImage = item.GetImageInfo(ImageType.Primary, 0); + var itemWithImage = item; + + if (primaryImage == null) + { + var parentWithImage = item.GetParents().FirstOrDefault(i => i.HasImage(ImageType.Primary)); + + if (parentWithImage != null) + { + itemWithImage = parentWithImage; + primaryImage = parentWithImage.GetImageInfo(ImageType.Primary, 0); + } + } + + if (primaryImage != null) + { + try + { + job.PrimaryImageTag = _imageProcessor.GetImageCacheTag(itemWithImage, ImageType.Primary); + job.PrimaryImageItemId = itemWithImage.Id.ToString("N"); + + } + catch (Exception ex) + { + _logger.ErrorException("Error getting image info", ex); + } + } + } + } + + private void FillMetadata(SyncJobItem jobItem) + { + var item = _libraryManager.GetItemById(jobItem.ItemId); + + if (item == null) + { + return; + } + + var primaryImage = item.GetImageInfo(ImageType.Primary, 0); + var itemWithImage = item; + + if (primaryImage == null) + { + var parentWithImage = item.GetParents().FirstOrDefault(i => i.HasImage(ImageType.Primary)); + + if (parentWithImage != null) + { + itemWithImage = parentWithImage; + primaryImage = parentWithImage.GetImageInfo(ImageType.Primary, 0); + } + } + + if (primaryImage != null) + { + try + { + jobItem.PrimaryImageTag = _imageProcessor.GetImageCacheTag(itemWithImage, ImageType.Primary); + jobItem.PrimaryImageItemId = itemWithImage.Id.ToString("N"); + + } + catch (Exception ex) + { + _logger.ErrorException("Error getting image info", ex); + } + } + } + + public async Task CancelJob(string id) + { + var job = GetJob(id); + + if (job == null) + { + throw new ArgumentException("Job not found."); + } + + await _repo.DeleteJob(id).ConfigureAwait(false); + + var path = GetSyncJobProcessor().GetTemporaryPath(id); + + try + { + _fileSystem.DeleteDirectory(path, true); + } + catch (IOException) + { + + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting directory {0}", ex, path); + } + + if (SyncJobCancelled != null) + { + EventHelper.FireEventIfNotNull(SyncJobCancelled, this, new GenericEventArgs<SyncJob> + { + Argument = job + + }, _logger); + } + } + + public SyncJob GetJob(string id) + { + return _repo.GetJob(id); + } + + public IEnumerable<SyncTarget> GetSyncTargets(string userId) + { + return _providers + .SelectMany(i => GetSyncTargets(i, userId)) + .OrderBy(i => i.Name); + } + + private IEnumerable<SyncTarget> GetSyncTargets(ISyncProvider provider) + { + return provider.GetAllSyncTargets().Select(i => new SyncTarget + { + Name = i.Name, + Id = GetSyncTargetId(provider, i) + }); + } + + private IEnumerable<SyncTarget> GetSyncTargets(ISyncProvider provider, string userId) + { + return provider.GetSyncTargets(userId).Select(i => new SyncTarget + { + Name = i.Name, + Id = GetSyncTargetId(provider, i) + }); + } + + private string GetSyncTargetId(ISyncProvider provider, SyncTarget target) + { + var hasUniqueId = provider as IHasUniqueTargetIds; + + if (hasUniqueId != null) + { + return target.Id; + } + + return target.Id; + //var providerId = GetSyncProviderId(provider); + //return (providerId + "-" + target.Id).GetMD5().ToString("N"); + } + + private string GetSyncProviderId(ISyncProvider provider) + { + return provider.GetType().Name.GetMD5().ToString("N"); + } + + public bool SupportsSync(BaseItem item) + { + if (item == null) + { + throw new ArgumentNullException("item"); + } + + if (item is Playlist) + { + return true; + } + + if (item is Person) + { + return false; + } + + if (item is Year) + { + return false; + } + + if (string.Equals(item.MediaType, MediaType.Video, StringComparison.OrdinalIgnoreCase) || + string.Equals(item.MediaType, MediaType.Audio, StringComparison.OrdinalIgnoreCase) || + string.Equals(item.MediaType, MediaType.Photo, StringComparison.OrdinalIgnoreCase) || + string.Equals(item.MediaType, MediaType.Game, StringComparison.OrdinalIgnoreCase) || + string.Equals(item.MediaType, MediaType.Book, StringComparison.OrdinalIgnoreCase)) + { + if (item.LocationType == LocationType.Virtual) + { + return false; + } + + var video = item as Video; + if (video != null) + { + if (video.IsPlaceHolder) + { + return false; + } + + if (video.IsShortcut) + { + return false; + } + } + + if (item.SourceType != SourceType.Library) + { + return false; + } + + return true; + } + + if (item.SourceType == SourceType.Channel) + { + return BaseItem.ChannelManager.SupportsSync(item.ChannelId); + } + + return item.LocationType == LocationType.FileSystem || item is Season; + } + + private string GetDefaultName(BaseItem item) + { + return item.Name; + } + + public async Task ReportSyncJobItemTransferred(string id) + { + var jobItem = _repo.GetJobItem(id); + + jobItem.Status = SyncJobItemStatus.Synced; + jobItem.Progress = 100; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(jobItem.TemporaryPath)) + { + try + { + _fileSystem.DeleteDirectory(jobItem.TemporaryPath, true); + } + catch (IOException) + { + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting temporary job file: {0}", ex, jobItem.OutputPath); + } + } + } + + private SyncJobProcessor GetSyncJobProcessor() + { + return new SyncJobProcessor(_libraryManager, _repo, this, _logger, _userManager, _tvSeriesManager, _mediaEncoder(), _subtitleEncoder(), _config, _fileSystem, _mediaSourceManager()); + } + + public SyncJobItem GetJobItem(string id) + { + return _repo.GetJobItem(id); + } + + public QueryResult<SyncJobItem> GetJobItems(SyncJobItemQuery query) + { + var result = _repo.GetJobItems(query); + + if (query.AddMetadata) + { + foreach (var item in result.Items) + { + FillMetadata(item); + } + } + + return result; + } + + private SyncedItem GetJobItemInfo(SyncJobItem jobItem) + { + var job = _repo.GetJob(jobItem.JobId); + + if (job == null) + { + _logger.Error("GetJobItemInfo job id {0} no longer exists", jobItem.JobId); + return null; + } + + var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); + + if (libraryItem == null) + { + _logger.Error("GetJobItemInfo library item with id {0} no longer exists", jobItem.ItemId); + return null; + } + + var syncedItem = new SyncedItem + { + SyncJobId = jobItem.JobId, + SyncJobItemId = jobItem.Id, + ServerId = _appHost.SystemId, + UserId = job.UserId, + SyncJobName = job.Name, + SyncJobDateCreated = job.DateCreated, + AdditionalFiles = jobItem.AdditionalFiles.Select(i => new ItemFileInfo + { + ImageType = i.ImageType, + Name = i.Name, + Type = i.Type, + Index = i.Index + + }).ToList() + }; + + var dtoOptions = new DtoOptions(); + + // Remove some bloat + dtoOptions.Fields.Remove(ItemFields.MediaStreams); + dtoOptions.Fields.Remove(ItemFields.IndexOptions); + dtoOptions.Fields.Remove(ItemFields.MediaSourceCount); + dtoOptions.Fields.Remove(ItemFields.Path); + dtoOptions.Fields.Remove(ItemFields.SeriesGenres); + dtoOptions.Fields.Remove(ItemFields.Settings); + dtoOptions.Fields.Remove(ItemFields.SyncInfo); + dtoOptions.Fields.Remove(ItemFields.BasicSyncInfo); + + syncedItem.Item = _dtoService().GetBaseItemDto(libraryItem, dtoOptions); + + var mediaSource = jobItem.MediaSource; + + syncedItem.Item.MediaSources = new List<MediaSourceInfo>(); + + syncedItem.OriginalFileName = Path.GetFileName(libraryItem.Path); + if (string.IsNullOrWhiteSpace(syncedItem.OriginalFileName)) + { + syncedItem.OriginalFileName = Path.GetFileName(mediaSource.Path); + } + + // This will be null for items that are not audio/video + if (mediaSource != null) + { + syncedItem.OriginalFileName = Path.ChangeExtension(syncedItem.OriginalFileName, Path.GetExtension(mediaSource.Path)); + syncedItem.Item.MediaSources.Add(mediaSource); + } + if (string.IsNullOrWhiteSpace(syncedItem.OriginalFileName)) + { + syncedItem.OriginalFileName = libraryItem.Name; + } + + return syncedItem; + } + + public Task ReportOfflineAction(UserAction action) + { + switch (action.Type) + { + case UserActionType.PlayedItem: + return ReportOfflinePlayedItem(action); + default: + throw new ArgumentException("Unexpected action type"); + } + } + + private Task ReportOfflinePlayedItem(UserAction action) + { + var item = _libraryManager.GetItemById(action.ItemId); + var userData = _userDataManager.GetUserData(action.UserId, item); + + userData.LastPlayedDate = action.Date; + _userDataManager.UpdatePlayState(item, userData, action.PositionTicks); + + return _userDataManager.SaveUserData(new Guid(action.UserId), item, userData, UserDataSaveReason.Import, CancellationToken.None); + } + + public async Task<List<SyncedItem>> GetReadySyncItems(string targetId) + { + var processor = GetSyncJobProcessor(); + + await processor.SyncJobItems(targetId, false, new Progress<double>(), CancellationToken.None).ConfigureAwait(false); + + var jobItemResult = GetJobItems(new SyncJobItemQuery + { + TargetId = targetId, + Statuses = new[] + { + SyncJobItemStatus.ReadyToTransfer, + SyncJobItemStatus.Transferring + } + }); + + var readyItems = jobItemResult.Items + .Select(GetJobItemInfo) + .Where(i => i != null) + .ToList(); + + _logger.Debug("Returning {0} ready sync items for targetId {1}", readyItems.Count, targetId); + + return readyItems; + } + + public async Task<SyncDataResponse> SyncData(SyncDataRequest request) + { + if (request.SyncJobItemIds != null) + { + return await SyncDataUsingSyncJobItemIds(request).ConfigureAwait(false); + } + + var jobItemResult = GetJobItems(new SyncJobItemQuery + { + TargetId = request.TargetId, + Statuses = new[] { SyncJobItemStatus.Synced } + }); + + var response = new SyncDataResponse(); + + foreach (var jobItem in jobItemResult.Items) + { + var requiresSaving = false; + var removeFromDevice = false; + + if (request.LocalItemIds.Contains(jobItem.ItemId, StringComparer.OrdinalIgnoreCase)) + { + var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); + + var job = _repo.GetJob(jobItem.JobId); + var user = _userManager.GetUserById(job.UserId); + + if (jobItem.IsMarkedForRemoval) + { + // Tell the device to remove it since it has been marked for removal + _logger.Info("Adding ItemIdsToRemove {0} because IsMarkedForRemoval is set.", jobItem.ItemId); + removeFromDevice = true; + } + else if (user == null) + { + // Tell the device to remove it since the user is gone now + _logger.Info("Adding ItemIdsToRemove {0} because the user is no longer valid.", jobItem.ItemId); + removeFromDevice = true; + } + else if (!IsLibraryItemAvailable(libraryItem)) + { + // Tell the device to remove it since it's no longer available + _logger.Info("Adding ItemIdsToRemove {0} because it is no longer available.", jobItem.ItemId); + removeFromDevice = true; + } + else if (job.UnwatchedOnly) + { + if (libraryItem is Video && libraryItem.IsPlayed(user)) + { + // Tell the device to remove it since it has been played + _logger.Info("Adding ItemIdsToRemove {0} because it has been marked played.", jobItem.ItemId); + removeFromDevice = true; + } + } + else if (libraryItem != null && libraryItem.DateModified.Ticks != jobItem.ItemDateModifiedTicks && jobItem.ItemDateModifiedTicks > 0) + { + _logger.Info("Setting status to Queued for {0} because the media has been modified since the original sync.", jobItem.ItemId); + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + requiresSaving = true; + } + } + else + { + // Content is no longer on the device + if (jobItem.IsMarkedForRemoval) + { + jobItem.Status = SyncJobItemStatus.RemovedFromDevice; + } + else + { + _logger.Info("Setting status to Queued for {0} because it is no longer on the device.", jobItem.ItemId); + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + } + requiresSaving = true; + } + + if (removeFromDevice) + { + response.ItemIdsToRemove.Add(jobItem.ItemId); + jobItem.IsMarkedForRemoval = true; + requiresSaving = true; + } + + if (requiresSaving) + { + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + } + } + + // Now check each item that's on the device + foreach (var itemId in request.LocalItemIds) + { + // See if it's already marked for removal + if (response.ItemIdsToRemove.Contains(itemId, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + // If there isn't a sync job for this item, mark it for removal + if (!jobItemResult.Items.Any(i => string.Equals(itemId, i.ItemId, StringComparison.OrdinalIgnoreCase))) + { + response.ItemIdsToRemove.Add(itemId); + } + } + + response.ItemIdsToRemove = response.ItemIdsToRemove.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + + var itemsOnDevice = request.LocalItemIds + .Except(response.ItemIdsToRemove) + .ToList(); + + SetUserAccess(request, response, itemsOnDevice); + + return response; + } + + private async Task<SyncDataResponse> SyncDataUsingSyncJobItemIds(SyncDataRequest request) + { + var jobItemResult = GetJobItems(new SyncJobItemQuery + { + TargetId = request.TargetId, + Statuses = new[] { SyncJobItemStatus.Synced } + }); + + var response = new SyncDataResponse(); + + foreach (var jobItem in jobItemResult.Items) + { + var requiresSaving = false; + var removeFromDevice = false; + + if (request.SyncJobItemIds.Contains(jobItem.Id, StringComparer.OrdinalIgnoreCase)) + { + var libraryItem = _libraryManager.GetItemById(jobItem.ItemId); + + var job = _repo.GetJob(jobItem.JobId); + var user = _userManager.GetUserById(job.UserId); + + if (jobItem.IsMarkedForRemoval) + { + // Tell the device to remove it since it has been marked for removal + _logger.Info("Adding ItemIdsToRemove {0} because IsMarkedForRemoval is set.", jobItem.Id); + removeFromDevice = true; + } + else if (user == null) + { + // Tell the device to remove it since the user is gone now + _logger.Info("Adding ItemIdsToRemove {0} because the user is no longer valid.", jobItem.Id); + removeFromDevice = true; + } + else if (!IsLibraryItemAvailable(libraryItem)) + { + // Tell the device to remove it since it's no longer available + _logger.Info("Adding ItemIdsToRemove {0} because it is no longer available.", jobItem.Id); + removeFromDevice = true; + } + else if (job.UnwatchedOnly) + { + if (libraryItem is Video && libraryItem.IsPlayed(user)) + { + // Tell the device to remove it since it has been played + _logger.Info("Adding ItemIdsToRemove {0} because it has been marked played.", jobItem.Id); + removeFromDevice = true; + } + } + else if (libraryItem != null && libraryItem.DateModified.Ticks != jobItem.ItemDateModifiedTicks && jobItem.ItemDateModifiedTicks > 0) + { + _logger.Info("Setting status to Queued for {0} because the media has been modified since the original sync.", jobItem.ItemId); + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + requiresSaving = true; + } + } + else + { + // Content is no longer on the device + if (jobItem.IsMarkedForRemoval) + { + jobItem.Status = SyncJobItemStatus.RemovedFromDevice; + } + else + { + _logger.Info("Setting status to Queued for {0} because it is no longer on the device.", jobItem.Id); + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + } + requiresSaving = true; + } + + if (removeFromDevice) + { + response.ItemIdsToRemove.Add(jobItem.Id); + jobItem.IsMarkedForRemoval = true; + requiresSaving = true; + } + + if (requiresSaving) + { + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + } + } + + // Now check each item that's on the device + foreach (var syncJobItemId in request.SyncJobItemIds) + { + // See if it's already marked for removal + if (response.ItemIdsToRemove.Contains(syncJobItemId, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + // If there isn't a sync job for this item, mark it for removal + if (!jobItemResult.Items.Any(i => string.Equals(syncJobItemId, i.Id, StringComparison.OrdinalIgnoreCase))) + { + response.ItemIdsToRemove.Add(syncJobItemId); + } + } + + response.ItemIdsToRemove = response.ItemIdsToRemove.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + + return response; + } + + private void SetUserAccess(SyncDataRequest request, SyncDataResponse response, List<string> itemIds) + { + var users = request.OfflineUserIds + .Select(_userManager.GetUserById) + .Where(i => i != null) + .ToList(); + + foreach (var itemId in itemIds) + { + var item = _libraryManager.GetItemById(itemId); + + if (item != null) + { + response.ItemUserAccess[itemId] = users + .Where(i => IsUserVisible(item, i)) + .Select(i => i.Id.ToString("N")) + .OrderBy(i => i) + .ToList(); + } + } + } + + private bool IsUserVisible(BaseItem item, User user) + { + return item.IsVisibleStandalone(user); + } + + private bool IsLibraryItemAvailable(BaseItem item) + { + if (item == null) + { + return false; + } + + return true; + } + + public async Task ReEnableJobItem(string id) + { + var jobItem = _repo.GetJobItem(id); + + if (jobItem.Status != SyncJobItemStatus.Failed && jobItem.Status != SyncJobItemStatus.Cancelled) + { + throw new ArgumentException("Operation is not valid for this job item"); + } + + jobItem.Status = SyncJobItemStatus.Queued; + jobItem.Progress = 0; + jobItem.IsMarkedForRemoval = false; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public async Task CancelItems(string targetId, IEnumerable<string> itemIds) + { + foreach (var item in itemIds) + { + var syncJobItemResult = GetJobItems(new SyncJobItemQuery + { + AddMetadata = false, + ItemId = item, + TargetId = targetId, + Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Converting, SyncJobItemStatus.Synced, SyncJobItemStatus.Failed } + }); + + foreach (var jobItem in syncJobItemResult.Items) + { + await CancelJobItem(jobItem.Id).ConfigureAwait(false); + } + } + } + + public async Task CancelJobItem(string id) + { + var jobItem = _repo.GetJobItem(id); + + if (jobItem.Status != SyncJobItemStatus.Queued && jobItem.Status != SyncJobItemStatus.ReadyToTransfer && jobItem.Status != SyncJobItemStatus.Converting && jobItem.Status != SyncJobItemStatus.Failed && jobItem.Status != SyncJobItemStatus.Synced && jobItem.Status != SyncJobItemStatus.Transferring) + { + throw new ArgumentException("Operation is not valid for this job item"); + } + + if (jobItem.Status != SyncJobItemStatus.Synced) + { + jobItem.Status = SyncJobItemStatus.Cancelled; + } + + jobItem.Progress = 0; + jobItem.IsMarkedForRemoval = true; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + + var path = processor.GetTemporaryPath(jobItem); + + try + { + _fileSystem.DeleteDirectory(path, true); + } + catch (IOException) + { + + } + catch (Exception ex) + { + _logger.ErrorException("Error deleting directory {0}", ex, path); + } + + //var jobItemsResult = GetJobItems(new SyncJobItemQuery + //{ + // AddMetadata = false, + // JobId = jobItem.JobId, + // Limit = 0, + // Statuses = new[] { SyncJobItemStatus.Converting, SyncJobItemStatus.Failed, SyncJobItemStatus.Queued, SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Synced, SyncJobItemStatus.Transferring } + //}); + + //if (jobItemsResult.TotalRecordCount == 0) + //{ + // await CancelJob(jobItem.JobId).ConfigureAwait(false); + //} + } + + public Task MarkJobItemForRemoval(string id) + { + return CancelJobItem(id); + } + + public async Task UnmarkJobItemForRemoval(string id) + { + var jobItem = _repo.GetJobItem(id); + + if (jobItem.Status != SyncJobItemStatus.Synced) + { + throw new ArgumentException("Operation is not valid for this job item"); + } + + jobItem.IsMarkedForRemoval = false; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public async Task ReportSyncJobItemTransferBeginning(string id) + { + var jobItem = _repo.GetJobItem(id); + + jobItem.Status = SyncJobItemStatus.Transferring; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public async Task ReportSyncJobItemTransferFailed(string id) + { + var jobItem = _repo.GetJobItem(id); + + jobItem.Status = SyncJobItemStatus.ReadyToTransfer; + + await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false); + + var processor = GetSyncJobProcessor(); + + await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); + } + + public Dictionary<string, SyncedItemProgress> GetSyncedItemProgresses(SyncJobItemQuery query) + { + return _repo.GetSyncedItemProgresses(query); + } + + public SyncJobOptions GetAudioOptions(SyncJobItem jobItem, SyncJob job) + { + var options = GetSyncJobOptions(jobItem.TargetId, null, null); + + if (job.Bitrate.HasValue) + { + options.DeviceProfile.MaxStaticBitrate = job.Bitrate.Value; + } + + return options; + } + + public ISyncProvider GetSyncProvider(SyncJobItem jobItem) + { + foreach (var provider in _providers) + { + foreach (var target in GetSyncTargets(provider)) + { + if (string.Equals(target.Id, jobItem.TargetId, StringComparison.OrdinalIgnoreCase)) + { + return provider; + } + } + } + return null; + } + + public SyncJobOptions GetVideoOptions(SyncJobItem jobItem, SyncJob job) + { + var options = GetSyncJobOptions(jobItem.TargetId, job.Profile, job.Quality); + + if (job.Bitrate.HasValue) + { + options.DeviceProfile.MaxStaticBitrate = job.Bitrate.Value; + } + + return options; + } + + private SyncJobOptions GetSyncJobOptions(string targetId, string profile, string quality) + { + foreach (var provider in _providers) + { + foreach (var target in GetSyncTargets(provider)) + { + if (string.Equals(target.Id, targetId, StringComparison.OrdinalIgnoreCase)) + { + return GetSyncJobOptions(provider, target, profile, quality); + } + } + } + + return GetDefaultSyncJobOptions(profile, quality); + } + + private SyncJobOptions GetSyncJobOptions(ISyncProvider provider, SyncTarget target, string profile, string quality) + { + var hasProfile = provider as IHasSyncQuality; + + if (hasProfile != null) + { + return hasProfile.GetSyncJobOptions(target, profile, quality); + } + + return GetDefaultSyncJobOptions(profile, quality); + } + + private SyncJobOptions GetDefaultSyncJobOptions(string profile, string quality) + { + var supportsAc3 = string.Equals(profile, "general", StringComparison.OrdinalIgnoreCase); + + var deviceProfile = new CloudSyncProfile(supportsAc3, false); + deviceProfile.MaxStaticBitrate = SyncHelper.AdjustBitrate(deviceProfile.MaxStaticBitrate, quality); + + return new SyncJobOptions + { + DeviceProfile = deviceProfile, + IsConverting = IsConverting(profile, quality) + }; + } + + private bool IsConverting(string profile, string quality) + { + return !string.Equals(profile, "original", StringComparison.OrdinalIgnoreCase); + } + + public IEnumerable<SyncQualityOption> GetQualityOptions(string targetId) + { + return GetQualityOptions(targetId, null); + } + + public IEnumerable<SyncQualityOption> GetQualityOptions(string targetId, User user) + { + foreach (var provider in _providers) + { + foreach (var target in GetSyncTargets(provider)) + { + if (string.Equals(target.Id, targetId, StringComparison.OrdinalIgnoreCase)) + { + return GetQualityOptions(provider, target, user); + } + } + } + + return new List<SyncQualityOption>(); + } + + private IEnumerable<SyncQualityOption> GetQualityOptions(ISyncProvider provider, SyncTarget target, User user) + { + var hasQuality = provider as IHasSyncQuality; + if (hasQuality != null) + { + var options = hasQuality.GetQualityOptions(target); + + if (user != null && !user.Policy.EnableSyncTranscoding) + { + options = options.Where(i => i.IsOriginalQuality); + } + + return options; + } + + // Default options for providers that don't override + return new List<SyncQualityOption> + { + new SyncQualityOption + { + Name = "High", + Id = "high", + IsDefault = true + }, + new SyncQualityOption + { + Name = "Medium", + Id = "medium" + }, + new SyncQualityOption + { + Name = "Low", + Id = "low" + }, + new SyncQualityOption + { + Name = "Custom", + Id = "custom" + } + }; + } + + public IEnumerable<SyncProfileOption> GetProfileOptions(string targetId, User user) + { + foreach (var provider in _providers) + { + foreach (var target in GetSyncTargets(provider)) + { + if (string.Equals(target.Id, targetId, StringComparison.OrdinalIgnoreCase)) + { + return GetProfileOptions(provider, target, user); + } + } + } + + return new List<SyncProfileOption>(); + } + + public IEnumerable<SyncProfileOption> GetProfileOptions(string targetId) + { + return GetProfileOptions(targetId, null); + } + + private IEnumerable<SyncProfileOption> GetProfileOptions(ISyncProvider provider, SyncTarget target, User user) + { + var hasQuality = provider as IHasSyncQuality; + if (hasQuality != null) + { + return hasQuality.GetProfileOptions(target); + } + + var list = new List<SyncProfileOption>(); + + list.Add(new SyncProfileOption + { + Name = "Original", + Id = "Original", + Description = "Syncs original files as-is.", + EnableQualityOptions = false + }); + + if (user == null || user.Policy.EnableSyncTranscoding) + { + list.Add(new SyncProfileOption + { + Name = "Baseline", + Id = "baseline", + Description = "Designed for compatibility with all devices, including web browsers. Targets H264/AAC video and MP3 audio." + }); + + list.Add(new SyncProfileOption + { + Name = "General", + Id = "general", + Description = "Designed for compatibility with Chromecast, Roku, Smart TV's, and other similar devices. Targets H264/AAC/AC3 video and MP3 audio.", + IsDefault = true + }); + } + + return list; + } + + protected internal void OnConversionComplete(SyncJobItem item) + { + var syncProvider = GetSyncProvider(item); + if (syncProvider is AppSyncProvider) + { + return; + } + + _taskManager.QueueIfNotRunning<ServerSyncScheduledTask>(); + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncNotificationEntryPoint.cs b/Emby.Server.Implementations/Sync/SyncNotificationEntryPoint.cs new file mode 100644 index 000000000..46cdb28a4 --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncNotificationEntryPoint.cs @@ -0,0 +1,48 @@ +using System.Threading; +using MediaBrowser.Controller.Plugins; +using MediaBrowser.Controller.Session; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Events; +using MediaBrowser.Model.Sync; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncNotificationEntryPoint : IServerEntryPoint + { + private readonly ISessionManager _sessionManager; + private readonly ISyncManager _syncManager; + + public SyncNotificationEntryPoint(ISyncManager syncManager, ISessionManager sessionManager) + { + _syncManager = syncManager; + _sessionManager = sessionManager; + } + + public void Run() + { + _syncManager.SyncJobItemUpdated += _syncManager_SyncJobItemUpdated; + } + + private async void _syncManager_SyncJobItemUpdated(object sender, GenericEventArgs<SyncJobItem> e) + { + var item = e.Argument; + + if (item.Status == SyncJobItemStatus.ReadyToTransfer) + { + try + { + await _sessionManager.SendMessageToUserDeviceSessions(item.TargetId, "SyncJobItemReady", item, CancellationToken.None).ConfigureAwait(false); + } + catch + { + + } + } + } + + public void Dispose() + { + _syncManager.SyncJobItemUpdated -= _syncManager_SyncJobItemUpdated; + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncRegistrationInfo.cs b/Emby.Server.Implementations/Sync/SyncRegistrationInfo.cs new file mode 100644 index 000000000..c2658c5c5 --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncRegistrationInfo.cs @@ -0,0 +1,31 @@ +using MediaBrowser.Common.Security; +using System.Threading.Tasks; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncRegistrationInfo : IRequiresRegistration + { + private readonly ISecurityManager _securityManager; + + public static SyncRegistrationInfo Instance; + + public SyncRegistrationInfo(ISecurityManager securityManager) + { + _securityManager = securityManager; + Instance = this; + } + + private bool _registered; + public bool IsRegistered + { + get { return _registered; } + } + + public async Task LoadRegistrationInfoAsync() + { + var info = await _securityManager.GetRegistrationStatus("sync").ConfigureAwait(false); + + _registered = info.IsValid; + } + } +} diff --git a/Emby.Server.Implementations/Sync/SyncedMediaSourceProvider.cs b/Emby.Server.Implementations/Sync/SyncedMediaSourceProvider.cs new file mode 100644 index 000000000..1e54885e6 --- /dev/null +++ b/Emby.Server.Implementations/Sync/SyncedMediaSourceProvider.cs @@ -0,0 +1,158 @@ +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Dto; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Emby.Server.Implementations.Sync +{ + public class SyncedMediaSourceProvider : IMediaSourceProvider + { + private readonly SyncManager _syncManager; + private readonly IServerApplicationHost _appHost; + private readonly ILogger _logger; + + public SyncedMediaSourceProvider(ISyncManager syncManager, IServerApplicationHost appHost, ILogger logger) + { + _appHost = appHost; + _logger = logger; + _syncManager = (SyncManager)syncManager; + } + + public async Task<IEnumerable<MediaSourceInfo>> GetMediaSources(IHasMediaSources item, CancellationToken cancellationToken) + { + var jobItemResult = _syncManager.GetJobItems(new SyncJobItemQuery + { + AddMetadata = false, + Statuses = new[] { SyncJobItemStatus.Synced }, + ItemId = item.Id.ToString("N") + }); + + var list = new List<MediaSourceInfo>(); + + if (jobItemResult.Items.Length > 0) + { + var targets = _syncManager.ServerSyncProviders + .SelectMany(i => i.GetAllSyncTargets().Select(t => new Tuple<IServerSyncProvider, SyncTarget>(i, t))) + .ToList(); + + var serverId = _appHost.SystemId; + + foreach (var jobItem in jobItemResult.Items) + { + var targetTuple = targets.FirstOrDefault(i => string.Equals(i.Item2.Id, jobItem.TargetId, StringComparison.OrdinalIgnoreCase)); + + if (targetTuple != null) + { + var syncTarget = targetTuple.Item2; + var syncProvider = targetTuple.Item1; + var dataProvider = _syncManager.GetDataProvider(targetTuple.Item1, syncTarget); + + var localItems = await dataProvider.GetItems(syncTarget, serverId, item.Id.ToString("N")).ConfigureAwait(false); + + foreach (var localItem in localItems) + { + foreach (var mediaSource in localItem.Item.MediaSources) + { + AddMediaSource(list, localItem, mediaSource, syncProvider, syncTarget); + } + } + } + } + } + + return list; + } + + private void AddMediaSource(List<MediaSourceInfo> list, + LocalItem item, + MediaSourceInfo mediaSource, + IServerSyncProvider provider, + SyncTarget target) + { + SetStaticMediaSourceInfo(item, mediaSource); + + var requiresDynamicAccess = provider as IHasDynamicAccess; + + if (requiresDynamicAccess != null) + { + mediaSource.RequiresOpening = true; + + var keyList = new List<string>(); + keyList.Add(provider.GetType().FullName.GetMD5().ToString("N")); + keyList.Add(target.Id.GetMD5().ToString("N")); + keyList.Add(item.Id); + mediaSource.OpenToken = string.Join(StreamIdDelimeterString, keyList.ToArray()); + } + + list.Add(mediaSource); + } + + // Do not use a pipe here because Roku http requests to the server will fail, without any explicit error message. + private const string StreamIdDelimeterString = "_"; + + public async Task<Tuple<MediaSourceInfo, IDirectStreamProvider>> OpenMediaSource(string openToken, CancellationToken cancellationToken) + { + var openKeys = openToken.Split(new[] { StreamIdDelimeterString[0] }, 3); + + var provider = _syncManager.ServerSyncProviders + .FirstOrDefault(i => string.Equals(openKeys[0], i.GetType().FullName.GetMD5().ToString("N"), StringComparison.OrdinalIgnoreCase)); + + var target = provider.GetAllSyncTargets() + .FirstOrDefault(i => string.Equals(openKeys[1], i.Id.GetMD5().ToString("N"), StringComparison.OrdinalIgnoreCase)); + + var dataProvider = _syncManager.GetDataProvider(provider, target); + var localItem = await dataProvider.Get(target, openKeys[2]).ConfigureAwait(false); + + var fileId = localItem.FileId; + if (string.IsNullOrWhiteSpace(fileId)) + { + } + + var requiresDynamicAccess = (IHasDynamicAccess)provider; + var dynamicInfo = await requiresDynamicAccess.GetSyncedFileInfo(fileId, target, cancellationToken).ConfigureAwait(false); + + var mediaSource = localItem.Item.MediaSources.First(); + mediaSource.LiveStreamId = Guid.NewGuid().ToString(); + SetStaticMediaSourceInfo(localItem, mediaSource); + + foreach (var stream in mediaSource.MediaStreams) + { + if (!string.IsNullOrWhiteSpace(stream.ExternalId)) + { + var dynamicStreamInfo = await requiresDynamicAccess.GetSyncedFileInfo(stream.ExternalId, target, cancellationToken).ConfigureAwait(false); + stream.Path = dynamicStreamInfo.Path; + } + } + + mediaSource.Path = dynamicInfo.Path; + mediaSource.Protocol = dynamicInfo.Protocol; + mediaSource.RequiredHttpHeaders = dynamicInfo.RequiredHttpHeaders; + + return new Tuple<MediaSourceInfo, IDirectStreamProvider>(mediaSource, null); + } + + private void SetStaticMediaSourceInfo(LocalItem item, MediaSourceInfo mediaSource) + { + mediaSource.Id = item.Id; + mediaSource.SupportsTranscoding = false; + if (mediaSource.Protocol == MediaBrowser.Model.MediaInfo.MediaProtocol.File) + { + mediaSource.ETag = item.Id; + } + } + + public Task CloseMediaSource(string liveStreamId) + { + throw new NotImplementedException(); + } + } +} diff --git a/Emby.Server.Implementations/Sync/TargetDataProvider.cs b/Emby.Server.Implementations/Sync/TargetDataProvider.cs new file mode 100644 index 000000000..a0e0f4313 --- /dev/null +++ b/Emby.Server.Implementations/Sync/TargetDataProvider.cs @@ -0,0 +1,188 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Sync; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Serialization; +using MediaBrowser.Model.Sync; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Model.IO; + +namespace Emby.Server.Implementations.Sync +{ + public class TargetDataProvider : ISyncDataProvider + { + private readonly SyncTarget _target; + private readonly IServerSyncProvider _provider; + + private readonly SemaphoreSlim _dataLock = new SemaphoreSlim(1, 1); + private List<LocalItem> _items; + + private readonly ILogger _logger; + private readonly IJsonSerializer _json; + private readonly IFileSystem _fileSystem; + private readonly IApplicationPaths _appPaths; + private readonly IServerApplicationHost _appHost; + private readonly IMemoryStreamProvider _memoryStreamProvider; + + public TargetDataProvider(IServerSyncProvider provider, SyncTarget target, IServerApplicationHost appHost, ILogger logger, IJsonSerializer json, IFileSystem fileSystem, IApplicationPaths appPaths, IMemoryStreamProvider memoryStreamProvider) + { + _logger = logger; + _json = json; + _provider = provider; + _target = target; + _fileSystem = fileSystem; + _appPaths = appPaths; + _memoryStreamProvider = memoryStreamProvider; + _appHost = appHost; + } + + private string[] GetRemotePath() + { + var parts = new List<string> + { + _appHost.FriendlyName, + "data.json" + }; + + parts = parts.Select(i => GetValidFilename(_provider, i)).ToList(); + + return parts.ToArray(); + } + + private string GetValidFilename(IServerSyncProvider provider, string filename) + { + // We can always add this method to the sync provider if it's really needed + return _fileSystem.GetValidFilename(filename); + } + + private async Task<List<LocalItem>> RetrieveItems(CancellationToken cancellationToken) + { + _logger.Debug("Getting {0} from {1}", string.Join(MediaSync.PathSeparatorString, GetRemotePath().ToArray()), _provider.Name); + + var fileResult = await _provider.GetFiles(GetRemotePath().ToArray(), _target, cancellationToken).ConfigureAwait(false); + + if (fileResult.Items.Length > 0) + { + using (var stream = await _provider.GetFile(fileResult.Items[0].FullName, _target, new Progress<double>(), cancellationToken)) + { + return _json.DeserializeFromStream<List<LocalItem>>(stream); + } + } + + return new List<LocalItem>(); + } + + private async Task EnsureData(CancellationToken cancellationToken) + { + if (_items == null) + { + _items = await RetrieveItems(cancellationToken).ConfigureAwait(false); + } + } + + private async Task SaveData(List<LocalItem> items, CancellationToken cancellationToken) + { + using (var stream = _memoryStreamProvider.CreateNew()) + { + _json.SerializeToStream(items, stream); + + // Save to sync provider + stream.Position = 0; + var remotePath = GetRemotePath(); + _logger.Debug("Saving data.json to {0}. Remote path: {1}", _provider.Name, string.Join("/", remotePath)); + + await _provider.SendFile(stream, remotePath, _target, new Progress<double>(), cancellationToken).ConfigureAwait(false); + } + } + + private async Task<T> GetData<T>(bool enableCache, Func<List<LocalItem>, T> dataFactory) + { + if (!enableCache) + { + var items = await RetrieveItems(CancellationToken.None).ConfigureAwait(false); + var newCache = items.ToList(); + var result = dataFactory(items); + await UpdateCache(newCache).ConfigureAwait(false); + return result; + } + + await _dataLock.WaitAsync().ConfigureAwait(false); + + try + { + await EnsureData(CancellationToken.None).ConfigureAwait(false); + + return dataFactory(_items); + } + finally + { + _dataLock.Release(); + } + } + + private async Task UpdateData(Func<List<LocalItem>, List<LocalItem>> action) + { + var items = await RetrieveItems(CancellationToken.None).ConfigureAwait(false); + items = action(items); + await SaveData(items.ToList(), CancellationToken.None).ConfigureAwait(false); + + await UpdateCache(null).ConfigureAwait(false); + } + + private async Task UpdateCache(List<LocalItem> list) + { + await _dataLock.WaitAsync().ConfigureAwait(false); + + try + { + _items = list; + } + finally + { + _dataLock.Release(); + } + } + + public Task<List<LocalItem>> GetLocalItems(SyncTarget target, string serverId) + { + return GetData(false, items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).ToList()); + } + + public Task AddOrUpdate(SyncTarget target, LocalItem item) + { + return UpdateData(items => + { + var list = items.Where(i => !string.Equals(i.Id, item.Id, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + list.Add(item); + + return list; + }); + } + + public Task Delete(SyncTarget target, string id) + { + return UpdateData(items => items.Where(i => !string.Equals(i.Id, id, StringComparison.OrdinalIgnoreCase)).ToList()); + } + + public Task<LocalItem> Get(SyncTarget target, string id) + { + return GetData(true, items => items.FirstOrDefault(i => string.Equals(i.Id, id, StringComparison.OrdinalIgnoreCase))); + } + + public Task<List<LocalItem>> GetItems(SyncTarget target, string serverId, string itemId) + { + return GetData(true, items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.ItemId, itemId, StringComparison.OrdinalIgnoreCase)).ToList()); + } + + public Task<List<LocalItem>> GetItemsBySyncJobItemId(SyncTarget target, string serverId, string syncJobItemId) + { + return GetData(false, items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.SyncJobItemId, syncJobItemId, StringComparison.OrdinalIgnoreCase)).ToList()); + } + } +} |
