rework server sync

4 files changed, 66 insertions, 45 deletions

diff --git a/MediaBrowser.Server.Implementations/Sync/MediaSync.cs b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs
index ad313670a..1a8b55a25 100644
--- a/MediaBrowser.Server.Implementations/Sync/MediaSync.cs
+++ b/MediaBrowser.Server.Implementations/Sync/MediaSync.cs
@@ -1,5 +1,4 @@
-using System.Globalization;
-using MediaBrowser.Common.Configuration;
+using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.IO;
 using MediaBrowser.Common.Progress;
 using MediaBrowser.Controller;
@@ -12,12 +11,14 @@ using MediaBrowser.Model.MediaInfo;
 using MediaBrowser.Model.Sync;
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Patterns.IO;
 
 namespace MediaBrowser.Server.Implementations.Sync
 {
@@ -29,6 +30,9 @@ namespace MediaBrowser.Server.Implementations.Sync
         private readonly IFileSystem _fileSystem;
         private readonly IConfigurationManager _config;
 
+        public const string PathSeparatorString = "/";
+        public const char PathSeparatorChar = '/';
+
         public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost, IFileSystem fileSystem, IConfigurationManager config)
         {
             _logger = logger;
@@ -71,7 +75,24 @@ namespace MediaBrowser.Server.Implementations.Sync
             SyncTarget target,
             CancellationToken cancellationToken)
         {
-            var jobItemIds = await dataProvider.GetSyncJobItemIds(target, serverId).ConfigureAwait(false);
+            var localItems = await dataProvider.GetLocalItems(target, serverId).ConfigureAwait(false);
+            var remoteFiles = await provider.GetFiles(new FileQuery(), target, cancellationToken).ConfigureAwait(false);
+            var remoteIds = remoteFiles.Items.Select(i => i.Id).ToList();
+
+            var jobItemIds = new List<string>();
+
+            foreach (var localItem in localItems)
+            {
+                // TODO: Remove this after a while
+                if (string.IsNullOrWhiteSpace(localItem.FileId))
+                {
+                    jobItemIds.Add(localItem.SyncJobItemId);
+                }
+                else if (remoteIds.Contains(localItem.FileId, StringComparer.OrdinalIgnoreCase))
+                {
+                    jobItemIds.Add(localItem.SyncJobItemId);
+                }
+            }
 
             var result = await _syncManager.SyncData(new SyncDataRequest
             {
@@ -163,7 +184,7 @@ namespace MediaBrowser.Server.Implementations.Sync
                 var fileTransferProgress = new ActionableProgress<double>();
                 fileTransferProgress.RegisterAction(pct => progress.Report(pct * .92));
 
-                var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath, target, options, fileTransferProgress, cancellationToken).ConfigureAwait(false);
+                var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath.Split(PathSeparatorChar), target, options, fileTransferProgress, cancellationToken).ConfigureAwait(false);
 
                 if (localItem.Item.MediaSources != null)
                 {
@@ -177,6 +198,8 @@ namespace MediaBrowser.Server.Implementations.Sync
                     }
                 }
 
+                localItem.FileId = sendFileResult.Id;
+
                 // Create db record
                 await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false);
 
@@ -228,10 +251,10 @@ namespace MediaBrowser.Server.Implementations.Sync
                     var sendFileResult = await SendFile(provider, mediaStream.Path, remotePath, target, options, new Progress<double>(), cancellationToken).ConfigureAwait(false);
 
                     // This is the path that will be used when talking to the provider
-                    mediaStream.ExternalId = remotePath;
+                    mediaStream.ExternalId = sendFileResult.Id;
 
                     // Keep track of all additional files for cleanup later.
-                    localItem.AdditionalFiles.Add(remotePath);
+                    localItem.AdditionalFiles.Add(sendFileResult.Id);
 
                     // This is the public path clients will use
                     mediaStream.Path = sendFileResult.Path;
@@ -256,17 +279,15 @@ namespace MediaBrowser.Server.Implementations.Sync
             }
         }
 
-        private string GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target)
+        private string[] GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target)
         {
-            var path = item.LocalPath;
-
             var filename = GetSubtitleSaveFileName(item, stream.Language, stream.IsForced) + "." + stream.Codec.ToLower();
 
-            var parentPath = provider.GetParentDirectoryPath(path, target);
-
-            path = Path.Combine(parentPath, filename);
+            var pathParts = item.LocalPath.Split(PathSeparatorChar);
+            var list = pathParts.Take(pathParts.Length - 1).ToList();
+            list.Add(filename);
 
-            return path;
+            return list.ToArray();
         }
 
         private string GetSubtitleSaveFileName(LocalItem item, string language, bool isForced)
@@ -300,12 +321,16 @@ namespace MediaBrowser.Server.Implementations.Sync
             foreach (var localItem in localItems)
             {
                 var files = localItem.AdditionalFiles.ToList();
-                files.Insert(0, localItem.LocalPath);
+
+                // TODO: Remove this. Have to check it for now since this is a new property
+                if (!string.IsNullOrWhiteSpace(localItem.FileId))
+                {
+                    files.Insert(0, localItem.FileId);
+                }
 
                 foreach (var file in files)
                 {
                     _logger.Debug("Removing {0} from {1}.", file, target.Name);
-
                     await provider.DeleteFile(file, target, cancellationToken).ConfigureAwait(false);
                 }
 
@@ -313,9 +338,9 @@ namespace MediaBrowser.Server.Implementations.Sync
             }
         }
 
-        private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string remotePath, SyncTarget target, SyncOptions options, IProgress<double> progress, CancellationToken cancellationToken)
+        private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string[] pathParts, SyncTarget target, SyncOptions options, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            _logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, remotePath);
+            _logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, string.Join("/", pathParts));
             using (var fileStream = _fileSystem.GetFileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read, true))
             {
                 Stream stream = fileStream;
@@ -325,7 +350,7 @@ namespace MediaBrowser.Server.Implementations.Sync
                     stream = new ThrottledStream(stream, options.UploadSpeedLimitBytes);
                 }
 
-                return await provider.SendFile(stream, remotePath, target, progress, cancellationToken).ConfigureAwait(false);
+                return await provider.SendFile(stream, pathParts, target, progress, cancellationToken).ConfigureAwait(false);
             }
         }
 
@@ -349,7 +374,7 @@ namespace MediaBrowser.Server.Implementations.Sync
             var path = GetDirectoryPath(provider, job, syncedItem, libraryItem, serverName);
             path.Add(GetLocalFileName(provider, libraryItem, originalFileName));
 
-            var localPath = provider.GetFullPath(path, target);
+            var localPath = string.Join(PathSeparatorString, path.ToArray());
 
             foreach (var mediaSource in libraryItem.MediaSources)
             {
diff --git a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs
index dbe43c2d1..6f0310e3c 100644
--- a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs
+++ b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs
@@ -793,8 +793,6 @@ namespace MediaBrowser.Server.Implementations.Sync
                 }
                 else
                 {
-                    _logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.ItemId);
-
                     // Content is no longer on the device
                     if (jobItem.IsMarkedForRemoval)
                     {
@@ -802,6 +800,7 @@ namespace MediaBrowser.Server.Implementations.Sync
                     }
                     else
                     {
+                        _logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.ItemId);
                         jobItem.Status = SyncJobItemStatus.Queued;
                     }
                     requiresSaving = true;
@@ -902,8 +901,6 @@ namespace MediaBrowser.Server.Implementations.Sync
                 }
                 else
                 {
-                    _logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.Id);
-
                     // Content is no longer on the device
                     if (jobItem.IsMarkedForRemoval)
                     {
@@ -911,6 +908,7 @@ namespace MediaBrowser.Server.Implementations.Sync
                     }
                     else
                     {
+                        _logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.Id);
                         jobItem.Status = SyncJobItemStatus.Queued;
                     }
                     requiresSaving = true;
diff --git a/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs b/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs
index 40d40d415..f7f320741 100644
--- a/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs
+++ b/MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs
@@ -109,8 +109,13 @@ namespace MediaBrowser.Server.Implementations.Sync
             var dataProvider = _syncManager.GetDataProvider(provider, target);
             var localItem = await dataProvider.Get(target, openKeys[2]).ConfigureAwait(false);
 
+            var fileId = localItem.FileId;
+            if (string.IsNullOrWhiteSpace(fileId))
+            {
+            }
+
             var requiresDynamicAccess = (IHasDynamicAccess)provider;
-            var dynamicInfo = await requiresDynamicAccess.GetSyncedFileInfo(localItem.LocalPath, target, cancellationToken).ConfigureAwait(false);
+            var dynamicInfo = await requiresDynamicAccess.GetSyncedFileInfo(fileId, target, cancellationToken).ConfigureAwait(false);
 
             var mediaSource = localItem.Item.MediaSources.First();
             mediaSource.LiveStreamId = Guid.NewGuid().ToString();
diff --git a/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs b/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs
index 3323ae148..dc95ee316 100644
--- a/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs
+++ b/MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs
@@ -1,5 +1,4 @@
 using MediaBrowser.Common.Configuration;
-using MediaBrowser.Common.Extensions;
 using MediaBrowser.Common.IO;
 using MediaBrowser.Controller;
 using MediaBrowser.Controller.Sync;
@@ -12,6 +11,7 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Patterns.IO;
 
 namespace MediaBrowser.Server.Implementations.Sync
 {
@@ -29,8 +29,6 @@ namespace MediaBrowser.Server.Implementations.Sync
         private readonly IApplicationPaths _appPaths;
         private readonly IServerApplicationHost _appHost;
 
-        private readonly SemaphoreSlim _cacheFileLock = new SemaphoreSlim(1, 1);
-
         public TargetDataProvider(IServerSyncProvider provider, SyncTarget target, IServerApplicationHost appHost, ILogger logger, IJsonSerializer json, IFileSystem fileSystem, IApplicationPaths appPaths)
         {
             _logger = logger;
@@ -42,7 +40,7 @@ namespace MediaBrowser.Server.Implementations.Sync
             _appHost = appHost;
         }
 
-        private string GetRemotePath()
+        private string[] GetRemotePath()
         {
             var parts = new List<string>
             {
@@ -52,7 +50,7 @@ namespace MediaBrowser.Server.Implementations.Sync
 
             parts = parts.Select(i => GetValidFilename(_provider, i)).ToList();
 
-            return _provider.GetFullPath(parts, _target);
+            return parts.ToArray();
         }
 
         private string GetValidFilename(IServerSyncProvider provider, string filename)
@@ -65,22 +63,22 @@ namespace MediaBrowser.Server.Implementations.Sync
         {
             if (_items == null)
             {
-                try
+                _logger.Debug("Getting {0} from {1}", string.Join(MediaSync.PathSeparatorString, GetRemotePath().ToArray()), _provider.Name);
+
+                var fileResult = await _provider.GetFiles(new FileQuery
                 {
-                    var path = GetRemotePath();
+                    FullPath = GetRemotePath().ToArray()
 
-                    _logger.Debug("Getting {0} from {1}", path, _provider.Name);
+                }, _target, cancellationToken).ConfigureAwait(false);
 
-                    using (var stream = await _provider.GetFile(path, _target, new Progress<double>(), cancellationToken))
+                if (fileResult.Items.Length > 0)
+                {
+                    using (var stream = await _provider.GetFile(fileResult.Items[0].Id, _target, new Progress<double>(), cancellationToken))
                     {
                         _items = _json.DeserializeFromStream<List<LocalItem>>(stream);
                     }
                 }
-                catch (FileNotFoundException)
-                {
-                    _items = new List<LocalItem>();
-                }
-                catch (DirectoryNotFoundException)
+                else
                 {
                     _items = new List<LocalItem>();
                 }
@@ -133,14 +131,9 @@ namespace MediaBrowser.Server.Implementations.Sync
             }
         }
 
-        public Task<List<string>> GetServerItemIds(SyncTarget target, string serverId)
-        {
-            return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).Select(i => i.ItemId).ToList());
-        }
-
-        public Task<List<string>> GetSyncJobItemIds(SyncTarget target, string serverId)
+        public Task<List<LocalItem>> GetLocalItems(SyncTarget target, string serverId)
         {
-            return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).Select(i => i.SyncJobItemId).Where(i => !string.IsNullOrWhiteSpace(i)).ToList());
+            return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).ToList());
         }
 
         public Task AddOrUpdate(SyncTarget target, LocalItem item)