fixes #585 - Use tmdb updates api for people

5 files changed, 318 insertions, 24 deletions

diff --git a/MediaBrowser.Providers/MediaBrowser.Providers.csproj b/MediaBrowser.Providers/MediaBrowser.Providers.csproj
index 9d664b370..972578b7d 100644
--- a/MediaBrowser.Providers/MediaBrowser.Providers.csproj
+++ b/MediaBrowser.Providers/MediaBrowser.Providers.csproj
@@ -67,6 +67,7 @@
     <Compile Include="Movies\MovieProviderFromXml.cs" />
     <Compile Include="Movies\OpenMovieDatabaseProvider.cs" />
     <Compile Include="Movies\PersonProviderFromXml.cs" />
+    <Compile Include="Movies\PersonUpdatesPreScanTask.cs" />
     <Compile Include="Movies\TmdbPersonProvider.cs" />
     <Compile Include="Music\AlbumInfoFromSongProvider.cs" />
     <Compile Include="Music\ArtistInfoFromSongProvider.cs" />
diff --git a/MediaBrowser.Providers/Movies/MovieDbProvider.cs b/MediaBrowser.Providers/Movies/MovieDbProvider.cs
index 424070f70..c61e37bde 100644
--- a/MediaBrowser.Providers/Movies/MovieDbProvider.cs
+++ b/MediaBrowser.Providers/Movies/MovieDbProvider.cs
@@ -31,7 +31,7 @@ namespace MediaBrowser.Providers.Movies
         /// <summary>
         /// The movie db
         /// </summary>
-        private readonly SemaphoreSlim _movieDbResourcePool = new SemaphoreSlim(1, 1);
+        internal readonly SemaphoreSlim MovieDbResourcePool = new SemaphoreSlim(1, 1);
 
         internal static MovieDbProvider Current { get; private set; }
 
@@ -72,7 +72,7 @@ namespace MediaBrowser.Providers.Movies
         {
             if (dispose)
             {
-                _movieDbResourcePool.Dispose();
+                MovieDbResourcePool.Dispose();
             }
         }
 
@@ -726,7 +726,7 @@ namespace MediaBrowser.Providers.Movies
         {
             var cancellationToken = options.CancellationToken;
 
-            await _movieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
+            await MovieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
 
             try
             {
@@ -746,7 +746,7 @@ namespace MediaBrowser.Providers.Movies
             {
                 _lastRequestDate = DateTime.Now;
 
-                _movieDbResourcePool.Release();
+                MovieDbResourcePool.Release();
             }
         }
 
diff --git a/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs b/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs
index 4356c9e2e..5ee3f1141 100644
--- a/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs
+++ b/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs
@@ -46,7 +46,7 @@ namespace MediaBrowser.Providers.Movies
         {
             get
             {
-                return "11";
+                return "12";
             }
         }
 
diff --git a/MediaBrowser.Providers/Movies/PersonUpdatesPreScanTask.cs b/MediaBrowser.Providers/Movies/PersonUpdatesPreScanTask.cs
new file mode 100644
index 000000000..0bc85f934
--- /dev/null
+++ b/MediaBrowser.Providers/Movies/PersonUpdatesPreScanTask.cs
@@ -0,0 +1,227 @@
+using MediaBrowser.Common.Net;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Model.Logging;
+using MediaBrowser.Model.Net;
+using MediaBrowser.Model.Serialization;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MediaBrowser.Providers.Movies
+{
+    public class PersonUpdatesPreScanTask : IPeoplePrescanTask
+    {
+        /// <summary>
+        /// The updates URL
+        /// </summary>
+        private const string UpdatesUrl = "http://api.themoviedb.org/3/person/changes?start_date={0}&api_key={1}&page={2}";
+
+        /// <summary>
+        /// The _HTTP client
+        /// </summary>
+        private readonly IHttpClient _httpClient;
+        /// <summary>
+        /// The _logger
+        /// </summary>
+        private readonly ILogger _logger;
+        /// <summary>
+        /// The _config
+        /// </summary>
+        private readonly IServerConfigurationManager _config;
+        private readonly IJsonSerializer _json;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="PersonUpdatesPreScanTask"/> class.
+        /// </summary>
+        /// <param name="logger">The logger.</param>
+        /// <param name="httpClient">The HTTP client.</param>
+        /// <param name="config">The config.</param>
+        public PersonUpdatesPreScanTask(ILogger logger, IHttpClient httpClient, IServerConfigurationManager config, IJsonSerializer json)
+        {
+            _logger = logger;
+            _httpClient = httpClient;
+            _config = config;
+            _json = json;
+        }
+
+        protected readonly CultureInfo UsCulture = new CultureInfo("en-US");
+
+        /// <summary>
+        /// Runs the specified progress.
+        /// </summary>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            if (!_config.Configuration.EnableInternetProviders)
+            {
+                progress.Report(100);
+                return;
+            }
+
+            var path = TmdbPersonProvider.GetPersonsDataPath(_config.CommonApplicationPaths);
+
+            Directory.CreateDirectory(path);
+
+            var timestampFile = Path.Combine(path, "time.txt");
+
+            var timestampFileInfo = new FileInfo(timestampFile);
+
+            // Don't check for tvdb updates anymore frequently than 24 hours
+            if (timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < 1)
+            {
+                return;
+            }
+
+            // Find out the last time we queried tvdb for updates
+            var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
+
+            var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList();
+
+            if (!string.IsNullOrEmpty(lastUpdateTime))
+            {
+                long lastUpdateTicks;
+
+                if (long.TryParse(lastUpdateTime, NumberStyles.Any, UsCulture, out lastUpdateTicks))
+                {
+                    var lastUpdateDate = new DateTime(lastUpdateTicks, DateTimeKind.Utc);
+
+                    var updatedIds = await GetIdsToUpdate(lastUpdateDate, 1, cancellationToken).ConfigureAwait(false);
+
+                    var existingDictionary = existingDirectories.ToDictionary(i => i, StringComparer.OrdinalIgnoreCase); 
+                    
+                    var idsToUpdate = updatedIds.Where(i => !string.IsNullOrWhiteSpace(i) && existingDictionary.ContainsKey(i));
+
+                    await UpdatePeople(idsToUpdate, path, progress, cancellationToken).ConfigureAwait(false);
+                }
+            }
+
+            File.WriteAllText(timestampFile, DateTime.UtcNow.Ticks.ToString(UsCulture), Encoding.UTF8);
+            progress.Report(100);
+        }
+
+        /// <summary>
+        /// Gets the ids to update.
+        /// </summary>
+        /// <param name="lastUpdateTime">The last update time.</param>
+        /// <param name="page">The page.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task{IEnumerable{System.String}}.</returns>
+        private async Task<IEnumerable<string>> GetIdsToUpdate(DateTime lastUpdateTime, int page, CancellationToken cancellationToken)
+        {
+            var hasMorePages = false;
+            var list = new List<string>();
+
+            // First get last time
+            using (var stream = await _httpClient.Get(new HttpRequestOptions
+            {
+                Url = string.Format(UpdatesUrl, lastUpdateTime.ToString("yyyy-MM-dd"), MovieDbProvider.ApiKey, page),
+                CancellationToken = cancellationToken,
+                EnableHttpCompression = true,
+                ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
+                AcceptHeader = MovieDbProvider.AcceptHeader
+
+            }).ConfigureAwait(false))
+            {
+                var obj = _json.DeserializeFromStream<RootObject>(stream);
+
+                var data = obj.results.Select(i => i.id.ToString(UsCulture));
+
+                list.AddRange(data);
+
+                hasMorePages = page < obj.total_pages;
+            }
+
+            if (hasMorePages)
+            {
+                var more = await GetIdsToUpdate(lastUpdateTime, page + 1, cancellationToken).ConfigureAwait(false);
+
+                list.AddRange(more);
+            }
+
+            return list;
+        }
+
+        /// <summary>
+        /// Updates the people.
+        /// </summary>
+        /// <param name="ids">The ids.</param>
+        /// <param name="peopleDataPath">The people data path.</param>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        private async Task UpdatePeople(IEnumerable<string> ids, string peopleDataPath, IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            var list = ids.ToList();
+            var numComplete = 0;
+
+            foreach (var id in list)
+            {
+                try
+                {
+                    await UpdatePerson(id, peopleDataPath, cancellationToken).ConfigureAwait(false);
+                }
+                catch (HttpException ex)
+                {
+                    // Already logged at lower levels, but don't fail the whole operation, unless timed out
+                    // We have to fail this to make it run again otherwise new episode data could potentially be missing
+                    if (ex.IsTimedOut)
+                    {
+                        throw;
+                    }
+                }
+
+                numComplete++;
+                double percent = numComplete;
+                percent /= list.Count;
+                percent *= 100;
+
+                progress.Report(percent);
+            }
+        }
+
+        /// <summary>
+        /// Updates the person.
+        /// </summary>
+        /// <param name="id">The id.</param>
+        /// <param name="peopleDataPath">The people data path.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        private Task UpdatePerson(string id, string peopleDataPath, CancellationToken cancellationToken)
+        {
+            _logger.Info("Updating person from tmdb " + id);
+
+            var personDataPath = Path.Combine(peopleDataPath, id);
+
+            Directory.CreateDirectory(peopleDataPath);
+
+            return TmdbPersonProvider.Current.DownloadPersonInfo(id, personDataPath, cancellationToken);
+        }
+
+        class Result
+        {
+            public int id { get; set; }
+            public bool? adult { get; set; }
+        }
+
+        class RootObject
+        {
+            public List<Result> results { get; set; }
+            public int page { get; set; }
+            public int total_pages { get; set; }
+            public int total_results { get; set; }
+
+            public RootObject()
+            {
+                results = new List<Result>();
+            }
+        }
+    }
+}
diff --git a/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs
index fdc4b86c5..f0c8e3dca 100644
--- a/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs
+++ b/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs
@@ -1,4 +1,6 @@
-using MediaBrowser.Common.Net;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Common.IO;
+using MediaBrowser.Common.Net;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Library;
@@ -24,6 +26,8 @@ namespace MediaBrowser.Providers.Movies
     {
         protected readonly IProviderManager ProviderManager;
 
+        internal static TmdbPersonProvider Current { get; private set; }
+        
         public TmdbPersonProvider(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
             : base(logManager, configurationManager)
         {
@@ -33,6 +37,7 @@ namespace MediaBrowser.Providers.Movies
             }
             JsonSerializer = jsonSerializer;
             ProviderManager = providerManager;
+            Current = this;
         }
 
         /// <summary>
@@ -75,12 +80,49 @@ namespace MediaBrowser.Providers.Movies
             }
         }
 
-        protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo)
+        protected override bool NeedsRefreshBasedOnCompareDate(BaseItem item, BaseProviderInfo providerInfo)
         {
-            if (HasAltMeta(item))
-                return false;
+            var provderId = item.GetProviderId(MetadataProviders.Tmdb);
 
-            return base.NeedsRefreshInternal(item, providerInfo);
+            if (!string.IsNullOrEmpty(provderId))
+            {
+                // Process images
+                var path = GetPersonDataPath(ConfigurationManager.ApplicationPaths, provderId);
+
+                try
+                {
+                    var files = new DirectoryInfo(path)
+                        .EnumerateFiles("*.json", SearchOption.TopDirectoryOnly)
+                        .Select(i => i.LastWriteTimeUtc)
+                        .ToList();
+
+                    if (files.Count > 0)
+                    {
+                        return files.Max() > providerInfo.LastRefreshed;
+                    }
+                }
+                catch (DirectoryNotFoundException)
+                {
+                    // Don't blow up
+                    return true;
+                }
+            }
+
+            return base.NeedsRefreshBasedOnCompareDate(item, providerInfo);
+        }
+
+        internal static string GetPersonDataPath(IApplicationPaths appPaths, string tmdbId)
+        {
+            var seriesDataPath = Path.Combine(GetPersonsDataPath(appPaths), tmdbId);
+
+            return seriesDataPath;
+        }
+
+        internal static string GetPersonsDataPath(IApplicationPaths appPaths)
+        {
+            var dataPath = Path.Combine(appPaths.DataPath, "tmdb-people");
+
+            return dataPath;
         }
 
         private bool HasAltMeta(BaseItem item)
@@ -181,29 +223,53 @@ namespace MediaBrowser.Providers.Movies
         /// <returns>Task.</returns>
         private async Task FetchInfo(Person person, string id, CancellationToken cancellationToken)
         {
-            string url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id);
-            PersonResult searchResult = null;
+            var personDataPath = GetPersonDataPath(ConfigurationManager.ApplicationPaths, id);
 
-            using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
-            {
-                Url = url,
-                CancellationToken = cancellationToken,
-                AcceptHeader = MovieDbProvider.AcceptHeader
+            Directory.CreateDirectory(personDataPath);
 
-            }).ConfigureAwait(false))
+            var files = Directory.EnumerateFiles(personDataPath, "*.json", SearchOption.TopDirectoryOnly)
+                .Select(Path.GetFileName)
+                .ToList();
+
+            const string dataFileName = "info.json";
+
+            // Only download if not already there
+            // The prescan task will take care of updates so we don't need to re-download here
+            if (!files.Contains(dataFileName, StringComparer.OrdinalIgnoreCase))
             {
-                searchResult = JsonSerializer.DeserializeFromStream<PersonResult>(json);
+                await DownloadPersonInfo(id, personDataPath, cancellationToken).ConfigureAwait(false);
             }
 
-            cancellationToken.ThrowIfCancellationRequested();
-
-            if (searchResult != null)
+            //if (!HasAltMeta(person))
             {
-                ProcessInfo(person, searchResult);
+                var info = JsonSerializer.DeserializeFromFile<PersonResult>(Path.Combine(personDataPath, dataFileName));
+
+                cancellationToken.ThrowIfCancellationRequested();
+
+                ProcessInfo(person, info);
 
                 Logger.Debug("TmdbPersonProvider downloaded and saved information for {0}", person.Name);
 
-                await FetchImages(person, searchResult.images, cancellationToken).ConfigureAwait(false);
+                await FetchImages(person, info.images, cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+        internal async Task DownloadPersonInfo(string id, string personDataPath, CancellationToken cancellationToken)
+        {
+            var url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id);
+
+            using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
+            {
+                Url = url,
+                CancellationToken = cancellationToken,
+                AcceptHeader = MovieDbProvider.AcceptHeader
+
+            }).ConfigureAwait(false))
+            {
+                using (var fs = new FileStream(Path.Combine(personDataPath, "info.json"), FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, true))
+                {
+                    await json.CopyToAsync(fs).ConfigureAwait(false);
+                }
             }
         }