diff options
5 files changed, 314 insertions, 40 deletions
diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index d5eac2cee..1a97201d8 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -75,6 +75,7 @@ <Compile Include="Entities\Audio\MusicAlbumDisc.cs" /> <Compile Include="Library\ILibraryPostScanTask.cs" /> <Compile Include="Library\ILibraryPrescanTask.cs" /> + <Compile Include="Providers\Movies\FanArtMovieUpdatesPrescanTask.cs" /> <Compile Include="Providers\Movies\MovieDbImagesProvider.cs" /> <Compile Include="Providers\Music\ArtistsPostScanTask.cs" /> <Compile Include="Providers\Music\FanArtUpdatesPrescanTask.cs" /> diff --git a/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs b/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs index 75aee1a04..dc71a860c 100644 --- a/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -7,6 +9,8 @@ using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using System; using System.Globalization; +using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -16,7 +20,7 @@ namespace MediaBrowser.Controller.Providers.Movies /// <summary> /// Class FanArtMovieProvider /// </summary> - class FanArtMovieProvider : FanartBaseProvider, IDisposable + class FanArtMovieProvider : FanartBaseProvider { /// <summary> /// Gets the HTTP client. @@ -24,10 +28,18 @@ namespace MediaBrowser.Controller.Providers.Movies /// <value>The HTTP client.</value> protected IHttpClient HttpClient { get; private set; } + /// <summary> + /// The _provider manager + /// </summary> private readonly IProviderManager _providerManager; + /// <summary> + /// The us culture + /// </summary> private static readonly CultureInfo UsCulture = new CultureInfo("en-US"); + internal static FanArtMovieProvider Current { get; private set; } + /// <summary> /// Initializes a new instance of the <see cref="FanArtMovieProvider" /> class. /// </summary> @@ -45,18 +57,7 @@ namespace MediaBrowser.Controller.Providers.Movies } HttpClient = httpClient; _providerManager = providerManager; - } - - /// <summary> - /// Releases unmanaged and - optionally - managed resources. - /// </summary> - /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param> - protected virtual void Dispose(bool dispose) - { - if (dispose) - { - FanArtResourcePool.Dispose(); - } + Current = this; } /// <summary> @@ -79,7 +80,7 @@ namespace MediaBrowser.Controller.Providers.Movies { get { - return "12"; + return "13"; } } @@ -140,10 +141,62 @@ namespace MediaBrowser.Controller.Providers.Movies /// <summary> /// Gets the comparison data. /// </summary> + /// <param name="id">The id.</param> /// <returns>Guid.</returns> private Guid GetComparisonData(string id) { - return string.IsNullOrEmpty(id) ? Guid.Empty : id.GetMD5(); + if (!string.IsNullOrEmpty(id)) + { + // Process images + var path = GetMovieDataPath(ConfigurationManager.ApplicationPaths, id); + + var files = new DirectoryInfo(path) + .EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly) + .Select(i => i.FullName + i.LastWriteTimeUtc.Ticks) + .ToArray(); + + if (files.Length > 0) + { + return string.Join(string.Empty, files).GetMD5(); + } + } + + return Guid.Empty; + } + + /// <summary> + /// Gets the movie data path. + /// </summary> + /// <param name="appPaths">The app paths.</param> + /// <param name="tmdbId">The TMDB id.</param> + /// <returns>System.String.</returns> + internal static string GetMovieDataPath(IApplicationPaths appPaths, string tmdbId) + { + var dataPath = Path.Combine(GetMoviesDataPath(appPaths), tmdbId); + + if (!Directory.Exists(dataPath)) + { + Directory.CreateDirectory(dataPath); + } + + return dataPath; + } + + /// <summary> + /// Gets the movie data path. + /// </summary> + /// <param name="appPaths">The app paths.</param> + /// <returns>System.String.</returns> + internal static string GetMoviesDataPath(IApplicationPaths appPaths) + { + var dataPath = Path.Combine(appPaths.DataPath, "fanart-movies"); + + if (!Directory.Exists(dataPath)) + { + Directory.CreateDirectory(dataPath); + } + + return dataPath; } /// <summary> @@ -165,15 +218,43 @@ namespace MediaBrowser.Controller.Providers.Movies item.ProviderData[Id] = data; } - var status = ProviderRefreshStatus.Success; + var movieId = item.GetProviderId(MetadataProviders.Tmdb); - var movie = item; + var movieDataPath = GetMovieDataPath(ConfigurationManager.ApplicationPaths, movieId); + var xmlPath = Path.Combine(movieDataPath, "fanart.xml"); - var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower(); - var url = string.Format(FanArtBaseUrl, ApiKey, movie.GetProviderId(MetadataProviders.Tmdb)); - var doc = new XmlDocument(); + // Only download the xml if it doesn't already exist. The prescan task will take care of getting updates + if (!File.Exists(xmlPath)) + { + await DownloadMovieXml(movieDataPath, movieId, cancellationToken).ConfigureAwait(false); + } + + if (File.Exists(xmlPath)) + { + await FetchFromXml(item, xmlPath, cancellationToken).ConfigureAwait(false); + } + + data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tmdb)); + SetLastRefreshed(item, DateTime.UtcNow); + return true; + } + + /// <summary> + /// Downloads the movie XML. + /// </summary> + /// <param name="movieDataPath">The movie data path.</param> + /// <param name="tmdbId">The TMDB id.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task.</returns> + internal async Task DownloadMovieXml(string movieDataPath, string tmdbId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + string url = string.Format(FanArtBaseUrl, ApiKey, tmdbId); - using (var xml = await HttpClient.Get(new HttpRequestOptions + var xmlPath = Path.Combine(movieDataPath, "fanart.xml"); + + using (var response = await HttpClient.Get(new HttpRequestOptions { Url = url, ResourcePool = FanArtResourcePool, @@ -181,9 +262,27 @@ namespace MediaBrowser.Controller.Providers.Movies }).ConfigureAwait(false)) { - doc.Load(xml); + using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await response.CopyToAsync(xmlFileStream).ConfigureAwait(false); + } } + } + /// <summary> + /// Fetches from XML. + /// </summary> + /// <param name="item">The item.</param> + /// <param name="xmlFilePath">The XML file path.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task.</returns> + private async Task FetchFromXml(BaseItem item, string xmlFilePath, CancellationToken cancellationToken) + { + var doc = new XmlDocument(); + doc.Load(xmlFilePath); + + var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower(); + cancellationToken.ThrowIfCancellationRequested(); var saveLocal = ConfigurationManager.Configuration.SaveLocalMeta && @@ -205,7 +304,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(movie, path, LogoFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } cancellationToken.ThrowIfCancellationRequested(); @@ -220,7 +319,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(movie, path, ArtFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } cancellationToken.ThrowIfCancellationRequested(); @@ -232,7 +331,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Disc, await _providerManager.DownloadAndSaveImage(movie, path, DiscFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Disc, await _providerManager.DownloadAndSaveImage(item, path, DiscFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } @@ -245,7 +344,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(movie, path, BannerFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } @@ -258,7 +357,7 @@ namespace MediaBrowser.Controller.Providers.Movies path = node != null ? node.Value : null; if (!string.IsNullOrEmpty(path)) { - movie.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(movie, path, ThumbFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); + item.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(item, path, ThumbFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); } } @@ -277,7 +376,7 @@ namespace MediaBrowser.Controller.Providers.Movies if (!string.IsNullOrEmpty(path)) { item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, path, ("backdrop" + (numBackdrops > 0 ? numBackdrops.ToString(UsCulture) : "") + ".jpg"), saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false)); - + numBackdrops++; if (item.BackdropImagePaths.Count >= ConfigurationManager.Configuration.MaxBackdrops) break; @@ -286,15 +385,6 @@ namespace MediaBrowser.Controller.Providers.Movies } } - - data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tmdb)); - SetLastRefreshed(movie, DateTime.UtcNow, status); - return true; - } - - public void Dispose() - { - Dispose(true); } } } diff --git a/MediaBrowser.Controller/Providers/Movies/FanArtMovieUpdatesPrescanTask.cs b/MediaBrowser.Controller/Providers/Movies/FanArtMovieUpdatesPrescanTask.cs new file mode 100644 index 000000000..727271f9c --- /dev/null +++ b/MediaBrowser.Controller/Providers/Movies/FanArtMovieUpdatesPrescanTask.cs @@ -0,0 +1,184 @@ +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers.Music; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Net; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Controller.Providers.Movies +{ + class FanArtMovieUpdatesPrescanTask : ILibraryPrescanTask + { + private const string UpdatesUrl = "http://api.fanart.tv/webservice/newmovies/{0}/{1}/"; + + /// <summary> + /// The _HTTP client + /// </summary> + private readonly IHttpClient _httpClient; + /// <summary> + /// The _logger + /// </summary> + private readonly ILogger _logger; + /// <summary> + /// The _config + /// </summary> + private readonly IServerConfigurationManager _config; + private readonly IJsonSerializer _jsonSerializer; + + private static readonly CultureInfo UsCulture = new CultureInfo("en-US"); + + public FanArtMovieUpdatesPrescanTask(IJsonSerializer jsonSerializer, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient) + { + _jsonSerializer = jsonSerializer; + _config = config; + _logger = logger; + _httpClient = httpClient; + } + + /// <summary> + /// Runs the specified progress. + /// </summary> + /// <param name="progress">The progress.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task.</returns> + public async Task Run(IProgress<double> progress, CancellationToken cancellationToken) + { + if (!_config.Configuration.EnableInternetProviders) + { + progress.Report(100); + return; + } + + var path = FanArtMovieProvider.GetMoviesDataPath(_config.CommonApplicationPaths); + + var timestampFile = Path.Combine(path, "time.txt"); + + var timestampFileInfo = new FileInfo(timestampFile); + + if (_config.Configuration.MetadataRefreshDays > 0 && timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < _config.Configuration.MetadataRefreshDays) + { + return; + } + + // Find out the last time we queried for updates + var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty; + + var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList(); + + // If this is our first time, don't do any updates and just record the timestamp + if (!string.IsNullOrEmpty(lastUpdateTime)) + { + var moviesToUpdate = await GetMovieIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false); + + progress.Report(5); + + await UpdateMovies(moviesToUpdate, path, progress, cancellationToken).ConfigureAwait(false); + } + + var newUpdateTime = Convert.ToInt64(DateTimeToUnixTimestamp(DateTime.UtcNow)).ToString(UsCulture); + + File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8); + + progress.Report(100); + } + + private async Task<IEnumerable<string>> GetMovieIdsToUpdate(IEnumerable<string> existingIds, string lastUpdateTime, CancellationToken cancellationToken) + { + // First get last time + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = string.Format(UpdatesUrl, FanartBaseProvider.ApiKey, lastUpdateTime), + CancellationToken = cancellationToken, + EnableHttpCompression = true, + ResourcePool = FanartBaseProvider.FanArtResourcePool + + }).ConfigureAwait(false)) + { + // If empty fanart will return a string of "null", rather than an empty list + using (var reader = new StreamReader(stream)) + { + var json = await reader.ReadToEndAsync().ConfigureAwait(false); + + if (string.Equals(json, "null", StringComparison.OrdinalIgnoreCase)) + { + return new List<string>(); + } + + var updates = _jsonSerializer.DeserializeFromString<List<FanArtUpdatesPrescanTask.FanArtUpdate>>(json); + + return updates.Select(i => i.id).Where(i => existingIds.Contains(i, StringComparer.OrdinalIgnoreCase)); + } + } + } + + private async Task UpdateMovies(IEnumerable<string> idList, string moviesDataPath, IProgress<double> progress, CancellationToken cancellationToken) + { + var list = idList.ToList(); + var numComplete = 0; + + foreach (var id in list) + { + try + { + await UpdateMovie(id, moviesDataPath, cancellationToken).ConfigureAwait(false); + } + catch (HttpException ex) + { + // Already logged at lower levels, but don't fail the whole operation, unless something other than a timeout + if (!ex.IsTimedOut) + { + throw; + } + } + + numComplete++; + double percent = numComplete; + percent /= list.Count; + percent *= 95; + + progress.Report(percent + 5); + } + } + + private Task UpdateMovie(string tmdbId, string movieDataPath, CancellationToken cancellationToken) + { + _logger.Info("Updating movie " + tmdbId); + + movieDataPath = Path.Combine(movieDataPath, tmdbId); + + if (!Directory.Exists(movieDataPath)) + { + Directory.CreateDirectory(movieDataPath); + } + + return FanArtMovieProvider.Current.DownloadMovieXml(movieDataPath, tmdbId, cancellationToken); + } + + /// <summary> + /// Dates the time to unix timestamp. + /// </summary> + /// <param name="dateTime">The date time.</param> + /// <returns>System.Double.</returns> + private static double DateTimeToUnixTimestamp(DateTime dateTime) + { + return (dateTime - new DateTime(1970, 1, 1).ToUniversalTime()).TotalSeconds; + } + + public class FanArtUpdate + { + public string id { get; set; } + public string name { get; set; } + public string new_images { get; set; } + public string total_images { get; set; } + } + } +} diff --git a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs index ced342a9c..dd9d8353c 100644 --- a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs @@ -3,7 +3,6 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; -using MediaBrowser.Model.Net; using MediaBrowser.Model.Serialization; using System; using System.Collections.Generic; diff --git a/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs b/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs index 893715b73..023df3177 100644 --- a/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs +++ b/MediaBrowser.Server.Implementations/HttpServer/HttpServer.cs @@ -358,7 +358,7 @@ namespace MediaBrowser.Server.Implementations.HttpServer { var address = endpoint.ToString(); - _localEndPoints.AddOrUpdate(address, address, (key, existing) => address); + _localEndPoints.GetOrAdd(address, address); } if (EnableHttpRequestLogging) |