diff options
7 files changed, 255 insertions, 83 deletions
diff --git a/MediaBrowser.Common.Implementations/BaseApplicationHost.cs b/MediaBrowser.Common.Implementations/BaseApplicationHost.cs index 4578da827..14d7f87c3 100644 --- a/MediaBrowser.Common.Implementations/BaseApplicationHost.cs +++ b/MediaBrowser.Common.Implementations/BaseApplicationHost.cs @@ -273,7 +273,7 @@ namespace MediaBrowser.Common.Implementations RegisterSingleInstance(TaskManager); - HttpClient = new HttpClientManager.HttpClientManager(ApplicationPaths, Logger); + HttpClient = new HttpClientManager.HttpClientManager(ApplicationPaths, Logger, JsonSerializer); RegisterSingleInstance(HttpClient); NetworkManager = new NetworkManager(); diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs index 1b1d37ff2..a8e5b3e79 100644 --- a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs @@ -1,14 +1,17 @@ using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Net; +using MediaBrowser.Model.Serialization; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Net; using System.Net.Cache; using System.Net.Http; using System.Text; @@ -31,13 +34,22 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager /// The _app paths /// </summary> private readonly IApplicationPaths _appPaths; - + + private readonly IJsonSerializer _jsonSerializer; + //private readonly FileSystemRepository _cacheRepository; + /// <summary> /// Initializes a new instance of the <see cref="HttpClientManager" /> class. /// </summary> /// <param name="appPaths">The kernel.</param> /// <param name="logger">The logger.</param> - public HttpClientManager(IApplicationPaths appPaths, ILogger logger) + /// <param name="jsonSerializer">The json serializer.</param> + /// <exception cref="System.ArgumentNullException"> + /// appPaths + /// or + /// logger + /// </exception> + public HttpClientManager(IApplicationPaths appPaths, ILogger logger, IJsonSerializer jsonSerializer) { if (appPaths == null) { @@ -47,9 +59,12 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { throw new ArgumentNullException("logger"); } - + _logger = logger; + _jsonSerializer = jsonSerializer; _appPaths = appPaths; + + //_cacheRepository = new FileSystemRepository(Path.Combine(_appPaths.CachePath, "http")); } /// <summary> @@ -77,8 +92,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { var handler = new WebRequestHandler { - //AutomaticDecompression = DecompressionMethods.Deflate, - CachePolicy = new RequestCachePolicy(RequestCacheLevel.Revalidate) + CachePolicy = new RequestCachePolicy(RequestCacheLevel.BypassCache) }; client = new HttpClient(handler); @@ -102,10 +116,42 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { ValidateParams(url, cancellationToken); + //var urlHash = url.GetMD5().ToString(); + //var infoPath = _cacheRepository.GetResourcePath(urlHash + ".js"); + //var responsePath = _cacheRepository.GetResourcePath(urlHash + ".dat"); + + //HttpResponseInfo cachedInfo = null; + + //try + //{ + // cachedInfo = _jsonSerializer.DeserializeFromFile<HttpResponseInfo>(infoPath); + //} + //catch (FileNotFoundException) + //{ + + //} + + //if (cachedInfo != null && !cachedInfo.MustRevalidate && cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow) + //{ + // return GetCachedResponse(responsePath); + //} + cancellationToken.ThrowIfCancellationRequested(); var message = new HttpRequestMessage(HttpMethod.Get, url); + //if (cachedInfo != null) + //{ + // if (!string.IsNullOrEmpty(cachedInfo.Etag)) + // { + // message.Headers.Add("If-None-Match", cachedInfo.Etag); + // } + // else if (cachedInfo.LastModified.HasValue) + // { + // message.Headers.IfModifiedSince = new DateTimeOffset(cachedInfo.LastModified.Value); + // } + //} + if (resourcePool != null) { await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -123,6 +169,20 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager cancellationToken.ThrowIfCancellationRequested(); + //cachedInfo = UpdateInfoCache(cachedInfo, url, infoPath, response); + + //if (response.StatusCode == HttpStatusCode.NotModified) + //{ + // return GetCachedResponse(responsePath); + //} + + //if (!string.IsNullOrEmpty(cachedInfo.Etag) || cachedInfo.LastModified.HasValue || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow)) + //{ + // await UpdateResponseCache(response, responsePath).ConfigureAwait(false); + + // return GetCachedResponse(responsePath); + //} + return await response.Content.ReadAsStreamAsync().ConfigureAwait(false); } } @@ -150,7 +210,108 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager } } } - + + /// <summary> + /// Gets the cached response. + /// </summary> + /// <param name="responsePath">The response path.</param> + /// <returns>Stream.</returns> + private Stream GetCachedResponse(string responsePath) + { + return File.OpenRead(responsePath); + } + + /// <summary> + /// Updates the cache. + /// </summary> + /// <param name="cachedInfo">The cached info.</param> + /// <param name="url">The URL.</param> + /// <param name="path">The path.</param> + /// <param name="response">The response.</param> + private HttpResponseInfo UpdateInfoCache(HttpResponseInfo cachedInfo, string url, string path, HttpResponseMessage response) + { + var fileExists = true; + + if (cachedInfo == null) + { + cachedInfo = new HttpResponseInfo(); + fileExists = false; + } + + cachedInfo.Url = url; + + var etag = response.Headers.ETag; + if (etag != null) + { + cachedInfo.Etag = etag.Tag; + } + + var modified = response.Content.Headers.LastModified; + + if (modified.HasValue) + { + cachedInfo.LastModified = modified.Value.UtcDateTime; + } + else if (response.Headers.Age.HasValue) + { + cachedInfo.LastModified = DateTime.UtcNow.Subtract(response.Headers.Age.Value); + } + + var expires = response.Content.Headers.Expires; + + if (expires.HasValue) + { + cachedInfo.Expires = expires.Value.UtcDateTime; + } + else + { + var cacheControl = response.Headers.CacheControl; + + if (cacheControl != null) + { + if (cacheControl.MaxAge.HasValue) + { + var baseline = cachedInfo.LastModified ?? DateTime.UtcNow; + cachedInfo.Expires = baseline.Add(cacheControl.MaxAge.Value); + } + + cachedInfo.MustRevalidate = cacheControl.MustRevalidate; + } + } + + if (string.IsNullOrEmpty(cachedInfo.Etag) && !cachedInfo.Expires.HasValue && !cachedInfo.LastModified.HasValue) + { + // Nothing to cache + if (fileExists) + { + File.Delete(path); + } + } + else + { + _jsonSerializer.SerializeToFile(cachedInfo, path); + } + + return cachedInfo; + } + + /// <summary> + /// Updates the response cache. + /// </summary> + /// <param name="response">The response.</param> + /// <param name="path">The path.</param> + /// <returns>Task.</returns> + private async Task UpdateResponseCache(HttpResponseMessage response, string path) + { + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + { + using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await stream.CopyToAsync(fs).ConfigureAwait(false); + } + } + } + /// <summary> /// Performs a POST request /// </summary> @@ -259,10 +420,9 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager options.CancellationToken.ThrowIfCancellationRequested(); - IEnumerable<string> lengthValues; + var contentLength = GetContentLength(response); - if (!response.Headers.TryGetValues("content-length", out lengthValues) && - !response.Content.Headers.TryGetValues("content-length", out lengthValues)) + if (!contentLength.HasValue) { // We're not able to track progress using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) @@ -275,9 +435,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager } else { - var length = long.Parse(string.Join(string.Empty, lengthValues.ToArray()), UsCulture); - - using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), options.Progress.Report, length)) + using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), options.Progress.Report, contentLength.Value)) { using (var fs = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { @@ -306,6 +464,23 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager return tempFile; } + /// <summary> + /// Gets the length of the content. + /// </summary> + /// <param name="response">The response.</param> + /// <returns>System.Nullable{System.Int64}.</returns> + private long? GetContentLength(HttpResponseMessage response) + { + IEnumerable<string> lengthValues; + + if (!response.Headers.TryGetValues("content-length", out lengthValues) && !response.Content.Headers.TryGetValues("content-length", out lengthValues)) + { + return null; + } + + return long.Parse(string.Join(string.Empty, lengthValues.ToArray()), UsCulture); + } + protected static readonly CultureInfo UsCulture = new CultureInfo("en-US"); /// <summary> @@ -348,41 +523,6 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager throw ex; } - - /// <summary> - /// Downloads the contents of a given url into a MemoryStream - /// </summary> - /// <param name="url">The URL.</param> - /// <param name="resourcePool">The resource pool.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>Task{MemoryStream}.</returns> - /// <exception cref="MediaBrowser.Model.Net.HttpException"></exception> - public async Task<MemoryStream> GetMemoryStream(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken) - { - ValidateParams(url, cancellationToken); - - cancellationToken.ThrowIfCancellationRequested(); - - _logger.Info("HttpClientManager.GetMemoryStream url: {0}", url); - - var ms = new MemoryStream(); - - try - { - using (var stream = await Get(url, resourcePool, cancellationToken).ConfigureAwait(false)) - { - await stream.CopyToAsync(ms, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); - } - - return ms; - } - catch - { - ms.Dispose(); - - throw; - } - } /// <summary> /// Validates the params. @@ -499,16 +639,5 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { return Post(url, postData, null, cancellationToken); } - - /// <summary> - /// Gets the memory stream. - /// </summary> - /// <param name="url">The URL.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>Task{MemoryStream}.</returns> - public Task<MemoryStream> GetMemoryStream(string url, CancellationToken cancellationToken) - { - return GetMemoryStream(url, null, cancellationToken); - } } } diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs new file mode 100644 index 000000000..240e99d79 --- /dev/null +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs @@ -0,0 +1,40 @@ +using System; + +namespace MediaBrowser.Common.Implementations.HttpClientManager +{ + /// <summary> + /// Class HttpResponseOutput + /// </summary> + public class HttpResponseInfo + { + /// <summary> + /// Gets or sets the URL. + /// </summary> + /// <value>The URL.</value> + public string Url { get; set; } + + /// <summary> + /// Gets or sets the etag. + /// </summary> + /// <value>The etag.</value> + public string Etag { get; set; } + + /// <summary> + /// Gets or sets the last modified. + /// </summary> + /// <value>The last modified.</value> + public DateTime? LastModified { get; set; } + + /// <summary> + /// Gets or sets the expires. + /// </summary> + /// <value>The expires.</value> + public DateTime? Expires { get; set; } + + /// <summary> + /// Gets or sets a value indicating whether [must revalidate]. + /// </summary> + /// <value><c>true</c> if [must revalidate]; otherwise, <c>false</c>.</value> + public bool MustRevalidate { get; set; } + } +} diff --git a/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj b/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj index 2fe309cee..1f5c93988 100644 --- a/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj +++ b/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj @@ -65,6 +65,7 @@ <Compile Include="BaseApplicationPaths.cs" /> <Compile Include="Configuration\BaseConfigurationManager.cs" /> <Compile Include="HttpClientManager\HttpClientManager.cs" /> + <Compile Include="HttpClientManager\HttpResponseInfo.cs" /> <Compile Include="Logging\LogHelper.cs" /> <Compile Include="Logging\NLogger.cs" /> <Compile Include="Logging\NlogManager.cs" /> diff --git a/MediaBrowser.Common/Net/IHttpClient.cs b/MediaBrowser.Common/Net/IHttpClient.cs index f443341a0..87f9b5d71 100644 --- a/MediaBrowser.Common/Net/IHttpClient.cs +++ b/MediaBrowser.Common/Net/IHttpClient.cs @@ -58,23 +58,5 @@ namespace MediaBrowser.Common.Net /// <exception cref="System.ArgumentNullException">progress</exception> /// <exception cref="MediaBrowser.Model.Net.HttpException"></exception> Task<string> GetTempFile(HttpRequestOptions options); - - /// <summary> - /// Downloads the contents of a given url into a MemoryStream - /// </summary> - /// <param name="url">The URL.</param> - /// <param name="resourcePool">The resource pool.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>Task{MemoryStream}.</returns> - /// <exception cref="MediaBrowser.Model.Net.HttpException"></exception> - Task<MemoryStream> GetMemoryStream(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken); - - /// <summary> - /// Gets the memory stream. - /// </summary> - /// <param name="url">The URL.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <returns>Task{MemoryStream}.</returns> - Task<MemoryStream> GetMemoryStream(string url, CancellationToken cancellationToken); } }
\ No newline at end of file diff --git a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs index 699c5473b..583e0bb97 100644 --- a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs @@ -335,7 +335,7 @@ namespace MediaBrowser.Controller.Providers.Movies var localPath = Path.Combine(item.MetaLocation, targetName); if (!item.ResolveArgs.ContainsMetaFileByName(targetName)) { - using (var sourceStream = await HttpClient.GetMemoryStream(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false)) + using (var sourceStream = await HttpClient.Get(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false)) { await ProviderManager.SaveToLibraryFilesystem(item, localPath, sourceStream, cancellationToken).ConfigureAwait(false); diff --git a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs index 97548140c..e4f57e8d9 100644 --- a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs +++ b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs @@ -202,7 +202,7 @@ namespace MediaBrowser.Server.Implementations.Providers { item.ProviderData[_supportedProvidersKey] = supportedProvidersInfo; } - + return result || providersChanged; } @@ -282,7 +282,7 @@ namespace MediaBrowser.Server.Implementations.Providers } catch (ObjectDisposedException) { - + } } @@ -358,7 +358,7 @@ namespace MediaBrowser.Server.Implementations.Providers Path.Combine(item.MetaLocation, targetName) : _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName); - var img = await _httpClient.GetMemoryStream(source, resourcePool, cancellationToken).ConfigureAwait(false); + var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false); if (ConfigurationManager.Configuration.SaveLocalMeta) // queue to media directories { @@ -422,12 +422,32 @@ namespace MediaBrowser.Server.Implementations.Providers throw new ArgumentNullException(); } - cancellationToken.ThrowIfCancellationRequested(); + if (cancellationToken.IsCancellationRequested) + { + dataToSave.Dispose(); + cancellationToken.ThrowIfCancellationRequested(); + } //Tell the watchers to ignore _directoryWatchers.TemporarilyIgnore(path); - dataToSave.Position = 0; + if (dataToSave.CanSeek) + { + dataToSave.Position = 0; + } + + if (!(dataToSave is MemoryStream || dataToSave is FileStream)) + { + var ms = new MemoryStream(); + + using (var input = dataToSave) + { + await input.CopyToAsync(ms).ConfigureAwait(false); + } + + ms.Position = 0; + dataToSave = ms; + } try { |