From 6f020735404a528f325631b63dcf0813ceb94a52 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Sat, 2 Jan 2016 16:54:37 -0500 Subject: better caching of remote data --- .../Updates/InstallationManager.cs | 103 +++++++++++++++------ 1 file changed, 76 insertions(+), 27 deletions(-) (limited to 'MediaBrowser.Common.Implementations/Updates/InstallationManager.cs') diff --git a/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs b/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs index dc642a0a8..014275331 100644 --- a/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs +++ b/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs @@ -185,7 +185,7 @@ namespace MediaBrowser.Common.Implementations.Updates } } - private Tuple, DateTime> _lastPackageListResult; + private DateTime _lastPackageUpdateTime; /// /// Gets all available packages. @@ -194,40 +194,89 @@ namespace MediaBrowser.Common.Implementations.Updates /// Task{List{PackageInfo}}. public async Task> GetAvailablePackagesWithoutRegistrationInfo(CancellationToken cancellationToken) { - if (_lastPackageListResult != null) + using (var stream = await GetCachedPackages(cancellationToken).ConfigureAwait(false)) { - TimeSpan cacheLength; + var packages = _jsonSerializer.DeserializeFromStream>(stream).ToList(); - switch (_config.CommonConfiguration.SystemUpdateLevel) + if ((DateTime.UtcNow - _lastPackageUpdateTime) > GetCacheLength()) { - case PackageVersionClass.Beta: - cacheLength = TimeSpan.FromMinutes(30); - break; - case PackageVersionClass.Dev: - cacheLength = TimeSpan.FromMinutes(3); - break; - default: - cacheLength = TimeSpan.FromHours(24); - break; + UpdateCachedPackages(CancellationToken.None, false); } - if ((DateTime.UtcNow - _lastPackageListResult.Item2) < cacheLength) - { - return _lastPackageListResult.Item1; - } + return packages; } + } - using (var json = await _httpClient.Get(MbAdmin.HttpUrl + "service/MB3Packages.json", cancellationToken).ConfigureAwait(false)) + private string PackageCachePath + { + get { return Path.Combine(_appPaths.CachePath, "serverpackages.json"); } + } + + private async Task GetCachedPackages(CancellationToken cancellationToken) + { + try + { + return _fileSystem.OpenRead(PackageCachePath); + } + catch (Exception) { - cancellationToken.ThrowIfCancellationRequested(); - var packages = _jsonSerializer.DeserializeFromStream>(json).ToList(); + } - packages = FilterPackages(packages).ToList(); + await UpdateCachedPackages(cancellationToken, true).ConfigureAwait(false); + return _fileSystem.OpenRead(PackageCachePath); + } + + private readonly SemaphoreSlim _updateSemaphore = new SemaphoreSlim(1, 1); + private async Task UpdateCachedPackages(CancellationToken cancellationToken, bool throwErrors) + { + await _updateSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - _lastPackageListResult = new Tuple, DateTime>(packages, DateTime.UtcNow); + try + { + if ((DateTime.UtcNow - _lastPackageUpdateTime) < GetCacheLength()) + { + return; + } - return _lastPackageListResult.Item1; + var tempFile = await _httpClient.GetTempFile(new HttpRequestOptions + { + Url = MbAdmin.HttpUrl + "service/MB3Packages.json", + CancellationToken = cancellationToken, + Progress = new Progress() + + }).ConfigureAwait(false); + + _fileSystem.CreateDirectory(Path.GetDirectoryName(PackageCachePath)); + + _fileSystem.CopyFile(tempFile, PackageCachePath, true); + _lastPackageUpdateTime = DateTime.UtcNow; + } + catch (Exception ex) + { + _logger.ErrorException("Error updating package cache", ex); + + if (throwErrors) + { + throw; + } + } + finally + { + _updateSemaphore.Release(); + } + } + + private TimeSpan GetCacheLength() + { + switch (_config.CommonConfiguration.SystemUpdateLevel) + { + case PackageVersionClass.Beta: + return TimeSpan.FromMinutes(30); + case PackageVersionClass.Dev: + return TimeSpan.FromMinutes(3); + default: + return TimeSpan.FromHours(24); } } @@ -554,7 +603,7 @@ namespace MediaBrowser.Common.Implementations.Updates if (packageChecksum != Guid.Empty) // support for legacy uploads for now { using (var crypto = new MD5CryptoServiceProvider()) - using (var stream = new BufferedStream(_fileSystem.OpenRead(tempFile), 100000)) + using (var stream = new BufferedStream(_fileSystem.OpenRead(tempFile), 100000)) { var check = Guid.Parse(BitConverter.ToString(crypto.ComputeHash(stream)).Replace("-", String.Empty)); if (check != packageChecksum) @@ -569,12 +618,12 @@ namespace MediaBrowser.Common.Implementations.Updates // Success - move it to the real target try { - _fileSystem.CreateDirectory(Path.GetDirectoryName(target)); - _fileSystem.CopyFile(tempFile, target, true); + _fileSystem.CreateDirectory(Path.GetDirectoryName(target)); + _fileSystem.CopyFile(tempFile, target, true); //If it is an archive - write out a version file so we know what it is if (isArchive) { - File.WriteAllText(target + ".ver", package.versionStr); + File.WriteAllText(target + ".ver", package.versionStr); } } catch (IOException e) -- cgit v1.2.3 From 68ae496d609c135609884e3d58d36656559a035e Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Sat, 2 Jan 2016 20:23:09 -0500 Subject: get stable and beta server packages from github --- MediaBrowser.Api/PackageService.cs | 2 +- .../HttpClientManager/HttpClientManager.cs | 2 +- .../MediaBrowser.Common.Implementations.csproj | 2 + .../Updates/GithubUpdater.cs | 195 +++++++++++++++++++++ .../Updates/InstallationManager.cs | 12 +- .../Updates/IInstallationManager.cs | 3 +- .../ScheduledTasks/PluginUpdateTask.cs | 2 +- .../ApplicationHost.cs | 13 +- 8 files changed, 220 insertions(+), 11 deletions(-) create mode 100644 MediaBrowser.Common.Implementations/Updates/GithubUpdater.cs (limited to 'MediaBrowser.Common.Implementations/Updates/InstallationManager.cs') diff --git a/MediaBrowser.Api/PackageService.cs b/MediaBrowser.Api/PackageService.cs index 4f9efad50..6d8378aae 100644 --- a/MediaBrowser.Api/PackageService.cs +++ b/MediaBrowser.Api/PackageService.cs @@ -233,7 +233,7 @@ namespace MediaBrowser.Api throw new ResourceNotFoundException(string.Format("Package not found: {0}", request.Name)); } - Task.Run(() => _installationManager.InstallPackage(package, new Progress(), CancellationToken.None)); + Task.Run(() => _installationManager.InstallPackage(package, true, new Progress(), CancellationToken.None)); } /// diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs index 054c5aaaf..c30cdf1a7 100644 --- a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs @@ -150,7 +150,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager request.Method = method; request.Timeout = options.TimeoutMs; - + if (httpWebRequest != null) { if (!string.IsNullOrEmpty(options.Host)) diff --git a/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj b/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj index 59e20ec6e..e87a6ad16 100644 --- a/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj +++ b/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj @@ -68,6 +68,7 @@ ..\packages\SimpleInjector.3.1.2\lib\net45\SimpleInjector.dll + @@ -107,6 +108,7 @@ + diff --git a/MediaBrowser.Common.Implementations/Updates/GithubUpdater.cs b/MediaBrowser.Common.Implementations/Updates/GithubUpdater.cs new file mode 100644 index 000000000..aff3dcf8d --- /dev/null +++ b/MediaBrowser.Common.Implementations/Updates/GithubUpdater.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.Net; +using MediaBrowser.Model.Serialization; +using MediaBrowser.Model.Updates; + +namespace MediaBrowser.Common.Implementations.Updates +{ + public class GithubUpdater + { + private readonly IHttpClient _httpClient; + private readonly IJsonSerializer _jsonSerializer; + private TimeSpan _cacheLength; + + public GithubUpdater(IHttpClient httpClient, IJsonSerializer jsonSerializer, TimeSpan cacheLength) + { + _httpClient = httpClient; + _jsonSerializer = jsonSerializer; + _cacheLength = cacheLength; + } + + public async Task CheckForUpdateResult(string organzation, string repository, Version minVersion, bool includePrerelease, string assetFilename, string packageName, string targetFilename, CancellationToken cancellationToken) + { + var url = string.Format("https://api.github.com/repos/{0}/{1}/releases", organzation, repository); + + var options = new HttpRequestOptions + { + Url = url, + EnableKeepAlive = false, + CancellationToken = cancellationToken, + UserAgent = "Emby/3.0" + + }; + + if (_cacheLength.Ticks > 0) + { + options.CacheMode = CacheMode.Unconditional; + options.CacheLength = _cacheLength; + } + + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = url, + EnableKeepAlive = false, + CancellationToken = cancellationToken, + UserAgent = "Emby/3.0" + + }).ConfigureAwait(false)) + { + var obj = _jsonSerializer.DeserializeFromStream(stream); + + var availableUpdate = CheckForUpdateResult(obj, minVersion, includePrerelease, assetFilename, packageName, targetFilename); + + return availableUpdate ?? new CheckForUpdateResult + { + IsUpdateAvailable = false + }; + } + } + + private CheckForUpdateResult CheckForUpdateResult(RootObject[] obj, Version minVersion, bool includePrerelease, string assetFilename, string packageName, string targetFilename) + { + if (!includePrerelease) + { + obj = obj.Where(i => !i.prerelease).ToArray(); + } + + // TODO: + // Filter using name and check for suffixes such as -beta, -dev? + + return obj.Select(i => CheckForUpdateResult(i, minVersion, assetFilename, packageName, targetFilename)).FirstOrDefault(i => i != null); + } + + private CheckForUpdateResult CheckForUpdateResult(RootObject obj, Version minVersion, string assetFilename, string packageName, string targetFilename) + { + Version version; + if (!Version.TryParse(obj.tag_name, out version)) + { + return null; + } + + if (version < minVersion) + { + return null; + } + + var asset = (obj.assets ?? new List()).FirstOrDefault(i => string.Equals(assetFilename, Path.GetFileName(i.browser_download_url), StringComparison.OrdinalIgnoreCase)); + + if (asset == null) + { + return null; + } + + return new CheckForUpdateResult + { + AvailableVersion = version.ToString(), + IsUpdateAvailable = version > minVersion, + Package = new PackageVersionInfo + { + classification = obj.prerelease ? PackageVersionClass.Beta : PackageVersionClass.Release, + name = packageName, + sourceUrl = asset.browser_download_url, + targetFilename = targetFilename, + versionStr = version.ToString(), + requiredVersionStr = "1.0.0" + } + }; + } + + public class Uploader + { + public string login { get; set; } + public int id { get; set; } + public string avatar_url { get; set; } + public string gravatar_id { get; set; } + public string url { get; set; } + public string html_url { get; set; } + public string followers_url { get; set; } + public string following_url { get; set; } + public string gists_url { get; set; } + public string starred_url { get; set; } + public string subscriptions_url { get; set; } + public string organizations_url { get; set; } + public string repos_url { get; set; } + public string events_url { get; set; } + public string received_events_url { get; set; } + public string type { get; set; } + public bool site_admin { get; set; } + } + + public class Asset + { + public string url { get; set; } + public int id { get; set; } + public string name { get; set; } + public object label { get; set; } + public Uploader uploader { get; set; } + public string content_type { get; set; } + public string state { get; set; } + public int size { get; set; } + public int download_count { get; set; } + public string created_at { get; set; } + public string updated_at { get; set; } + public string browser_download_url { get; set; } + } + + public class Author + { + public string login { get; set; } + public int id { get; set; } + public string avatar_url { get; set; } + public string gravatar_id { get; set; } + public string url { get; set; } + public string html_url { get; set; } + public string followers_url { get; set; } + public string following_url { get; set; } + public string gists_url { get; set; } + public string starred_url { get; set; } + public string subscriptions_url { get; set; } + public string organizations_url { get; set; } + public string repos_url { get; set; } + public string events_url { get; set; } + public string received_events_url { get; set; } + public string type { get; set; } + public bool site_admin { get; set; } + } + + public class RootObject + { + public string url { get; set; } + public string assets_url { get; set; } + public string upload_url { get; set; } + public string html_url { get; set; } + public int id { get; set; } + public string tag_name { get; set; } + public string target_commitish { get; set; } + public string name { get; set; } + public bool draft { get; set; } + public Author author { get; set; } + public bool prerelease { get; set; } + public string created_at { get; set; } + public string published_at { get; set; } + public List assets { get; set; } + public string tarball_url { get; set; } + public string zipball_url { get; set; } + public string body { get; set; } + } + } +} diff --git a/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs b/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs index 014275331..d155f11c7 100644 --- a/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs +++ b/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs @@ -438,11 +438,12 @@ namespace MediaBrowser.Common.Implementations.Updates /// Installs the package. /// /// The package. + /// if set to true [is plugin]. /// The progress. /// The cancellation token. /// Task. /// package - public async Task InstallPackage(PackageVersionInfo package, IProgress progress, CancellationToken cancellationToken) + public async Task InstallPackage(PackageVersionInfo package, bool isPlugin, IProgress progress, CancellationToken cancellationToken) { if (package == null) { @@ -495,7 +496,7 @@ namespace MediaBrowser.Common.Implementations.Updates try { - await InstallPackageInternal(package, innerProgress, linkedToken).ConfigureAwait(false); + await InstallPackageInternal(package, isPlugin, innerProgress, linkedToken).ConfigureAwait(false); lock (CurrentInstallations) { @@ -551,18 +552,17 @@ namespace MediaBrowser.Common.Implementations.Updates /// Installs the package internal. /// /// The package. + /// if set to true [is plugin]. /// The progress. /// The cancellation token. /// Task. - private async Task InstallPackageInternal(PackageVersionInfo package, IProgress progress, CancellationToken cancellationToken) + private async Task InstallPackageInternal(PackageVersionInfo package, bool isPlugin, IProgress progress, CancellationToken cancellationToken) { // Do the install await PerformPackageInstallation(progress, package, cancellationToken).ConfigureAwait(false); - var extension = Path.GetExtension(package.targetFilename) ?? ""; - // Do plugin-specific processing - if (!string.Equals(extension, ".zip", StringComparison.OrdinalIgnoreCase) && !string.Equals(extension, ".rar", StringComparison.OrdinalIgnoreCase) && !string.Equals(extension, ".7z", StringComparison.OrdinalIgnoreCase)) + if (isPlugin) { // Set last update time if we were installed before var plugin = _applicationHost.Plugins.FirstOrDefault(p => string.Equals(p.Id.ToString(), package.guid, StringComparison.OrdinalIgnoreCase)) diff --git a/MediaBrowser.Common/Updates/IInstallationManager.cs b/MediaBrowser.Common/Updates/IInstallationManager.cs index 7d721da6f..68853f05e 100644 --- a/MediaBrowser.Common/Updates/IInstallationManager.cs +++ b/MediaBrowser.Common/Updates/IInstallationManager.cs @@ -105,11 +105,12 @@ namespace MediaBrowser.Common.Updates /// Installs the package. /// /// The package. + /// if set to true [is plugin]. /// The progress. /// The cancellation token. /// Task. /// package - Task InstallPackage(PackageVersionInfo package, IProgress progress, CancellationToken cancellationToken); + Task InstallPackage(PackageVersionInfo package, bool isPlugin, IProgress progress, CancellationToken cancellationToken); /// /// Uninstalls a plugin diff --git a/MediaBrowser.Server.Implementations/ScheduledTasks/PluginUpdateTask.cs b/MediaBrowser.Server.Implementations/ScheduledTasks/PluginUpdateTask.cs index 96f469f69..457f5a33d 100644 --- a/MediaBrowser.Server.Implementations/ScheduledTasks/PluginUpdateTask.cs +++ b/MediaBrowser.Server.Implementations/ScheduledTasks/PluginUpdateTask.cs @@ -72,7 +72,7 @@ namespace MediaBrowser.Server.Implementations.ScheduledTasks try { - await _installationManager.InstallPackage(i, new Progress(), cancellationToken).ConfigureAwait(false); + await _installationManager.InstallPackage(i, true, new Progress(), cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/MediaBrowser.Server.Startup.Common/ApplicationHost.cs b/MediaBrowser.Server.Startup.Common/ApplicationHost.cs index 7fd0185b7..2f03e13f8 100644 --- a/MediaBrowser.Server.Startup.Common/ApplicationHost.cs +++ b/MediaBrowser.Server.Startup.Common/ApplicationHost.cs @@ -101,6 +101,7 @@ using System.Reflection; using System.Threading; using System.Threading.Tasks; using CommonIO; +using MediaBrowser.Common.Implementations.Updates; namespace MediaBrowser.Server.Startup.Common { @@ -1300,6 +1301,16 @@ namespace MediaBrowser.Server.Startup.Common /// Task{CheckForUpdateResult}. public override async Task CheckForApplicationUpdate(CancellationToken cancellationToken, IProgress progress) { + if (ConfigurationManager.CommonConfiguration.SystemUpdateLevel != PackageVersionClass.Dev) + { + var includePreRelease = ConfigurationManager.CommonConfiguration.SystemUpdateLevel != PackageVersionClass.Release; + + var cacheLength = TimeSpan.FromHours(1); + + return await new GithubUpdater(HttpClient, JsonSerializer, cacheLength) + .CheckForUpdateResult("MediaBrowser", "Emby", ApplicationVersion, includePreRelease, "emby.windows.zip", "MBServer", "Mbserver.zip", cancellationToken).ConfigureAwait(false); + } + var availablePackages = await InstallationManager.GetAvailablePackagesWithoutRegistrationInfo(cancellationToken).ConfigureAwait(false); var version = InstallationManager.GetLatestCompatibleVersion(availablePackages, _remotePackageName, null, ApplicationVersion, ConfigurationManager.CommonConfiguration.SystemUpdateLevel); @@ -1331,7 +1342,7 @@ namespace MediaBrowser.Server.Startup.Common /// Task. public override async Task UpdateApplication(PackageVersionInfo package, CancellationToken cancellationToken, IProgress progress) { - await InstallationManager.InstallPackage(package, progress, cancellationToken).ConfigureAwait(false); + await InstallationManager.InstallPackage(package, false, progress, cancellationToken).ConfigureAwait(false); HasUpdateAvailable = false; -- cgit v1.2.3 From dbc9beab22a2ec4a9e2680505256cb478550a31f Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Tue, 12 Jan 2016 15:12:50 -0500 Subject: add error handling to package retrieval --- .../Updates/InstallationManager.cs | 39 +++++++++++----------- 1 file changed, 19 insertions(+), 20 deletions(-) (limited to 'MediaBrowser.Common.Implementations/Updates/InstallationManager.cs') diff --git a/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs b/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs index d155f11c7..8e0df9005 100644 --- a/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs +++ b/MediaBrowser.Common.Implementations/Updates/InstallationManager.cs @@ -194,37 +194,36 @@ namespace MediaBrowser.Common.Implementations.Updates /// Task{List{PackageInfo}}. public async Task> GetAvailablePackagesWithoutRegistrationInfo(CancellationToken cancellationToken) { - using (var stream = await GetCachedPackages(cancellationToken).ConfigureAwait(false)) + try { - var packages = _jsonSerializer.DeserializeFromStream>(stream).ToList(); - - if ((DateTime.UtcNow - _lastPackageUpdateTime) > GetCacheLength()) + using (var stream = _fileSystem.OpenRead(PackageCachePath)) { - UpdateCachedPackages(CancellationToken.None, false); - } - - return packages; - } - } + var packages = _jsonSerializer.DeserializeFromStream>(stream).ToList(); - private string PackageCachePath - { - get { return Path.Combine(_appPaths.CachePath, "serverpackages.json"); } - } + if ((DateTime.UtcNow - _lastPackageUpdateTime) > GetCacheLength()) + { + UpdateCachedPackages(CancellationToken.None, false); + } - private async Task GetCachedPackages(CancellationToken cancellationToken) - { - try - { - return _fileSystem.OpenRead(PackageCachePath); + return packages; + } } catch (Exception) { } + _lastPackageUpdateTime = DateTime.MinValue; await UpdateCachedPackages(cancellationToken, true).ConfigureAwait(false); - return _fileSystem.OpenRead(PackageCachePath); + using (var stream = _fileSystem.OpenRead(PackageCachePath)) + { + return _jsonSerializer.DeserializeFromStream>(stream).ToList(); + } + } + + private string PackageCachePath + { + get { return Path.Combine(_appPaths.CachePath, "serverpackages.json"); } } private readonly SemaphoreSlim _updateSemaphore = new SemaphoreSlim(1, 1); -- cgit v1.2.3