From 65a9d618ccfb5e015eb2dccf437e8795ac5350d5 Mon Sep 17 00:00:00 2001 From: dkanada Date: Sat, 22 Feb 2020 15:04:52 +0900 Subject: add config options for musicbrainz --- .../ConfigurationOptions.cs | 1 - .../Library/LibraryManager.cs | 3 +- .../Entities/Audio/MusicArtist.cs | 1 + MediaBrowser.Controller/Entities/Folder.cs | 8 +- .../MediaBrowser.Providers.csproj | 5 + .../Music/MusicBrainzAlbumProvider.cs | 800 --------------------- .../Music/MusicBrainzArtistProvider.cs | 281 -------- MediaBrowser.Providers/Music/MusicExternalIds.cs | 95 --- .../Plugins/MusicBrainz/AlbumProvider.cs | 799 ++++++++++++++++++++ .../Plugins/MusicBrainz/ArtistProvider.cs | 298 ++++++++ .../Configuration/PluginConfiguration.cs | 16 + .../Plugins/MusicBrainz/Configuration/config.html | 69 ++ .../Plugins/MusicBrainz/MusicBrainzExternalIds.cs | 102 +++ .../Plugins/MusicBrainz/Plugin.cs | 35 + 14 files changed, 1330 insertions(+), 1183 deletions(-) delete mode 100644 MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs delete mode 100644 MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs create mode 100644 MediaBrowser.Providers/Plugins/MusicBrainz/AlbumProvider.cs create mode 100644 MediaBrowser.Providers/Plugins/MusicBrainz/ArtistProvider.cs create mode 100644 MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/PluginConfiguration.cs create mode 100644 MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/config.html create mode 100644 MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzExternalIds.cs create mode 100644 MediaBrowser.Providers/Plugins/MusicBrainz/Plugin.cs diff --git a/Emby.Server.Implementations/ConfigurationOptions.cs b/Emby.Server.Implementations/ConfigurationOptions.cs index 2ea7ff6e9..d0f3d6723 100644 --- a/Emby.Server.Implementations/ConfigurationOptions.cs +++ b/Emby.Server.Implementations/ConfigurationOptions.cs @@ -8,7 +8,6 @@ namespace Emby.Server.Implementations public static Dictionary Configuration => new Dictionary { { "HttpListenerHost:DefaultRedirectPath", "web/index.html" }, - { "MusicBrainz:BaseUrl", "https://www.musicbrainz.org" }, { FfmpegProbeSizeKey, "1G" }, { FfmpegAnalyzeDurationKey, "200M" } }; diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 55566ed51..8a66ffdaa 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -944,7 +944,6 @@ namespace Emby.Server.Implementations.Library IncludeItemTypes = new[] { typeof(T).Name }, Name = name, DtoOptions = options - }).Cast() .OrderBy(i => i.IsAccessedByName ? 1 : 0) .Cast() @@ -1080,7 +1079,7 @@ namespace Emby.Server.Implementations.Library var innerProgress = new ActionableProgress(); - innerProgress.RegisterAction(pct => progress.Report(pct * pct * 0.96)); + innerProgress.RegisterAction(pct => progress.Report(pct * 0.96)); // Validate the entire media library await RootFolder.ValidateChildren(innerProgress, cancellationToken, new MetadataRefreshOptions(new DirectoryService(_fileSystem)), recursive: true).ConfigureAwait(false); diff --git a/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs b/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs index efe0d3cf7..5e3056ccb 100644 --- a/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs +++ b/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs @@ -198,6 +198,7 @@ namespace MediaBrowser.Controller.Entities.Audio return true; } } + return base.RequiresRefresh(); } diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 07fbe6035..a892be7a9 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -322,10 +322,10 @@ namespace MediaBrowser.Controller.Entities ProviderManager.OnRefreshProgress(this, 5); } - //build a dictionary of the current children we have now by Id so we can compare quickly and easily + // Build a dictionary of the current children we have now by Id so we can compare quickly and easily var currentChildren = GetActualChildrenDictionary(); - //create a list for our validated children + // Create a list for our validated children var newItems = new List(); cancellationToken.ThrowIfCancellationRequested(); @@ -391,7 +391,7 @@ namespace MediaBrowser.Controller.Entities var folder = this; innerProgress.RegisterAction(p => { - double newPct = .80 * p + 10; + double newPct = 0.80 * p + 10; progress.Report(newPct); ProviderManager.OnRefreshProgress(folder, newPct); }); @@ -421,7 +421,7 @@ namespace MediaBrowser.Controller.Entities var folder = this; innerProgress.RegisterAction(p => { - double newPct = .10 * p + 90; + double newPct = 0.10 * p + 90; progress.Report(newPct); if (recursive) { diff --git a/MediaBrowser.Providers/MediaBrowser.Providers.csproj b/MediaBrowser.Providers/MediaBrowser.Providers.csproj index 5593c5036..48c16b643 100644 --- a/MediaBrowser.Providers/MediaBrowser.Providers.csproj +++ b/MediaBrowser.Providers/MediaBrowser.Providers.csproj @@ -24,4 +24,9 @@ true + + + + + diff --git a/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs b/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs deleted file mode 100644 index 8e71b625e..000000000 --- a/MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs +++ /dev/null @@ -1,800 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using System.Xml; -using MediaBrowser.Common; -using MediaBrowser.Common.Net; -using MediaBrowser.Controller.Entities.Audio; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.Entities; -using MediaBrowser.Model.Providers; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; - -namespace MediaBrowser.Providers.Music -{ - public class MusicBrainzAlbumProvider : IRemoteMetadataProvider, IHasOrder - { - /// - /// The Jellyfin user-agent is unrestricted but source IP must not exceed - /// one request per second, therefore we rate limit to avoid throttling. - /// Be prudent, use a value slightly above the minimun required. - /// https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting - /// - private const long MusicBrainzQueryIntervalMs = 1050u; - - /// - /// For each single MB lookup/search, this is the maximum number of - /// attempts that shall be made whilst receiving a 503 Server - /// Unavailable (indicating throttled) response. - /// - private const uint MusicBrainzQueryAttempts = 5u; - - internal static MusicBrainzAlbumProvider Current; - - private readonly IHttpClient _httpClient; - private readonly IApplicationHost _appHost; - private readonly ILogger _logger; - - private readonly string _musicBrainzBaseUrl; - - private Stopwatch _stopWatchMusicBrainz = new Stopwatch(); - - public MusicBrainzAlbumProvider( - IHttpClient httpClient, - IApplicationHost appHost, - ILogger logger, - IConfiguration configuration) - { - _httpClient = httpClient; - _appHost = appHost; - _logger = logger; - - _musicBrainzBaseUrl = configuration["MusicBrainz:BaseUrl"]; - - // Use a stopwatch to ensure we don't exceed the MusicBrainz rate limit - _stopWatchMusicBrainz.Start(); - - Current = this; - } - - /// - public string Name => "MusicBrainz"; - - /// - public int Order => 0; - - /// - public async Task> GetSearchResults(AlbumInfo searchInfo, CancellationToken cancellationToken) - { - var releaseId = searchInfo.GetReleaseId(); - var releaseGroupId = searchInfo.GetReleaseGroupId(); - - string url; - - if (!string.IsNullOrEmpty(releaseId)) - { - url = "/ws/2/release/?query=reid:" + releaseId.ToString(CultureInfo.InvariantCulture); - } - else if (!string.IsNullOrEmpty(releaseGroupId)) - { - url = "/ws/2/release?release-group=" + releaseGroupId.ToString(CultureInfo.InvariantCulture); - } - else - { - var artistMusicBrainzId = searchInfo.GetMusicBrainzArtistId(); - - if (!string.IsNullOrWhiteSpace(artistMusicBrainzId)) - { - url = string.Format( - CultureInfo.InvariantCulture, - "/ws/2/release/?query=\"{0}\" AND arid:{1}", - WebUtility.UrlEncode(searchInfo.Name), - artistMusicBrainzId); - } - else - { - // I'm sure there is a better way but for now it resolves search for 12" Mixes - var queryName = searchInfo.Name.Replace("\"", string.Empty); - - url = string.Format( - CultureInfo.InvariantCulture, - "/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", - WebUtility.UrlEncode(queryName), - WebUtility.UrlEncode(searchInfo.GetAlbumArtist())); - } - } - - if (!string.IsNullOrWhiteSpace(url)) - { - using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - using (var stream = response.Content) - { - return GetResultsFromResponse(stream); - } - } - - return Enumerable.Empty(); - } - - private IEnumerable GetResultsFromResponse(Stream stream) - { - using (var oReader = new StreamReader(stream, Encoding.UTF8)) - { - var settings = new XmlReaderSettings() - { - ValidationType = ValidationType.None, - CheckCharacters = false, - IgnoreProcessingInstructions = true, - IgnoreComments = true - }; - - using (var reader = XmlReader.Create(oReader, settings)) - { - var results = ReleaseResult.Parse(reader); - - return results.Select(i => - { - var result = new RemoteSearchResult - { - Name = i.Title, - ProductionYear = i.Year - }; - - if (i.Artists.Count > 0) - { - result.AlbumArtist = new RemoteSearchResult - { - SearchProviderName = Name, - Name = i.Artists[0].Item1 - }; - - result.AlbumArtist.SetProviderId(MetadataProviders.MusicBrainzArtist, i.Artists[0].Item2); - } - - if (!string.IsNullOrWhiteSpace(i.ReleaseId)) - { - result.SetProviderId(MetadataProviders.MusicBrainzAlbum, i.ReleaseId); - } - - if (!string.IsNullOrWhiteSpace(i.ReleaseGroupId)) - { - result.SetProviderId(MetadataProviders.MusicBrainzReleaseGroup, i.ReleaseGroupId); - } - - return result; - - }); - } - } - } - - /// - public async Task> GetMetadata(AlbumInfo id, CancellationToken cancellationToken) - { - var releaseId = id.GetReleaseId(); - var releaseGroupId = id.GetReleaseGroupId(); - - var result = new MetadataResult - { - Item = new MusicAlbum() - }; - - // If we have a release group Id but not a release Id... - if (string.IsNullOrWhiteSpace(releaseId) && !string.IsNullOrWhiteSpace(releaseGroupId)) - { - releaseId = await GetReleaseIdFromReleaseGroupId(releaseGroupId, cancellationToken).ConfigureAwait(false); - result.HasMetadata = true; - } - - if (string.IsNullOrWhiteSpace(releaseId)) - { - var artistMusicBrainzId = id.GetMusicBrainzArtistId(); - - var releaseResult = await GetReleaseResult(artistMusicBrainzId, id.GetAlbumArtist(), id.Name, cancellationToken).ConfigureAwait(false); - - if (releaseResult != null) - { - if (!string.IsNullOrWhiteSpace(releaseResult.ReleaseId)) - { - releaseId = releaseResult.ReleaseId; - result.HasMetadata = true; - } - - if (!string.IsNullOrWhiteSpace(releaseResult.ReleaseGroupId)) - { - releaseGroupId = releaseResult.ReleaseGroupId; - result.HasMetadata = true; - } - - result.Item.ProductionYear = releaseResult.Year; - result.Item.Overview = releaseResult.Overview; - } - } - - // If we have a release Id but not a release group Id... - if (!string.IsNullOrWhiteSpace(releaseId) && string.IsNullOrWhiteSpace(releaseGroupId)) - { - releaseGroupId = await GetReleaseGroupFromReleaseId(releaseId, cancellationToken).ConfigureAwait(false); - result.HasMetadata = true; - } - - if (!string.IsNullOrWhiteSpace(releaseId) || !string.IsNullOrWhiteSpace(releaseGroupId)) - { - result.HasMetadata = true; - } - - if (result.HasMetadata) - { - if (!string.IsNullOrEmpty(releaseId)) - { - result.Item.SetProviderId(MetadataProviders.MusicBrainzAlbum, releaseId); - } - - if (!string.IsNullOrEmpty(releaseGroupId)) - { - result.Item.SetProviderId(MetadataProviders.MusicBrainzReleaseGroup, releaseGroupId); - } - } - - return result; - } - - private Task GetReleaseResult(string artistMusicBrainId, string artistName, string albumName, CancellationToken cancellationToken) - { - if (!string.IsNullOrEmpty(artistMusicBrainId)) - { - return GetReleaseResult(albumName, artistMusicBrainId, cancellationToken); - } - - if (string.IsNullOrWhiteSpace(artistName)) - { - return Task.FromResult(new ReleaseResult()); - } - - return GetReleaseResultByArtistName(albumName, artistName, cancellationToken); - } - - private async Task GetReleaseResult(string albumName, string artistId, CancellationToken cancellationToken) - { - var url = string.Format("/ws/2/release/?query=\"{0}\" AND arid:{1}", - WebUtility.UrlEncode(albumName), - artistId); - - using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - using (var stream = response.Content) - using (var oReader = new StreamReader(stream, Encoding.UTF8)) - { - var settings = new XmlReaderSettings() - { - ValidationType = ValidationType.None, - CheckCharacters = false, - IgnoreProcessingInstructions = true, - IgnoreComments = true - }; - - using (var reader = XmlReader.Create(oReader, settings)) - { - return ReleaseResult.Parse(reader).FirstOrDefault(); - } - } - } - - private async Task GetReleaseResultByArtistName(string albumName, string artistName, CancellationToken cancellationToken) - { - var url = string.Format( - CultureInfo.InvariantCulture, - "/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", - WebUtility.UrlEncode(albumName), - WebUtility.UrlEncode(artistName)); - - using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - using (var stream = response.Content) - using (var oReader = new StreamReader(stream, Encoding.UTF8)) - { - var settings = new XmlReaderSettings() - { - ValidationType = ValidationType.None, - CheckCharacters = false, - IgnoreProcessingInstructions = true, - IgnoreComments = true - }; - - using (var reader = XmlReader.Create(oReader, settings)) - { - return ReleaseResult.Parse(reader).FirstOrDefault(); - } - } - } - - private class ReleaseResult - { - public string ReleaseId; - public string ReleaseGroupId; - public string Title; - public string Overview; - public int? Year; - - public List> Artists = new List>(); - - public static IEnumerable Parse(XmlReader reader) - { - reader.MoveToContent(); - reader.Read(); - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "release-list": - { - if (reader.IsEmptyElement) - { - reader.Read(); - continue; - } - - using (var subReader = reader.ReadSubtree()) - { - return ParseReleaseList(subReader).ToList(); - } - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return Enumerable.Empty(); - } - - private static IEnumerable ParseReleaseList(XmlReader reader) - { - reader.MoveToContent(); - reader.Read(); - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "release": - { - if (reader.IsEmptyElement) - { - reader.Read(); - continue; - } - var releaseId = reader.GetAttribute("id"); - - using (var subReader = reader.ReadSubtree()) - { - var release = ParseRelease(subReader, releaseId); - if (release != null) - { - yield return release; - } - } - break; - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - } - - private static ReleaseResult ParseRelease(XmlReader reader, string releaseId) - { - var result = new ReleaseResult - { - ReleaseId = releaseId - }; - - reader.MoveToContent(); - reader.Read(); - - // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "title": - { - result.Title = reader.ReadElementContentAsString(); - break; - } - case "date": - { - var val = reader.ReadElementContentAsString(); - if (DateTime.TryParse(val, out var date)) - { - result.Year = date.Year; - } - break; - } - case "annotation": - { - result.Overview = reader.ReadElementContentAsString(); - break; - } - case "release-group": - { - result.ReleaseGroupId = reader.GetAttribute("id"); - reader.Skip(); - break; - } - case "artist-credit": - { - // TODO - - /* - * - - -SARCASTIC+ZOOKEEPER -SARCASTIC+ZOOKEEPER - - - - */ - using (var subReader = reader.ReadSubtree()) - { - var artist = ParseArtistCredit(subReader); - - if (!string.IsNullOrEmpty(artist.Item1)) - { - result.Artists.Add(artist); - } - } - - break; - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return result; - } - } - - private static ValueTuple ParseArtistCredit(XmlReader reader) - { - reader.MoveToContent(); - reader.Read(); - - // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "name-credit": - { - using (var subReader = reader.ReadSubtree()) - { - return ParseArtistNameCredit(subReader); - } - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return new ValueTuple(); - } - - private static (string, string) ParseArtistNameCredit(XmlReader reader) - { - reader.MoveToContent(); - reader.Read(); - - // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "artist": - { - var id = reader.GetAttribute("id"); - using (var subReader = reader.ReadSubtree()) - { - return ParseArtistArtistCredit(subReader, id); - } - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return (null, null); - } - - private static (string name, string id) ParseArtistArtistCredit(XmlReader reader, string artistId) - { - reader.MoveToContent(); - reader.Read(); - - string name = null; - - // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "name": - { - name = reader.ReadElementContentAsString(); - break; - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return (name, artistId); - } - - private async Task GetReleaseIdFromReleaseGroupId(string releaseGroupId, CancellationToken cancellationToken) - { - var url = "/ws/2/release?release-group=" + releaseGroupId.ToString(CultureInfo.InvariantCulture); - - using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - using (var stream = response.Content) - using (var oReader = new StreamReader(stream, Encoding.UTF8)) - { - var settings = new XmlReaderSettings() - { - ValidationType = ValidationType.None, - CheckCharacters = false, - IgnoreProcessingInstructions = true, - IgnoreComments = true - }; - - using (var reader = XmlReader.Create(oReader, settings)) - { - var result = ReleaseResult.Parse(reader).FirstOrDefault(); - - if (result != null) - { - return result.ReleaseId; - } - } - } - - return null; - } - - /// - /// Gets the release group id internal. - /// - /// The release entry id. - /// The cancellation token. - /// Task{System.String}. - private async Task GetReleaseGroupFromReleaseId(string releaseEntryId, CancellationToken cancellationToken) - { - var url = "/ws/2/release-group/?query=reid:" + releaseEntryId.ToString(CultureInfo.InvariantCulture); - - using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - using (var stream = response.Content) - using (var oReader = new StreamReader(stream, Encoding.UTF8)) - { - var settings = new XmlReaderSettings() - { - ValidationType = ValidationType.None, - CheckCharacters = false, - IgnoreProcessingInstructions = true, - IgnoreComments = true - }; - - using (var reader = XmlReader.Create(oReader, settings)) - { - reader.MoveToContent(); - reader.Read(); - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "release-group-list": - { - if (reader.IsEmptyElement) - { - reader.Read(); - continue; - } - using (var subReader = reader.ReadSubtree()) - { - return GetFirstReleaseGroupId(subReader); - } - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return null; - } - } - } - - private string GetFirstReleaseGroupId(XmlReader reader) - { - reader.MoveToContent(); - reader.Read(); - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "release-group": - { - return reader.GetAttribute("id"); - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return null; - } - - /// - /// Makes request to MusicBrainz server and awaits a response. - /// A 503 Service Unavailable response indicates throttling to maintain a rate limit. - /// A number of retries shall be made in order to try and satisfy the request before - /// giving up and returning null. - /// - internal async Task GetMusicBrainzResponse(string url, CancellationToken cancellationToken) - { - var options = new HttpRequestOptions - { - Url = _musicBrainzBaseUrl.TrimEnd('/') + url, - CancellationToken = cancellationToken, - // MusicBrainz request a contact email address is supplied, as comment, in user agent field: - // https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting#User-Agent - UserAgent = string.Format( - CultureInfo.InvariantCulture, - "{0} ( {1} )", - _appHost.ApplicationUserAgent, - _appHost.ApplicationUserAgentAddress), - BufferContent = false - }; - - HttpResponseInfo response; - var attempts = 0u; - - do - { - attempts++; - - if (_stopWatchMusicBrainz.ElapsedMilliseconds < MusicBrainzQueryIntervalMs) - { - // MusicBrainz is extremely adamant about limiting to one request per second - var delayMs = MusicBrainzQueryIntervalMs - _stopWatchMusicBrainz.ElapsedMilliseconds; - await Task.Delay((int)delayMs, cancellationToken).ConfigureAwait(false); - } - - // Write time since last request to debug log as evidence we're meeting rate limit - // requirement, before resetting stopwatch back to zero. - _logger.LogDebug("GetMusicBrainzResponse: Time since previous request: {0} ms", _stopWatchMusicBrainz.ElapsedMilliseconds); - _stopWatchMusicBrainz.Restart(); - - response = await _httpClient.SendAsync(options, "GET").ConfigureAwait(false); - - // We retry a finite number of times, and only whilst MB is indcating 503 (throttling) - } - while (attempts < MusicBrainzQueryAttempts && response.StatusCode == HttpStatusCode.ServiceUnavailable); - - // Log error if unable to query MB database due to throttling - if (attempts == MusicBrainzQueryAttempts && response.StatusCode == HttpStatusCode.ServiceUnavailable) - { - _logger.LogError("GetMusicBrainzResponse: 503 Service Unavailable (throttled) response received {0} times whilst requesting {1}", attempts, options.Url); - } - - return response; - } - - /// - public Task GetImageResponse(string url, CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } - } -} diff --git a/MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs b/MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs deleted file mode 100644 index 5d675392c..000000000 --- a/MediaBrowser.Providers/Music/MusicBrainzArtistProvider.cs +++ /dev/null @@ -1,281 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using System.Xml; -using MediaBrowser.Common.Net; -using MediaBrowser.Controller.Entities.Audio; -using MediaBrowser.Controller.Extensions; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.Entities; -using MediaBrowser.Model.Providers; - -namespace MediaBrowser.Providers.Music -{ - public class MusicBrainzArtistProvider : IRemoteMetadataProvider - { - /// - public async Task> GetSearchResults(ArtistInfo searchInfo, CancellationToken cancellationToken) - { - var musicBrainzId = searchInfo.GetMusicBrainzArtistId(); - - if (!string.IsNullOrWhiteSpace(musicBrainzId)) - { - var url = "/ws/2/artist/?query=arid:{0}" + musicBrainzId.ToString(CultureInfo.InvariantCulture); - - using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - using (var stream = response.Content) - { - return GetResultsFromResponse(stream); - } - } - else - { - // They seem to throw bad request failures on any term with a slash - var nameToSearch = searchInfo.Name.Replace('/', ' '); - - var url = string.Format("/ws/2/artist/?query=\"{0}\"&dismax=true", UrlEncode(nameToSearch)); - - using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - using (var stream = response.Content) - { - var results = GetResultsFromResponse(stream).ToList(); - - if (results.Count > 0) - { - return results; - } - } - - if (HasDiacritics(searchInfo.Name)) - { - // Try again using the search with accent characters url - url = string.Format("/ws/2/artist/?query=artistaccent:\"{0}\"", UrlEncode(nameToSearch)); - - using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) - { - using (var stream = response.Content) - { - return GetResultsFromResponse(stream); - } - } - } - } - - return Enumerable.Empty(); - } - - private IEnumerable GetResultsFromResponse(Stream stream) - { - using (var oReader = new StreamReader(stream, Encoding.UTF8)) - { - var settings = new XmlReaderSettings() - { - ValidationType = ValidationType.None, - CheckCharacters = false, - IgnoreProcessingInstructions = true, - IgnoreComments = true - }; - - using (var reader = XmlReader.Create(oReader, settings)) - { - reader.MoveToContent(); - reader.Read(); - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "artist-list": - { - if (reader.IsEmptyElement) - { - reader.Read(); - continue; - } - using (var subReader = reader.ReadSubtree()) - { - return ParseArtistList(subReader).ToList(); - } - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - return Enumerable.Empty(); - } - } - } - - private IEnumerable ParseArtistList(XmlReader reader) - { - reader.MoveToContent(); - reader.Read(); - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "artist": - { - if (reader.IsEmptyElement) - { - reader.Read(); - continue; - } - var mbzId = reader.GetAttribute("id"); - - using (var subReader = reader.ReadSubtree()) - { - var artist = ParseArtist(subReader, mbzId); - if (artist != null) - { - yield return artist; - } - } - break; - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - } - - private RemoteSearchResult ParseArtist(XmlReader reader, string artistId) - { - var result = new RemoteSearchResult(); - - reader.MoveToContent(); - reader.Read(); - - // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "name": - { - result.Name = reader.ReadElementContentAsString(); - break; - } - case "annotation": - { - result.Overview = reader.ReadElementContentAsString(); - break; - } - default: - { - // there is sort-name if ever needed - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - - result.SetProviderId(MetadataProviders.MusicBrainzArtist, artistId); - - if (string.IsNullOrWhiteSpace(artistId) || string.IsNullOrWhiteSpace(result.Name)) - { - return null; - } - - return result; - } - - public async Task> GetMetadata(ArtistInfo id, CancellationToken cancellationToken) - { - var result = new MetadataResult - { - Item = new MusicArtist() - }; - - var musicBrainzId = id.GetMusicBrainzArtistId(); - - if (string.IsNullOrWhiteSpace(musicBrainzId)) - { - var searchResults = await GetSearchResults(id, cancellationToken).ConfigureAwait(false); - - var singleResult = searchResults.FirstOrDefault(); - - if (singleResult != null) - { - musicBrainzId = singleResult.GetProviderId(MetadataProviders.MusicBrainzArtist); - //result.Item.Name = singleResult.Name; - result.Item.Overview = singleResult.Overview; - } - } - - if (!string.IsNullOrWhiteSpace(musicBrainzId)) - { - result.HasMetadata = true; - result.Item.SetProviderId(MetadataProviders.MusicBrainzArtist, musicBrainzId); - } - - return result; - } - - /// - /// Determines whether the specified text has diacritics. - /// - /// The text. - /// true if the specified text has diacritics; otherwise, false. - private bool HasDiacritics(string text) - { - return !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal); - } - - /// - /// Encodes an URL. - /// - /// The name. - /// System.String. - private static string UrlEncode(string name) - { - return WebUtility.UrlEncode(name); - } - - public string Name => "MusicBrainz"; - - public Task GetImageResponse(string url, CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } - } -} diff --git a/MediaBrowser.Providers/Music/MusicExternalIds.cs b/MediaBrowser.Providers/Music/MusicExternalIds.cs index 585c98af9..b431c1e41 100644 --- a/MediaBrowser.Providers/Music/MusicExternalIds.cs +++ b/MediaBrowser.Providers/Music/MusicExternalIds.cs @@ -5,101 +5,6 @@ using MediaBrowser.Model.Entities; namespace MediaBrowser.Providers.Music { - public class MusicBrainzReleaseGroupExternalId : IExternalId - { - /// - public string Name => "MusicBrainz Release Group"; - - /// - public string Key => MetadataProviders.MusicBrainzReleaseGroup.ToString(); - - /// - public string UrlFormatString => "https://musicbrainz.org/release-group/{0}"; - - /// - public bool Supports(IHasProviderIds item) - => item is Audio || item is MusicAlbum; - } - - public class MusicBrainzAlbumArtistExternalId : IExternalId - { - /// - public string Name => "MusicBrainz Album Artist"; - - /// - public string Key => MetadataProviders.MusicBrainzAlbumArtist.ToString(); - - /// - public string UrlFormatString => "https://musicbrainz.org/artist/{0}"; - - /// - public bool Supports(IHasProviderIds item) - => item is Audio; - } - - public class MusicBrainzAlbumExternalId : IExternalId - { - /// - public string Name => "MusicBrainz Album"; - - /// - public string Key => MetadataProviders.MusicBrainzAlbum.ToString(); - - /// - public string UrlFormatString => "https://musicbrainz.org/release/{0}"; - - /// - public bool Supports(IHasProviderIds item) - => item is Audio || item is MusicAlbum; - } - - public class MusicBrainzArtistExternalId : IExternalId - { - /// - public string Name => "MusicBrainz"; - - /// - public string Key => MetadataProviders.MusicBrainzArtist.ToString(); - - /// - public string UrlFormatString => "https://musicbrainz.org/artist/{0}"; - - /// - public bool Supports(IHasProviderIds item) => item is MusicArtist; - } - - public class MusicBrainzOtherArtistExternalId : IExternalId - { - /// - public string Name => "MusicBrainz Artist"; - - /// - - public string Key => MetadataProviders.MusicBrainzArtist.ToString(); - - /// - public string UrlFormatString => "https://musicbrainz.org/artist/{0}"; - - /// - public bool Supports(IHasProviderIds item) - => item is Audio || item is MusicAlbum; - } - - public class MusicBrainzTrackId : IExternalId - { - /// - public string Name => "MusicBrainz Track"; - - /// - public string Key => MetadataProviders.MusicBrainzTrack.ToString(); - - /// - public string UrlFormatString => "https://musicbrainz.org/track/{0}"; - - /// - public bool Supports(IHasProviderIds item) => item is Audio; - } - public class ImvdbId : IExternalId { /// diff --git a/MediaBrowser.Providers/Plugins/MusicBrainz/AlbumProvider.cs b/MediaBrowser.Providers/Plugins/MusicBrainz/AlbumProvider.cs new file mode 100644 index 000000000..0fddb0557 --- /dev/null +++ b/MediaBrowser.Providers/Plugins/MusicBrainz/AlbumProvider.cs @@ -0,0 +1,799 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using MediaBrowser.Common; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Providers; +using MediaBrowser.Providers.Plugins.MusicBrainz; +using Microsoft.Extensions.Logging; + +namespace MediaBrowser.Providers.Music +{ + public class MusicBrainzAlbumProvider : IRemoteMetadataProvider, IHasOrder + { + /// + /// The Jellyfin user-agent is unrestricted but source IP must not exceed + /// one request per second, therefore we rate limit to avoid throttling. + /// Be prudent, use a value slightly above the minimun required. + /// https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting + /// + private readonly long _musicBrainzQueryIntervalMs = 1050u; + + /// + /// For each single MB lookup/search, this is the maximum number of + /// attempts that shall be made whilst receiving a 503 Server + /// Unavailable (indicating throttled) response. + /// + private const uint MusicBrainzQueryAttempts = 5u; + + internal static MusicBrainzAlbumProvider Current; + + private readonly IHttpClient _httpClient; + private readonly IApplicationHost _appHost; + private readonly ILogger _logger; + + private readonly string _musicBrainzBaseUrl; + + private Stopwatch _stopWatchMusicBrainz = new Stopwatch(); + + public MusicBrainzAlbumProvider( + IHttpClient httpClient, + IApplicationHost appHost, + ILogger logger) + { + _httpClient = httpClient; + _appHost = appHost; + _logger = logger; + + _musicBrainzBaseUrl = Plugin.Instance.Configuration.Server; + _musicBrainzQueryIntervalMs = Plugin.Instance.Configuration.RateLimit; + + // Use a stopwatch to ensure we don't exceed the MusicBrainz rate limit + _stopWatchMusicBrainz.Start(); + + Current = this; + } + + /// + public string Name => "MusicBrainz"; + + /// + public int Order => 0; + + /// + public async Task> GetSearchResults(AlbumInfo searchInfo, CancellationToken cancellationToken) + { + // TODO maybe remove when artist metadata can be disabled + if (!Plugin.Instance.Configuration.Enable) + { + return Enumerable.Empty(); + } + + var releaseId = searchInfo.GetReleaseId(); + var releaseGroupId = searchInfo.GetReleaseGroupId(); + + string url; + + if (!string.IsNullOrEmpty(releaseId)) + { + url = "/ws/2/release/?query=reid:" + releaseId.ToString(CultureInfo.InvariantCulture); + } + else if (!string.IsNullOrEmpty(releaseGroupId)) + { + url = "/ws/2/release?release-group=" + releaseGroupId.ToString(CultureInfo.InvariantCulture); + } + else + { + var artistMusicBrainzId = searchInfo.GetMusicBrainzArtistId(); + + if (!string.IsNullOrWhiteSpace(artistMusicBrainzId)) + { + url = string.Format( + CultureInfo.InvariantCulture, + "/ws/2/release/?query=\"{0}\" AND arid:{1}", + WebUtility.UrlEncode(searchInfo.Name), + artistMusicBrainzId); + } + else + { + // I'm sure there is a better way but for now it resolves search for 12" Mixes + var queryName = searchInfo.Name.Replace("\"", string.Empty); + + url = string.Format( + CultureInfo.InvariantCulture, + "/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", + WebUtility.UrlEncode(queryName), + WebUtility.UrlEncode(searchInfo.GetAlbumArtist())); + } + } + + if (!string.IsNullOrWhiteSpace(url)) + { + using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + using (var stream = response.Content) + { + return GetResultsFromResponse(stream); + } + } + + return Enumerable.Empty(); + } + + private IEnumerable GetResultsFromResponse(Stream stream) + { + using (var oReader = new StreamReader(stream, Encoding.UTF8)) + { + var settings = new XmlReaderSettings() + { + ValidationType = ValidationType.None, + CheckCharacters = false, + IgnoreProcessingInstructions = true, + IgnoreComments = true + }; + + using (var reader = XmlReader.Create(oReader, settings)) + { + var results = ReleaseResult.Parse(reader); + + return results.Select(i => + { + var result = new RemoteSearchResult + { + Name = i.Title, + ProductionYear = i.Year + }; + + if (i.Artists.Count > 0) + { + result.AlbumArtist = new RemoteSearchResult + { + SearchProviderName = Name, + Name = i.Artists[0].Item1 + }; + + result.AlbumArtist.SetProviderId(MetadataProviders.MusicBrainzArtist, i.Artists[0].Item2); + } + + if (!string.IsNullOrWhiteSpace(i.ReleaseId)) + { + result.SetProviderId(MetadataProviders.MusicBrainzAlbum, i.ReleaseId); + } + + if (!string.IsNullOrWhiteSpace(i.ReleaseGroupId)) + { + result.SetProviderId(MetadataProviders.MusicBrainzReleaseGroup, i.ReleaseGroupId); + } + + return result; + }); + } + } + } + + /// + public async Task> GetMetadata(AlbumInfo id, CancellationToken cancellationToken) + { + var releaseId = id.GetReleaseId(); + var releaseGroupId = id.GetReleaseGroupId(); + + var result = new MetadataResult + { + Item = new MusicAlbum() + }; + + // TODO maybe remove when artist metadata can be disabled + if (!Plugin.Instance.Configuration.Enable) + { + return result; + } + + // If we have a release group Id but not a release Id... + if (string.IsNullOrWhiteSpace(releaseId) && !string.IsNullOrWhiteSpace(releaseGroupId)) + { + releaseId = await GetReleaseIdFromReleaseGroupId(releaseGroupId, cancellationToken).ConfigureAwait(false); + result.HasMetadata = true; + } + + if (string.IsNullOrWhiteSpace(releaseId)) + { + var artistMusicBrainzId = id.GetMusicBrainzArtistId(); + + var releaseResult = await GetReleaseResult(artistMusicBrainzId, id.GetAlbumArtist(), id.Name, cancellationToken).ConfigureAwait(false); + + if (releaseResult != null) + { + if (!string.IsNullOrWhiteSpace(releaseResult.ReleaseId)) + { + releaseId = releaseResult.ReleaseId; + result.HasMetadata = true; + } + + if (!string.IsNullOrWhiteSpace(releaseResult.ReleaseGroupId)) + { + releaseGroupId = releaseResult.ReleaseGroupId; + result.HasMetadata = true; + } + + result.Item.ProductionYear = releaseResult.Year; + result.Item.Overview = releaseResult.Overview; + } + } + + // If we have a release Id but not a release group Id... + if (!string.IsNullOrWhiteSpace(releaseId) && string.IsNullOrWhiteSpace(releaseGroupId)) + { + releaseGroupId = await GetReleaseGroupFromReleaseId(releaseId, cancellationToken).ConfigureAwait(false); + result.HasMetadata = true; + } + + if (!string.IsNullOrWhiteSpace(releaseId) || !string.IsNullOrWhiteSpace(releaseGroupId)) + { + result.HasMetadata = true; + } + + if (result.HasMetadata) + { + if (!string.IsNullOrEmpty(releaseId)) + { + result.Item.SetProviderId(MetadataProviders.MusicBrainzAlbum, releaseId); + } + + if (!string.IsNullOrEmpty(releaseGroupId)) + { + result.Item.SetProviderId(MetadataProviders.MusicBrainzReleaseGroup, releaseGroupId); + } + } + + return result; + } + + private Task GetReleaseResult(string artistMusicBrainId, string artistName, string albumName, CancellationToken cancellationToken) + { + if (!string.IsNullOrEmpty(artistMusicBrainId)) + { + return GetReleaseResult(albumName, artistMusicBrainId, cancellationToken); + } + + if (string.IsNullOrWhiteSpace(artistName)) + { + return Task.FromResult(new ReleaseResult()); + } + + return GetReleaseResultByArtistName(albumName, artistName, cancellationToken); + } + + private async Task GetReleaseResult(string albumName, string artistId, CancellationToken cancellationToken) + { + var url = string.Format("/ws/2/release/?query=\"{0}\" AND arid:{1}", + WebUtility.UrlEncode(albumName), + artistId); + + using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + using (var stream = response.Content) + using (var oReader = new StreamReader(stream, Encoding.UTF8)) + { + var settings = new XmlReaderSettings() + { + ValidationType = ValidationType.None, + CheckCharacters = false, + IgnoreProcessingInstructions = true, + IgnoreComments = true + }; + + using (var reader = XmlReader.Create(oReader, settings)) + { + return ReleaseResult.Parse(reader).FirstOrDefault(); + } + } + } + + private async Task GetReleaseResultByArtistName(string albumName, string artistName, CancellationToken cancellationToken) + { + var url = string.Format( + CultureInfo.InvariantCulture, + "/ws/2/release/?query=\"{0}\" AND artist:\"{1}\"", + WebUtility.UrlEncode(albumName), + WebUtility.UrlEncode(artistName)); + + using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + using (var stream = response.Content) + using (var oReader = new StreamReader(stream, Encoding.UTF8)) + { + var settings = new XmlReaderSettings() + { + ValidationType = ValidationType.None, + CheckCharacters = false, + IgnoreProcessingInstructions = true, + IgnoreComments = true + }; + + using (var reader = XmlReader.Create(oReader, settings)) + { + return ReleaseResult.Parse(reader).FirstOrDefault(); + } + } + } + + private class ReleaseResult + { + public string ReleaseId; + public string ReleaseGroupId; + public string Title; + public string Overview; + public int? Year; + + public List> Artists = new List>(); + + public static IEnumerable Parse(XmlReader reader) + { + reader.MoveToContent(); + reader.Read(); + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "release-list": + { + if (reader.IsEmptyElement) + { + reader.Read(); + continue; + } + + using (var subReader = reader.ReadSubtree()) + { + return ParseReleaseList(subReader).ToList(); + } + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return Enumerable.Empty(); + } + + private static IEnumerable ParseReleaseList(XmlReader reader) + { + reader.MoveToContent(); + reader.Read(); + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "release": + { + if (reader.IsEmptyElement) + { + reader.Read(); + continue; + } + var releaseId = reader.GetAttribute("id"); + + using (var subReader = reader.ReadSubtree()) + { + var release = ParseRelease(subReader, releaseId); + if (release != null) + { + yield return release; + } + } + break; + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + } + + private static ReleaseResult ParseRelease(XmlReader reader, string releaseId) + { + var result = new ReleaseResult + { + ReleaseId = releaseId + }; + + reader.MoveToContent(); + reader.Read(); + + // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "title": + { + result.Title = reader.ReadElementContentAsString(); + break; + } + case "date": + { + var val = reader.ReadElementContentAsString(); + if (DateTime.TryParse(val, out var date)) + { + result.Year = date.Year; + } + break; + } + case "annotation": + { + result.Overview = reader.ReadElementContentAsString(); + break; + } + case "release-group": + { + result.ReleaseGroupId = reader.GetAttribute("id"); + reader.Skip(); + break; + } + case "artist-credit": + { + using (var subReader = reader.ReadSubtree()) + { + var artist = ParseArtistCredit(subReader); + + if (!string.IsNullOrEmpty(artist.Item1)) + { + result.Artists.Add(artist); + } + } + + break; + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return result; + } + } + + private static ValueTuple ParseArtistCredit(XmlReader reader) + { + reader.MoveToContent(); + reader.Read(); + + // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "name-credit": + { + using (var subReader = reader.ReadSubtree()) + { + return ParseArtistNameCredit(subReader); + } + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return new ValueTuple(); + } + + private static (string, string) ParseArtistNameCredit(XmlReader reader) + { + reader.MoveToContent(); + reader.Read(); + + // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "artist": + { + var id = reader.GetAttribute("id"); + using (var subReader = reader.ReadSubtree()) + { + return ParseArtistArtistCredit(subReader, id); + } + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return (null, null); + } + + private static (string name, string id) ParseArtistArtistCredit(XmlReader reader, string artistId) + { + reader.MoveToContent(); + reader.Read(); + + string name = null; + + // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "name": + { + name = reader.ReadElementContentAsString(); + break; + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return (name, artistId); + } + + private async Task GetReleaseIdFromReleaseGroupId(string releaseGroupId, CancellationToken cancellationToken) + { + var url = "/ws/2/release?release-group=" + releaseGroupId.ToString(CultureInfo.InvariantCulture); + + using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + using (var stream = response.Content) + using (var oReader = new StreamReader(stream, Encoding.UTF8)) + { + var settings = new XmlReaderSettings() + { + ValidationType = ValidationType.None, + CheckCharacters = false, + IgnoreProcessingInstructions = true, + IgnoreComments = true + }; + + using (var reader = XmlReader.Create(oReader, settings)) + { + var result = ReleaseResult.Parse(reader).FirstOrDefault(); + + if (result != null) + { + return result.ReleaseId; + } + } + } + + return null; + } + + /// + /// Gets the release group id internal. + /// + /// The release entry id. + /// The cancellation token. + /// Task{System.String}. + private async Task GetReleaseGroupFromReleaseId(string releaseEntryId, CancellationToken cancellationToken) + { + var url = "/ws/2/release-group/?query=reid:" + releaseEntryId.ToString(CultureInfo.InvariantCulture); + + using (var response = await GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + using (var stream = response.Content) + using (var oReader = new StreamReader(stream, Encoding.UTF8)) + { + var settings = new XmlReaderSettings() + { + ValidationType = ValidationType.None, + CheckCharacters = false, + IgnoreProcessingInstructions = true, + IgnoreComments = true + }; + + using (var reader = XmlReader.Create(oReader, settings)) + { + reader.MoveToContent(); + reader.Read(); + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "release-group-list": + { + if (reader.IsEmptyElement) + { + reader.Read(); + continue; + } + using (var subReader = reader.ReadSubtree()) + { + return GetFirstReleaseGroupId(subReader); + } + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return null; + } + } + } + + private string GetFirstReleaseGroupId(XmlReader reader) + { + reader.MoveToContent(); + reader.Read(); + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "release-group": + { + return reader.GetAttribute("id"); + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return null; + } + + /// + /// Makes request to MusicBrainz server and awaits a response. + /// A 503 Service Unavailable response indicates throttling to maintain a rate limit. + /// A number of retries shall be made in order to try and satisfy the request before + /// giving up and returning null. + /// + internal async Task GetMusicBrainzResponse(string url, CancellationToken cancellationToken) + { + var options = new HttpRequestOptions + { + Url = _musicBrainzBaseUrl.TrimEnd('/') + url, + CancellationToken = cancellationToken, + // MusicBrainz request a contact email address is supplied, as comment, in user agent field: + // https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting#User-Agent + UserAgent = string.Format( + CultureInfo.InvariantCulture, + "{0} ( {1} )", + _appHost.ApplicationUserAgent, + _appHost.ApplicationUserAgentAddress), + BufferContent = false + }; + + HttpResponseInfo response; + var attempts = 0u; + + do + { + attempts++; + + if (_stopWatchMusicBrainz.ElapsedMilliseconds < _musicBrainzQueryIntervalMs) + { + // MusicBrainz is extremely adamant about limiting to one request per second + var delayMs = _musicBrainzQueryIntervalMs - _stopWatchMusicBrainz.ElapsedMilliseconds; + await Task.Delay((int)delayMs, cancellationToken).ConfigureAwait(false); + } + + // Write time since last request to debug log as evidence we're meeting rate limit + // requirement, before resetting stopwatch back to zero. + _logger.LogDebug("GetMusicBrainzResponse: Time since previous request: {0} ms", _stopWatchMusicBrainz.ElapsedMilliseconds); + _stopWatchMusicBrainz.Restart(); + + response = await _httpClient.SendAsync(options, "GET").ConfigureAwait(false); + + // We retry a finite number of times, and only whilst MB is indicating 503 (throttling) + } + while (attempts < MusicBrainzQueryAttempts && response.StatusCode == HttpStatusCode.ServiceUnavailable); + + // Log error if unable to query MB database due to throttling + if (attempts == MusicBrainzQueryAttempts && response.StatusCode == HttpStatusCode.ServiceUnavailable) + { + _logger.LogError("GetMusicBrainzResponse: 503 Service Unavailable (throttled) response received {0} times whilst requesting {1}", attempts, options.Url); + } + + return response; + } + + /// + public Task GetImageResponse(string url, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } +} diff --git a/MediaBrowser.Providers/Plugins/MusicBrainz/ArtistProvider.cs b/MediaBrowser.Providers/Plugins/MusicBrainz/ArtistProvider.cs new file mode 100644 index 000000000..260a3b6e7 --- /dev/null +++ b/MediaBrowser.Providers/Plugins/MusicBrainz/ArtistProvider.cs @@ -0,0 +1,298 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Extensions; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Providers; +using MediaBrowser.Providers.Plugins.MusicBrainz; + +namespace MediaBrowser.Providers.Music +{ + public class MusicBrainzArtistProvider : IRemoteMetadataProvider + { + /// + public async Task> GetSearchResults(ArtistInfo searchInfo, CancellationToken cancellationToken) + { + // TODO maybe remove when artist metadata can be disabled + if (!Plugin.Instance.Configuration.Enable) + { + return Enumerable.Empty(); + } + + var musicBrainzId = searchInfo.GetMusicBrainzArtistId(); + + if (!string.IsNullOrWhiteSpace(musicBrainzId)) + { + var url = "/ws/2/artist/?query=arid:{0}" + musicBrainzId.ToString(CultureInfo.InvariantCulture); + + using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + using (var stream = response.Content) + { + return GetResultsFromResponse(stream); + } + } + else + { + // They seem to throw bad request failures on any term with a slash + var nameToSearch = searchInfo.Name.Replace('/', ' '); + + var url = string.Format("/ws/2/artist/?query=\"{0}\"&dismax=true", UrlEncode(nameToSearch)); + + using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + using (var stream = response.Content) + { + var results = GetResultsFromResponse(stream).ToList(); + + if (results.Count > 0) + { + return results; + } + } + + if (HasDiacritics(searchInfo.Name)) + { + // Try again using the search with accent characters url + url = string.Format("/ws/2/artist/?query=artistaccent:\"{0}\"", UrlEncode(nameToSearch)); + + using (var response = await MusicBrainzAlbumProvider.Current.GetMusicBrainzResponse(url, cancellationToken).ConfigureAwait(false)) + { + using (var stream = response.Content) + { + return GetResultsFromResponse(stream); + } + } + } + } + + return Enumerable.Empty(); + } + + private IEnumerable GetResultsFromResponse(Stream stream) + { + using (var oReader = new StreamReader(stream, Encoding.UTF8)) + { + var settings = new XmlReaderSettings() + { + ValidationType = ValidationType.None, + CheckCharacters = false, + IgnoreProcessingInstructions = true, + IgnoreComments = true + }; + + using (var reader = XmlReader.Create(oReader, settings)) + { + reader.MoveToContent(); + reader.Read(); + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "artist-list": + { + if (reader.IsEmptyElement) + { + reader.Read(); + continue; + } + using (var subReader = reader.ReadSubtree()) + { + return ParseArtistList(subReader).ToList(); + } + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + return Enumerable.Empty(); + } + } + } + + private IEnumerable ParseArtistList(XmlReader reader) + { + reader.MoveToContent(); + reader.Read(); + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "artist": + { + if (reader.IsEmptyElement) + { + reader.Read(); + continue; + } + var mbzId = reader.GetAttribute("id"); + + using (var subReader = reader.ReadSubtree()) + { + var artist = ParseArtist(subReader, mbzId); + if (artist != null) + { + yield return artist; + } + } + break; + } + default: + { + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + } + + private RemoteSearchResult ParseArtist(XmlReader reader, string artistId) + { + var result = new RemoteSearchResult(); + + reader.MoveToContent(); + reader.Read(); + + // http://stackoverflow.com/questions/2299632/why-does-xmlreader-skip-every-other-element-if-there-is-no-whitespace-separator + + // Loop through each element + while (!reader.EOF && reader.ReadState == ReadState.Interactive) + { + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.Name) + { + case "name": + { + result.Name = reader.ReadElementContentAsString(); + break; + } + case "annotation": + { + result.Overview = reader.ReadElementContentAsString(); + break; + } + default: + { + // there is sort-name if ever needed + reader.Skip(); + break; + } + } + } + else + { + reader.Read(); + } + } + + result.SetProviderId(MetadataProviders.MusicBrainzArtist, artistId); + + if (string.IsNullOrWhiteSpace(artistId) || string.IsNullOrWhiteSpace(result.Name)) + { + return null; + } + + return result; + } + + public async Task> GetMetadata(ArtistInfo id, CancellationToken cancellationToken) + { + var result = new MetadataResult + { + Item = new MusicArtist() + }; + + // TODO maybe remove when artist metadata can be disabled + if (!Plugin.Instance.Configuration.Enable) + { + return result; + } + + var musicBrainzId = id.GetMusicBrainzArtistId(); + + if (string.IsNullOrWhiteSpace(musicBrainzId)) + { + var searchResults = await GetSearchResults(id, cancellationToken).ConfigureAwait(false); + + var singleResult = searchResults.FirstOrDefault(); + + if (singleResult != null) + { + musicBrainzId = singleResult.GetProviderId(MetadataProviders.MusicBrainzArtist); + result.Item.Overview = singleResult.Overview; + + if (Plugin.Instance.Configuration.ReplaceArtistName) + { + result.Item.Name = singleResult.Name; + } + } + } + + if (!string.IsNullOrWhiteSpace(musicBrainzId)) + { + result.HasMetadata = true; + result.Item.SetProviderId(MetadataProviders.MusicBrainzArtist, musicBrainzId); + } + + return result; + } + + /// + /// Determines whether the specified text has diacritics. + /// + /// The text. + /// true if the specified text has diacritics; otherwise, false. + private bool HasDiacritics(string text) + { + return !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal); + } + + /// + /// Encodes an URL. + /// + /// The name. + /// System.String. + private static string UrlEncode(string name) + { + return WebUtility.UrlEncode(name); + } + + public string Name => "MusicBrainz"; + + public Task GetImageResponse(string url, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } +} diff --git a/MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/PluginConfiguration.cs b/MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/PluginConfiguration.cs new file mode 100644 index 000000000..6bce0b447 --- /dev/null +++ b/MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/PluginConfiguration.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using MediaBrowser.Model.Plugins; + +namespace MediaBrowser.Providers.Plugins.MusicBrainz +{ + public class PluginConfiguration : BasePluginConfiguration + { + public bool Enable { get; set; } = false; + + public bool ReplaceArtistName { get; set; } = false; + + public string Server { get; set; } = "https://www.musicbrainz.org"; + + public long RateLimit { get; set; } = 1000u; + } +} diff --git a/MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/config.html b/MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/config.html new file mode 100644 index 000000000..dfffa9065 --- /dev/null +++ b/MediaBrowser.Providers/Plugins/MusicBrainz/Configuration/config.html @@ -0,0 +1,69 @@ + + + + MusicBrainz + + +
+
+
+
+
+ +
This can either be a mirror of the official server or a custom server.
+
+
+ +
Span of time between each request in milliseconds.
+
+ + +
+
+ +
+
+
+
+ +
+ + diff --git a/MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzExternalIds.cs b/MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzExternalIds.cs new file mode 100644 index 000000000..0184d4241 --- /dev/null +++ b/MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzExternalIds.cs @@ -0,0 +1,102 @@ +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Entities; + +namespace MediaBrowser.Providers.Music +{ + public class MusicBrainzReleaseGroupExternalId : IExternalId + { + /// + public string Name => "MusicBrainz Release Group"; + + /// + public string Key => MetadataProviders.MusicBrainzReleaseGroup.ToString(); + + /// + public string UrlFormatString => "https://musicbrainz.org/release-group/{0}"; + + /// + public bool Supports(IHasProviderIds item) + => item is Audio || item is MusicAlbum; + } + + public class MusicBrainzAlbumArtistExternalId : IExternalId + { + /// + public string Name => "MusicBrainz Album Artist"; + + /// + public string Key => MetadataProviders.MusicBrainzAlbumArtist.ToString(); + + /// + public string UrlFormatString => "https://musicbrainz.org/artist/{0}"; + + /// + public bool Supports(IHasProviderIds item) + => item is Audio; + } + + public class MusicBrainzAlbumExternalId : IExternalId + { + /// + public string Name => "MusicBrainz Album"; + + /// + public string Key => MetadataProviders.MusicBrainzAlbum.ToString(); + + /// + public string UrlFormatString => "https://musicbrainz.org/release/{0}"; + + /// + public bool Supports(IHasProviderIds item) + => item is Audio || item is MusicAlbum; + } + + public class MusicBrainzArtistExternalId : IExternalId + { + /// + public string Name => "MusicBrainz"; + + /// + public string Key => MetadataProviders.MusicBrainzArtist.ToString(); + + /// + public string UrlFormatString => "https://musicbrainz.org/artist/{0}"; + + /// + public bool Supports(IHasProviderIds item) => item is MusicArtist; + } + + public class MusicBrainzOtherArtistExternalId : IExternalId + { + /// + public string Name => "MusicBrainz Artist"; + + /// + + public string Key => MetadataProviders.MusicBrainzArtist.ToString(); + + /// + public string UrlFormatString => "https://musicbrainz.org/artist/{0}"; + + /// + public bool Supports(IHasProviderIds item) + => item is Audio || item is MusicAlbum; + } + + public class MusicBrainzTrackId : IExternalId + { + /// + public string Name => "MusicBrainz Track"; + + /// + public string Key => MetadataProviders.MusicBrainzTrack.ToString(); + + /// + public string UrlFormatString => "https://musicbrainz.org/track/{0}"; + + /// + public bool Supports(IHasProviderIds item) => item is Audio; + } +} diff --git a/MediaBrowser.Providers/Plugins/MusicBrainz/Plugin.cs b/MediaBrowser.Providers/Plugins/MusicBrainz/Plugin.cs new file mode 100644 index 000000000..2211d513f --- /dev/null +++ b/MediaBrowser.Providers/Plugins/MusicBrainz/Plugin.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Plugins; +using MediaBrowser.Model.Plugins; +using MediaBrowser.Model.Serialization; + +namespace MediaBrowser.Providers.Plugins.MusicBrainz +{ + public class Plugin : BasePlugin, IHasWebPages + { + public Plugin(IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer) + : base(applicationPaths, xmlSerializer) + { + Instance = this; + } + + public IEnumerable GetPages() + { + yield return new PluginPageInfo + { + Name = Name, + EmbeddedResourcePath = GetType().Namespace + ".Configuration.config.html" + }; + } + + public override Guid Id => new Guid("8c95c4d2-e50c-4fb0-a4f3-6c06ff0f9a1a"); + + public override string Name => "MusicBrainz"; + + public override string Description => "Get artist and album metadata from any MusicBrainz server."; + + public static Plugin Instance { get; private set; } + } +} -- cgit v1.2.3