diff options
65 files changed, 2406 insertions, 360 deletions
diff --git a/.github/ISSUE_TEMPLATE/issue report.yml b/.github/ISSUE_TEMPLATE/issue report.yml index 4f58c5bc5..269968839 100644 --- a/.github/ISSUE_TEMPLATE/issue report.yml +++ b/.github/ISSUE_TEMPLATE/issue report.yml @@ -1,6 +1,7 @@ name: Issue Report description: File an issue report labels: [bug, triage] +type: Bug body: - type: markdown id: introduction @@ -140,7 +141,9 @@ body: - **Reverse Proxy**: [e.g. none, nginx, apache, etc.] - **Base URL**: [e.g. none, yes: /example] - **Networking**: [e.g. Host, Bridge/NAT] - - **Storage**: [e.g. local, NFS, cloud] + - **Jellyfin Data Storage**: [e.g. local SATA SSD, local HDD] + - **Media Storage**: [e.g. Local HDD, SMB Share] + - **External Integrations**: [e.g. Jellystat, Jellyseerr] value: | - OS: - Linux Kernel: @@ -155,7 +158,9 @@ body: - Reverse Proxy: - Base URL: - Networking: - - Storage: + - Jellyfin Data Storage: + - Media Storage: + - External Integrations: render: markdown validations: required: true diff --git a/.github/workflows/ci-codeql-analysis.yml b/.github/workflows/ci-codeql-analysis.yml index 4a042d86d..334a2edcf 100644 --- a/.github/workflows/ci-codeql-analysis.yml +++ b/.github/workflows/ci-codeql-analysis.yml @@ -27,11 +27,11 @@ jobs: dotnet-version: '9.0.x' - name: Initialize CodeQL - uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + uses: github/codeql-action/init@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 with: languages: ${{ matrix.language }} queries: +security-extended - name: Autobuild - uses: github/codeql-action/autobuild@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + uses: github/codeql-action/autobuild@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + uses: github/codeql-action/analyze@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 diff --git a/Directory.Packages.props b/Directory.Packages.props index 7992f61c2..a0a09d745 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,7 +17,7 @@ <PackageVersion Include="Diacritics" Version="3.3.29" /> <PackageVersion Include="DiscUtils.Udf" Version="0.16.13" /> <PackageVersion Include="DotNet.Glob" Version="3.1.3" /> - <PackageVersion Include="FsCheck.Xunit" Version="3.2.0" /> + <PackageVersion Include="FsCheck.Xunit" Version="3.3.0" /> <PackageVersion Include="HarfBuzzSharp.NativeAssets.Linux" Version="8.3.1.1" /> <PackageVersion Include="ICU4N.Transliterator" Version="60.1.0-alpha.356" /> <PackageVersion Include="IDisposableAnalyzers" Version="4.0.8" /> @@ -49,8 +49,9 @@ <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.5" /> <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.5" /> <PackageVersion Include="Microsoft.Extensions.Options" Version="9.0.5" /> - <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.14.0" /> + <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.14.1" /> <PackageVersion Include="MimeTypes" Version="2.5.2" /> + <PackageVersion Include="Morestachio" Version="5.0.1.631" /> <PackageVersion Include="Moq" Version="4.18.4" /> <PackageVersion Include="NEbml" Version="0.12.0" /> <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" /> @@ -58,6 +59,7 @@ <PackageVersion Include="prometheus-net.AspNetCore" Version="8.2.1" /> <PackageVersion Include="prometheus-net.DotNetRuntime" Version="4.4.1" /> <PackageVersion Include="prometheus-net" Version="8.2.1" /> + <PackageVersion Include="Polly" Version="8.5.2" /> <PackageVersion Include="Serilog.AspNetCore" Version="9.0.0" /> <PackageVersion Include="Serilog.Enrichers.Thread" Version="4.0.0" /> <PackageVersion Include="Serilog.Settings.Configuration" Version="9.0.0" /> diff --git a/Emby.Server.Implementations/Localization/Core/de.json b/Emby.Server.Implementations/Localization/Core/de.json index f5ae43bb5..a411e27e1 100644 --- a/Emby.Server.Implementations/Localization/Core/de.json +++ b/Emby.Server.Implementations/Localization/Core/de.json @@ -77,7 +77,7 @@ "SubtitleDownloadFailureFromForItem": "Untertitel von {0} für {1} konnten nicht heruntergeladen werden", "Sync": "Synchronisation", "System": "System", - "TvShows": "TV-Serien", + "TvShows": "Serien", "User": "Benutzer", "UserCreatedWithName": "Benutzer {0} wurde erstellt", "UserDeletedWithName": "Benutzer {0} wurde gelöscht", diff --git a/Emby.Server.Implementations/Localization/Core/pt.json b/Emby.Server.Implementations/Localization/Core/pt.json index edfefe84a..52427f24b 100644 --- a/Emby.Server.Implementations/Localization/Core/pt.json +++ b/Emby.Server.Implementations/Localization/Core/pt.json @@ -80,7 +80,7 @@ "Sync": "Sincronização", "Songs": "Músicas", "Shows": "Séries", - "Playlists": "Listas de Reprodução", + "Playlists": "Playlists", "Photos": "Fotografias", "Movies": "Filmes", "FailedLoginAttemptWithUserName": "Tentativa de início de sessão falhada a partir de {0}", diff --git a/Emby.Server.Implementations/Session/SessionManager.cs b/Emby.Server.Implementations/Session/SessionManager.cs index 924f50286..8cbd957a8 100644 --- a/Emby.Server.Implementations/Session/SessionManager.cs +++ b/Emby.Server.Implementations/Session/SessionManager.cs @@ -508,13 +508,11 @@ namespace Emby.Server.Implementations.Session ArgumentException.ThrowIfNullOrEmpty(deviceId); var key = GetSessionKey(appName, deviceId); - - CheckDisposed(); - - if (!_activeConnections.TryGetValue(key, out var sessionInfo)) + SessionInfo newSession = CreateSessionInfo(key, appName, appVersion, deviceId, deviceName, remoteEndPoint, user); + SessionInfo sessionInfo = _activeConnections.GetOrAdd(key, newSession); + if (ReferenceEquals(newSession, sessionInfo)) { - sessionInfo = CreateSession(key, appName, appVersion, deviceId, deviceName, remoteEndPoint, user); - _activeConnections[key] = sessionInfo; + OnSessionStarted(newSession); } sessionInfo.UserId = user?.Id ?? Guid.Empty; @@ -538,7 +536,7 @@ namespace Emby.Server.Implementations.Session return sessionInfo; } - private SessionInfo CreateSession( + private SessionInfo CreateSessionInfo( string key, string appName, string appVersion, @@ -582,7 +580,6 @@ namespace Emby.Server.Implementations.Session sessionInfo.HasCustomDeviceName = true; } - OnSessionStarted(sessionInfo); return sessionInfo; } diff --git a/Jellyfin.Api/Controllers/PlaylistsController.cs b/Jellyfin.Api/Controllers/PlaylistsController.cs index ec5fdab38..79c71d23a 100644 --- a/Jellyfin.Api/Controllers/PlaylistsController.cs +++ b/Jellyfin.Api/Controllers/PlaylistsController.cs @@ -450,22 +450,41 @@ public class PlaylistsController : BaseJellyfinApiController { var callingUserId = User.GetUserId(); - var playlist = _playlistManager.GetPlaylistForUser(Guid.Parse(playlistId), callingUserId); - if (playlist is null) + if (!callingUserId.IsEmpty()) { - return NotFound("Playlist not found"); + var playlist = _playlistManager.GetPlaylistForUser(Guid.Parse(playlistId), callingUserId); + if (playlist is null) + { + return NotFound("Playlist not found"); + } + + var isPermitted = playlist.OwnerUserId.Equals(callingUserId) + || playlist.Shares.Any(s => s.CanEdit && s.UserId.Equals(callingUserId)); + + if (!isPermitted) + { + return Forbid(); + } } + else + { + var isApiKey = User.GetIsApiKey(); - var isPermitted = playlist.OwnerUserId.Equals(callingUserId) - || playlist.Shares.Any(s => s.CanEdit && s.UserId.Equals(callingUserId)); + if (!isApiKey) + { + return Forbid(); + } + } - if (!isPermitted) + try { - return Forbid(); + await _playlistManager.RemoveItemFromPlaylistAsync(playlistId, entryIds).ConfigureAwait(false); + return NoContent(); + } + catch (ArgumentException) + { + return NotFound(); } - - await _playlistManager.RemoveItemFromPlaylistAsync(playlistId, entryIds).ConfigureAwait(false); - return NoContent(); } /// <summary> diff --git a/Jellyfin.Api/Middleware/IpBasedAccessValidationMiddleware.cs b/Jellyfin.Api/Middleware/IpBasedAccessValidationMiddleware.cs index 842a69dd9..a0ed6c812 100644 --- a/Jellyfin.Api/Middleware/IpBasedAccessValidationMiddleware.cs +++ b/Jellyfin.Api/Middleware/IpBasedAccessValidationMiddleware.cs @@ -1,8 +1,10 @@ using System.Net; using System.Threading.Tasks; +using System.Web; using MediaBrowser.Common.Extensions; using MediaBrowser.Common.Net; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; namespace Jellyfin.Api.Middleware; @@ -12,14 +14,17 @@ namespace Jellyfin.Api.Middleware; public class IPBasedAccessValidationMiddleware { private readonly RequestDelegate _next; + private readonly ILogger<IPBasedAccessValidationMiddleware> _logger; /// <summary> /// Initializes a new instance of the <see cref="IPBasedAccessValidationMiddleware"/> class. /// </summary> /// <param name="next">The next delegate in the pipeline.</param> - public IPBasedAccessValidationMiddleware(RequestDelegate next) + /// <param name="logger">The logger to log to.</param> + public IPBasedAccessValidationMiddleware(RequestDelegate next, ILogger<IPBasedAccessValidationMiddleware> logger) { _next = next; + _logger = logger; } /// <summary> @@ -32,16 +37,23 @@ public class IPBasedAccessValidationMiddleware { if (httpContext.IsLocal()) { - // Running locally. + // Accessing from the same machine as the server. await _next(httpContext).ConfigureAwait(false); return; } - var remoteIP = httpContext.Connection.RemoteIpAddress ?? IPAddress.Loopback; + var remoteIP = httpContext.GetNormalizedRemoteIP(); - if (!networkManager.HasRemoteAccess(remoteIP)) + var result = networkManager.ShouldAllowServerAccess(remoteIP); + if (result != RemoteAccessPolicyResult.Allow) { // No access from network, respond with 503 instead of 200. + _logger.LogWarning( + "Blocking request to {Path} by {RemoteIP} due to IP filtering rule, reason: {Reason}", + // url-encode to block log injection + HttpUtility.UrlEncode(httpContext.Request.Path), + remoteIP, + result); httpContext.Response.StatusCode = StatusCodes.Status503ServiceUnavailable; return; } diff --git a/Jellyfin.Api/Middleware/LanFilteringMiddleware.cs b/Jellyfin.Api/Middleware/LanFilteringMiddleware.cs deleted file mode 100644 index 35b0a1dd0..000000000 --- a/Jellyfin.Api/Middleware/LanFilteringMiddleware.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.Net; -using System.Threading.Tasks; -using MediaBrowser.Common.Extensions; -using MediaBrowser.Common.Net; -using MediaBrowser.Controller.Configuration; -using Microsoft.AspNetCore.Http; - -namespace Jellyfin.Api.Middleware; - -/// <summary> -/// Validates the LAN host IP based on application configuration. -/// </summary> -public class LanFilteringMiddleware -{ - private readonly RequestDelegate _next; - - /// <summary> - /// Initializes a new instance of the <see cref="LanFilteringMiddleware"/> class. - /// </summary> - /// <param name="next">The next delegate in the pipeline.</param> - public LanFilteringMiddleware(RequestDelegate next) - { - _next = next; - } - - /// <summary> - /// Executes the middleware action. - /// </summary> - /// <param name="httpContext">The current HTTP context.</param> - /// <param name="networkManager">The network manager.</param> - /// <param name="serverConfigurationManager">The server configuration manager.</param> - /// <returns>The async task.</returns> - public async Task Invoke(HttpContext httpContext, INetworkManager networkManager, IServerConfigurationManager serverConfigurationManager) - { - if (serverConfigurationManager.GetNetworkConfiguration().EnableRemoteAccess) - { - await _next(httpContext).ConfigureAwait(false); - return; - } - - var host = httpContext.GetNormalizedRemoteIP(); - if (!networkManager.IsInLocalNetwork(host)) - { - // No access from network, respond with 503 instead of 200. - httpContext.Response.StatusCode = StatusCodes.Status503ServiceUnavailable; - return; - } - - await _next(httpContext).ConfigureAwait(false); - } -} diff --git a/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs b/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs index fbbb5bca7..63c80634f 100644 --- a/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs +++ b/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Reflection; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.DbConfiguration; +using Jellyfin.Database.Implementations.Locking; using Jellyfin.Database.Providers.Sqlite; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Configuration; @@ -41,6 +44,28 @@ public static class ServiceCollectionExtensions return items; } + private static JellyfinDbProviderFactory? LoadDatabasePlugin(CustomDatabaseOptions customProviderOptions, IApplicationPaths applicationPaths) + { + var plugin = Directory.EnumerateDirectories(applicationPaths.PluginsPath) + .Where(e => Path.GetFileName(e)!.StartsWith(customProviderOptions.PluginName, StringComparison.OrdinalIgnoreCase)) + .Order() + .FirstOrDefault() + ?? throw new InvalidOperationException($"The requested custom database plugin with the name '{customProviderOptions.PluginName}' could not been found in '{applicationPaths.PluginsPath}'"); + + var dbProviderAssembly = Path.Combine(plugin, Path.ChangeExtension(customProviderOptions.PluginAssembly, "dll")); + if (!File.Exists(dbProviderAssembly)) + { + throw new InvalidOperationException($"Could not find the requested assembly at '{dbProviderAssembly}'"); + } + + // we have to load the assembly without proxy to ensure maximum performance for this. + var assembly = Assembly.LoadFrom(dbProviderAssembly); + var dbProviderType = assembly.GetExportedTypes().FirstOrDefault(f => f.IsAssignableTo(typeof(IJellyfinDatabaseProvider))) + ?? throw new InvalidOperationException($"Could not find any type implementing the '{nameof(IJellyfinDatabaseProvider)}' interface."); + + return (services) => (IJellyfinDatabaseProvider)ActivatorUtilities.CreateInstance(services, dbProviderType); + } + /// <summary> /// Adds the <see cref="IDbContextFactory{TContext}"/> interface to the service collection with second level caching enabled. /// </summary> @@ -54,7 +79,6 @@ public static class ServiceCollectionExtensions IConfiguration configuration) { var efCoreConfiguration = configurationManager.GetConfiguration<DatabaseConfigurationOptions>("database"); - var providers = GetSupportedDbProviders(); JellyfinDbProviderFactory? providerFactory = null; if (efCoreConfiguration?.DatabaseType is null) @@ -73,22 +97,51 @@ public static class ServiceCollectionExtensions efCoreConfiguration = new DatabaseConfigurationOptions() { DatabaseType = "Jellyfin-SQLite", + LockingBehavior = DatabaseLockingBehaviorTypes.NoLock }; configurationManager.SaveConfiguration("database", efCoreConfiguration); } } - if (!providers.TryGetValue(efCoreConfiguration.DatabaseType.ToUpperInvariant(), out providerFactory!)) + if (efCoreConfiguration.DatabaseType.Equals("PLUGIN_PROVIDER", StringComparison.OrdinalIgnoreCase)) + { + if (efCoreConfiguration.CustomProviderOptions is null) + { + throw new InvalidOperationException("The custom database provider must declare the custom provider options to work"); + } + + providerFactory = LoadDatabasePlugin(efCoreConfiguration.CustomProviderOptions, configurationManager.ApplicationPaths); + } + else { - throw new InvalidOperationException($"Jellyfin cannot find the database provider of type '{efCoreConfiguration.DatabaseType}'. Supported types are {string.Join(", ", providers.Keys)}"); + var providers = GetSupportedDbProviders(); + if (!providers.TryGetValue(efCoreConfiguration.DatabaseType.ToUpperInvariant(), out providerFactory!)) + { + throw new InvalidOperationException($"Jellyfin cannot find the database provider of type '{efCoreConfiguration.DatabaseType}'. Supported types are {string.Join(", ", providers.Keys)}"); + } } serviceCollection.AddSingleton<IJellyfinDatabaseProvider>(providerFactory!); + switch (efCoreConfiguration.LockingBehavior) + { + case DatabaseLockingBehaviorTypes.NoLock: + serviceCollection.AddSingleton<IEntityFrameworkCoreLockingBehavior, NoLockBehavior>(); + break; + case DatabaseLockingBehaviorTypes.Pessimistic: + serviceCollection.AddSingleton<IEntityFrameworkCoreLockingBehavior, PessimisticLockBehavior>(); + break; + case DatabaseLockingBehaviorTypes.Optimistic: + serviceCollection.AddSingleton<IEntityFrameworkCoreLockingBehavior, OptimisticLockBehavior>(); + break; + } + serviceCollection.AddPooledDbContextFactory<JellyfinDbContext>((serviceProvider, opt) => { var provider = serviceProvider.GetRequiredService<IJellyfinDatabaseProvider>(); provider.Initialise(opt); + var lockingBehavior = serviceProvider.GetRequiredService<IEntityFrameworkCoreLockingBehavior>(); + lockingBehavior.Initialise(opt); }); return serviceCollection; diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs index 706f009ac..8bd108c44 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs @@ -10,4 +10,6 @@ internal class BackupOptions public bool Trickplay { get; set; } public bool Subtitles { get; set; } + + public bool Database { get; set; } } diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index c3f5b0103..e266d5a3b 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -14,6 +14,9 @@ using Jellyfin.Server.Implementations.SystemBackupService; using MediaBrowser.Controller; using MediaBrowser.Controller.SystemBackupService; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Implementations.FullSystemBackup; @@ -29,6 +32,7 @@ public class BackupService : IBackupService private readonly IServerApplicationHost _applicationHost; private readonly IServerApplicationPaths _applicationPaths; private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; + private readonly IHostApplicationLifetime _hostApplicationLifetime; private static readonly JsonSerializerOptions _serializerSettings = new JsonSerializerOptions(JsonSerializerDefaults.General) { AllowTrailingCommas = true, @@ -45,18 +49,21 @@ public class BackupService : IBackupService /// <param name="applicationHost">The Application host.</param> /// <param name="applicationPaths">The application paths.</param> /// <param name="jellyfinDatabaseProvider">The Jellyfin database Provider in use.</param> + /// <param name="applicationLifetime">The SystemManager.</param> public BackupService( ILogger<BackupService> logger, IDbContextFactory<JellyfinDbContext> dbProvider, IServerApplicationHost applicationHost, IServerApplicationPaths applicationPaths, - IJellyfinDatabaseProvider jellyfinDatabaseProvider) + IJellyfinDatabaseProvider jellyfinDatabaseProvider, + IHostApplicationLifetime applicationLifetime) { _logger = logger; _dbProvider = dbProvider; _applicationHost = applicationHost; _applicationPaths = applicationPaths; _jellyfinDatabaseProvider = jellyfinDatabaseProvider; + _hostApplicationLifetime = applicationLifetime; } /// <inheritdoc/> @@ -65,6 +72,11 @@ public class BackupService : IBackupService _applicationHost.RestoreBackupPath = archivePath; _applicationHost.ShouldRestart = true; _applicationHost.NotifyPendingRestart(); + _ = Task.Run(async () => + { + await Task.Delay(500).ConfigureAwait(false); + _hostApplicationLifetime.StopApplication(); + }); } /// <inheritdoc/> @@ -129,63 +141,90 @@ public class BackupService : IBackupService CopyDirectory(_applicationPaths.DataPath, "Data/"); CopyDirectory(_applicationPaths.RootFolderPath, "Root/"); - _logger.LogInformation("Begin restoring Database"); - var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); - await using (dbContext.ConfigureAwait(false)) + if (manifest.Options.Database) { - dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; - var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) - .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) - .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) - .ToArray(); - - var tableNames = entityTypes.Select(f => dbContext.Model.FindEntityType(f.Type.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!); - _logger.LogInformation("Begin purging database"); - await _jellyfinDatabaseProvider.PurgeDatabase(dbContext, tableNames).ConfigureAwait(false); - _logger.LogInformation("Database Purged"); - - foreach (var entityType in entityTypes) + _logger.LogInformation("Begin restoring Database"); + var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) { - _logger.LogInformation("Read backup of {Table}", entityType.Type.Name); + // restore migration history manually + var historyEntry = zipArchive.GetEntry($"Database\\{nameof(HistoryRow)}.json"); + if (historyEntry is null) + { + _logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin operation"); + throw new InvalidOperationException("Cannot restore backup that has no History data."); + } - var zipEntry = zipArchive.GetEntry($"Database\\{entityType.Type.Name}.json"); - if (zipEntry is null) + HistoryRow[] historyEntries; + var historyArchive = historyEntry.Open(); + await using (historyArchive.ConfigureAwait(false)) { - _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); - continue; + historyEntries = await JsonSerializer.DeserializeAsync<HistoryRow[]>(historyArchive).ConfigureAwait(false) ?? + throw new InvalidOperationException("Cannot restore backup that has no History data."); } - var zipEntryStream = zipEntry.Open(); - await using (zipEntryStream.ConfigureAwait(false)) + var historyRepository = dbContext.GetService<IHistoryRepository>(); + await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); + foreach (var item in historyEntries) { - _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name); - var records = 0; - await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable<JsonObject>(zipEntryStream, _serializerSettings).ConfigureAwait(false)!) + var insertScript = historyRepository.GetInsertScript(item); + await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false); + } + + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) + .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) + .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) + .ToArray(); + + var tableNames = entityTypes.Select(f => dbContext.Model.FindEntityType(f.Type.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!); + _logger.LogInformation("Begin purging database"); + await _jellyfinDatabaseProvider.PurgeDatabase(dbContext, tableNames).ConfigureAwait(false); + _logger.LogInformation("Database Purged"); + + foreach (var entityType in entityTypes) + { + _logger.LogInformation("Read backup of {Table}", entityType.Type.Name); + + var zipEntry = zipArchive.GetEntry($"Database\\{entityType.Type.Name}.json"); + if (zipEntry is null) { - var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]); - if (entity is null) - { - throw new InvalidOperationException($"Cannot deserialize entity '{item}'"); - } + _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); + continue; + } - try - { - records++; - dbContext.Add(entity); - } - catch (Exception ex) + var zipEntryStream = zipEntry.Open(); + await using (zipEntryStream.ConfigureAwait(false)) + { + _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name); + var records = 0; + await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable<JsonObject>(zipEntryStream, _serializerSettings).ConfigureAwait(false)!) { - _logger.LogError(ex, "Could not store entity {Entity} continue anyway.", item); + var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]); + if (entity is null) + { + throw new InvalidOperationException($"Cannot deserialize entity '{item}'"); + } + + try + { + records++; + dbContext.Add(entity); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not store entity {Entity} continue anyway.", item); + } } - } - _logger.LogInformation("Prepared to restore {Number} entries for {Table}", records, entityType.Type.Name); + _logger.LogInformation("Prepared to restore {Number} entries for {Table}", records, entityType.Type.Name); + } } - } - _logger.LogInformation("Try restore Database"); - await dbContext.SaveChangesAsync().ConfigureAwait(false); - _logger.LogInformation("Restored database."); + _logger.LogInformation("Try restore Database"); + await dbContext.SaveChangesAsync().ConfigureAwait(false); + _logger.LogInformation("Restored database."); + } } _logger.LogInformation("Restored Jellyfin system from {Date}.", manifest.DateCreated); @@ -242,22 +281,30 @@ public class BackupService : IBackupService await using (dbContext.ConfigureAwait(false)) { dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; - var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) + static IAsyncEnumerable<object> GetValues(IQueryable dbSet, Type type) + { + var method = dbSet.GetType().GetMethod(nameof(DbSet<object>.AsAsyncEnumerable))!; + var enumerable = method.Invoke(dbSet, null)!; + return (IAsyncEnumerable<object>)enumerable; + } + + // include the migration history as well + var historyRepository = dbContext.GetService<IHistoryRepository>(); + var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); + + ICollection<(Type Type, Func<IAsyncEnumerable<object>> ValueFactory)> entityTypes = [ + .. typeof(JellyfinDbContext) + .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) - .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) - .ToArray(); + .Select(e => (Type: e.PropertyType, ValueFactory: new Func<IAsyncEnumerable<object>>(() => GetValues((IQueryable)e.GetValue(dbContext)!, e.PropertyType)))), + (Type: typeof(HistoryRow), ValueFactory: new Func<IAsyncEnumerable<object>>(() => migrations.ToAsyncEnumerable())) + ]; manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray(); var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false); await using (transaction.ConfigureAwait(false)) { _logger.LogInformation("Begin Database backup"); - static IAsyncEnumerable<object> GetValues(IQueryable dbSet, Type type) - { - var method = dbSet.GetType().GetMethod(nameof(DbSet<object>.AsAsyncEnumerable))!; - var enumerable = method.Invoke(dbSet, null)!; - return (IAsyncEnumerable<object>)enumerable; - } foreach (var entityType in entityTypes) { @@ -272,7 +319,7 @@ public class BackupService : IBackupService { jsonSerializer.WriteStartArray(); - var set = GetValues(entityType.Set!, entityType.Type.PropertyType).ConfigureAwait(false); + var set = entityType.ValueFactory().ConfigureAwait(false); await foreach (var item in set.ConfigureAwait(false)) { entities++; @@ -447,7 +494,8 @@ public class BackupService : IBackupService { Metadata = options.Metadata, Subtitles = options.Subtitles, - Trickplay = options.Trickplay + Trickplay = options.Trickplay, + Database = options.Database }; } @@ -457,7 +505,8 @@ public class BackupService : IBackupService { Metadata = options.Metadata, Subtitles = options.Subtitles, - Trickplay = options.Trickplay + Trickplay = options.Trickplay, + Database = options.Database }; } } diff --git a/Jellyfin.Server.Implementations/Item/ChapterRepository.cs b/Jellyfin.Server.Implementations/Item/ChapterRepository.cs index 9f2d47346..e0d23a261 100644 --- a/Jellyfin.Server.Implementations/Item/ChapterRepository.cs +++ b/Jellyfin.Server.Implementations/Item/ChapterRepository.cs @@ -112,7 +112,12 @@ public class ChapterRepository : IChapterRepository ImagePath = chapterInfo.ImagePath, Name = chapterInfo.Name, }; - chapterEntity.ImageTag = _imageProcessor.GetImageCacheTag(baseItemPath, chapterEntity.ImageDateModified); + + if (!string.IsNullOrEmpty(chapterInfo.ImagePath)) + { + chapterEntity.ImageTag = _imageProcessor.GetImageCacheTag(baseItemPath, chapterEntity.ImageDateModified); + } + return chapterEntity; } } diff --git a/Jellyfin.Server.Implementations/Item/PeopleRepository.cs b/Jellyfin.Server.Implementations/Item/PeopleRepository.cs index 4e898119b..be58e2a52 100644 --- a/Jellyfin.Server.Implementations/Item/PeopleRepository.cs +++ b/Jellyfin.Server.Implementations/Item/PeopleRepository.cs @@ -54,7 +54,7 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I public IReadOnlyList<string> GetPeopleNames(InternalPeopleQuery filter) { using var context = _dbProvider.CreateDbContext(); - var dbQuery = TranslateQuery(context.Peoples.AsNoTracking(), context, filter); + var dbQuery = TranslateQuery(context.Peoples.AsNoTracking(), context, filter).Select(e => e.Name).Distinct(); // dbQuery = dbQuery.OrderBy(e => e.ListOrder); if (filter.Limit > 0) @@ -62,7 +62,7 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I dbQuery = dbQuery.Take(filter.Limit); } - return dbQuery.Select(e => e.Name).ToArray(); + return dbQuery.ToArray(); } /// <inheritdoc /> @@ -141,8 +141,13 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I if (filter.User is not null && filter.IsFavorite.HasValue) { var personType = itemTypeLookup.BaseItemKindNames[BaseItemKind.Person]; - query = query - .Where(e => context.BaseItems.Any(b => b.Type == personType && b.Name == e.Name && b.UserData!.Any(u => u.IsFavorite == filter.IsFavorite && u.UserId.Equals(filter.User.Id)))); + var oldQuery = query; + + query = context.UserData + .Where(u => u.Item!.Type == personType && u.IsFavorite == filter.IsFavorite && u.UserId.Equals(filter.User.Id)) + .Join(oldQuery, e => e.Item!.Name, e => e.Name, (item, person) => person) + .Distinct() + .AsNoTracking(); } if (!filter.ItemId.IsEmpty()) diff --git a/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs b/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs index b2df86244..6f2d2a107 100644 --- a/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs +++ b/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; using AsyncKeyedLock; +using J2N.Collections.Generic.Extensions; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using MediaBrowser.Common.Configuration; @@ -80,7 +81,7 @@ public class TrickplayManager : ITrickplayManager public async Task MoveGeneratedTrickplayDataAsync(Video video, LibraryOptions libraryOptions, CancellationToken cancellationToken) { var options = _config.Configuration.TrickplayOptions; - if (!CanGenerateTrickplay(video, options.Interval, libraryOptions)) + if (libraryOptions is null || !libraryOptions.EnableTrickplayImageExtraction || !CanGenerateTrickplay(video, options.Interval)) { return; } @@ -137,25 +138,87 @@ public class TrickplayManager : ITrickplayManager /// <inheritdoc /> public async Task RefreshTrickplayDataAsync(Video video, bool replace, LibraryOptions libraryOptions, CancellationToken cancellationToken) { - _logger.LogDebug("Trickplay refresh for {ItemId} (replace existing: {Replace})", video.Id, replace); - var options = _config.Configuration.TrickplayOptions; - if (options.Interval < 1000) + if (!CanGenerateTrickplay(video, options.Interval) || libraryOptions is null) { - _logger.LogWarning("Trickplay image interval {Interval} is too small, reset to the minimum valid value of 1000", options.Interval); - options.Interval = 1000; + return; } - foreach (var width in options.WidthResolutions) + var dbContext = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) { - cancellationToken.ThrowIfCancellationRequested(); - await RefreshTrickplayDataInternal( - video, - replace, - width, - options, - libraryOptions, - cancellationToken).ConfigureAwait(false); + var saveWithMedia = libraryOptions.SaveTrickplayWithMedia; + var trickplayDirectory = _pathManager.GetTrickplayDirectory(video, saveWithMedia); + if (!libraryOptions.EnableTrickplayImageExtraction || replace) + { + // Prune existing data + if (Directory.Exists(trickplayDirectory)) + { + try + { + Directory.Delete(trickplayDirectory, true); + } + catch (Exception ex) + { + _logger.LogWarning("Unable to clear trickplay directory: {Directory}: {Exception}", trickplayDirectory, ex); + } + } + + await dbContext.TrickplayInfos + .Where(i => i.ItemId.Equals(video.Id)) + .ExecuteDeleteAsync(cancellationToken) + .ConfigureAwait(false); + + if (!replace) + { + return; + } + } + + _logger.LogDebug("Trickplay refresh for {ItemId} (replace existing: {Replace})", video.Id, replace); + + if (options.Interval < 1000) + { + _logger.LogWarning("Trickplay image interval {Interval} is too small, reset to the minimum valid value of 1000", options.Interval); + options.Interval = 1000; + } + + foreach (var width in options.WidthResolutions) + { + cancellationToken.ThrowIfCancellationRequested(); + await RefreshTrickplayDataInternal( + video, + replace, + width, + options, + saveWithMedia, + cancellationToken).ConfigureAwait(false); + } + + // Cleanup old trickplay files + if (Directory.Exists(trickplayDirectory)) + { + var existingFolders = Directory.GetDirectories(trickplayDirectory).ToList(); + var trickplayInfos = await dbContext.TrickplayInfos + .AsNoTracking() + .Where(i => i.ItemId.Equals(video.Id)) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + var expectedFolders = trickplayInfos.Select(i => GetTrickplayDirectory(video, i.TileWidth, i.TileHeight, i.Width, saveWithMedia)).ToList(); + var foldersToRemove = existingFolders.Except(expectedFolders); + foreach (var folder in foldersToRemove) + { + try + { + _logger.LogWarning("Pruning trickplay files for {Item}", video.Path); + Directory.Delete(folder, true); + } + catch (Exception ex) + { + _logger.LogWarning("Unable to remove trickplay directory: {Directory}: {Exception}", folder, ex); + } + } + } } } @@ -164,14 +227,9 @@ public class TrickplayManager : ITrickplayManager bool replace, int width, TrickplayOptions options, - LibraryOptions libraryOptions, + bool saveWithMedia, CancellationToken cancellationToken) { - if (!CanGenerateTrickplay(video, options.Interval, libraryOptions)) - { - return; - } - var imgTempDir = string.Empty; using (await _resourcePool.LockAsync(cancellationToken).ConfigureAwait(false)) @@ -215,7 +273,6 @@ public class TrickplayManager : ITrickplayManager var tileWidth = options.TileWidth; var tileHeight = options.TileHeight; - var saveWithMedia = libraryOptions is not null && libraryOptions.SaveTrickplayWithMedia; var outputDir = new DirectoryInfo(GetTrickplayDirectory(video, tileWidth, tileHeight, actualWidth, saveWithMedia)); // Import existing trickplay tiles @@ -402,7 +459,7 @@ public class TrickplayManager : ITrickplayManager return trickplayInfo; } - private bool CanGenerateTrickplay(Video video, int interval, LibraryOptions libraryOptions) + private bool CanGenerateTrickplay(Video video, int interval) { var videoType = video.VideoType; if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay) @@ -430,11 +487,6 @@ public class TrickplayManager : ITrickplayManager return false; } - if (libraryOptions is null || !libraryOptions.EnableTrickplayImageExtraction) - { - return false; - } - // Can't extract images if there are no video streams return video.GetMediaStreams().Count > 0; } diff --git a/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs b/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs index 6066893de..a56baba33 100644 --- a/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs +++ b/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs @@ -69,16 +69,6 @@ namespace Jellyfin.Server.Extensions } /// <summary> - /// Adds LAN based access filtering to the application pipeline. - /// </summary> - /// <param name="appBuilder">The application builder.</param> - /// <returns>The updated application builder.</returns> - public static IApplicationBuilder UseLanFiltering(this IApplicationBuilder appBuilder) - { - return appBuilder.UseMiddleware<LanFilteringMiddleware>(); - } - - /// <summary> /// Enables url decoding before binding to the application pipeline. /// </summary> /// <param name="appBuilder">The <see cref="IApplicationBuilder"/>.</param> diff --git a/Jellyfin.Server/Helpers/StartupHelpers.cs b/Jellyfin.Server/Helpers/StartupHelpers.cs index bbf6d31f1..93c996166 100644 --- a/Jellyfin.Server/Helpers/StartupHelpers.cs +++ b/Jellyfin.Server/Helpers/StartupHelpers.cs @@ -3,18 +3,19 @@ using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; -using System.Net; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Text; using System.Threading.Tasks; using Emby.Server.Implementations; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Extensions; using MediaBrowser.Model.IO; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Serilog; +using Serilog.Extensions.Logging; using ILogger = Microsoft.Extensions.Logging.ILogger; namespace Jellyfin.Server.Helpers; @@ -257,11 +258,14 @@ public static class StartupHelpers { try { + var startupLogger = new LoggerProviderCollection(); + startupLogger.AddProvider(new SetupServer.SetupLoggerFactory()); // Serilog.Log is used by SerilogLoggerFactory when no logger is specified Log.Logger = new LoggerConfiguration() .ReadFrom.Configuration(configuration) .Enrich.FromLogContext() .Enrich.WithThreadId() + .WriteTo.Async(e => e.Providers(startupLogger)) .CreateLogger(); } catch (Exception ex) diff --git a/Jellyfin.Server/Jellyfin.Server.csproj b/Jellyfin.Server/Jellyfin.Server.csproj index 452b03efb..14c4285fe 100644 --- a/Jellyfin.Server/Jellyfin.Server.csproj +++ b/Jellyfin.Server/Jellyfin.Server.csproj @@ -48,6 +48,7 @@ <PackageReference Include="Microsoft.Extensions.Configuration.Json" /> <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" /> <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" /> + <PackageReference Include="Morestachio" /> <PackageReference Include="prometheus-net" /> <PackageReference Include="prometheus-net.AspNetCore" /> <PackageReference Include="Serilog.AspNetCore" /> @@ -79,6 +80,9 @@ <None Update="wwwroot\api-docs\banner-dark.svg"> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> </None> + <None Update="ServerSetupApp/index.mstemplate.html"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> </ItemGroup> </Project> diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs b/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs index 5c8322ef7..70e54125b 100644 --- a/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs +++ b/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs @@ -47,9 +47,9 @@ public sealed class JellyfinMigrationAttribute : Attribute public bool RunMigrationOnSetup { get; set; } /// <summary> - /// Gets or Sets the stage the annoated migration should be executed at. Defaults to <see cref="JellyfinMigrationStageTypes.CoreInitialisaition"/>. + /// Gets or Sets the stage the annoated migration should be executed at. Defaults to <see cref="JellyfinMigrationStageTypes.CoreInitialisation"/>. /// </summary> - public JellyfinMigrationStageTypes Stage { get; set; } = JellyfinMigrationStageTypes.CoreInitialisaition; + public JellyfinMigrationStageTypes Stage { get; set; } = JellyfinMigrationStageTypes.CoreInitialisation; /// <summary> /// Gets the ordering of the migration. diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs b/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs new file mode 100644 index 000000000..6c8da7e82 --- /dev/null +++ b/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs @@ -0,0 +1,35 @@ +using System; + +namespace Jellyfin.Server.Migrations; + +/// <summary> +/// Marks an <see cref="JellyfinMigrationAttribute"/> migration and instructs the <see cref="JellyfinMigrationService"/> to perform a backup. +/// </summary> +[AttributeUsage(System.AttributeTargets.Class, Inherited = true, AllowMultiple = true)] +public sealed class JellyfinMigrationBackupAttribute : System.Attribute +{ + /// <summary> + /// Gets or Sets a value indicating whether a backup of the old library.db should be performed. + /// </summary> + public bool LegacyLibraryDb { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the Database should be performed. + /// </summary> + public bool JellyfinDb { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the metadata folder should be performed. + /// </summary> + public bool Metadata { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the Trickplay folder should be performed. + /// </summary> + public bool Trickplay { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the Subtitles folder should be performed. + /// </summary> + public bool Subtitles { get; set; } +} diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationService.cs b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs index 3d6ed73bc..5331b43e3 100644 --- a/Jellyfin.Server/Migrations/JellyfinMigrationService.cs +++ b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -9,13 +10,13 @@ using Emby.Server.Implementations.Serialization; using Jellyfin.Database.Implementations; using Jellyfin.Server.Implementations.SystemBackupService; using Jellyfin.Server.Migrations.Stages; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.SystemBackupService; using MediaBrowser.Model.Configuration; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations; @@ -25,21 +26,41 @@ namespace Jellyfin.Server.Migrations; /// </summary> internal class JellyfinMigrationService { + private const string DbFilename = "library.db"; private readonly IDbContextFactory<JellyfinDbContext> _dbContextFactory; private readonly ILoggerFactory _loggerFactory; + private readonly IStartupLogger _startupLogger; + private readonly IBackupService? _backupService; + private readonly IJellyfinDatabaseProvider? _jellyfinDatabaseProvider; + private readonly IApplicationPaths _applicationPaths; + private (string? LibraryDb, string? JellyfinDb, BackupManifestDto? FullBackup) _backupKey; /// <summary> /// Initializes a new instance of the <see cref="JellyfinMigrationService"/> class. /// </summary> /// <param name="dbContextFactory">Provides access to the jellyfin database.</param> /// <param name="loggerFactory">The logger factory.</param> - public JellyfinMigrationService(IDbContextFactory<JellyfinDbContext> dbContextFactory, ILoggerFactory loggerFactory) + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="applicationPaths">Application paths for library.db backup.</param> + /// <param name="backupService">The jellyfin backup service.</param> + /// <param name="jellyfinDatabaseProvider">The jellyfin database provider.</param> + public JellyfinMigrationService( + IDbContextFactory<JellyfinDbContext> dbContextFactory, + ILoggerFactory loggerFactory, + IStartupLogger startupLogger, + IApplicationPaths applicationPaths, + IBackupService? backupService = null, + IJellyfinDatabaseProvider? jellyfinDatabaseProvider = null) { _dbContextFactory = dbContextFactory; _loggerFactory = loggerFactory; + _startupLogger = startupLogger; + _backupService = backupService; + _jellyfinDatabaseProvider = jellyfinDatabaseProvider; + _applicationPaths = applicationPaths; #pragma warning disable CS0618 // Type or member is obsolete Migrations = [.. typeof(IMigrationRoutine).Assembly.GetTypes().Where(e => typeof(IMigrationRoutine).IsAssignableFrom(e) || typeof(IAsyncMigrationRoutine).IsAssignableFrom(e)) - .Select(e => (Type: e, Metadata: e.GetCustomAttribute<JellyfinMigrationAttribute>())) + .Select(e => (Type: e, Metadata: e.GetCustomAttribute<JellyfinMigrationAttribute>(), Backup: e.GetCustomAttributes<JellyfinMigrationBackupAttribute>())) .Where(e => e.Metadata != null) .GroupBy(e => e.Metadata!.Stage) .Select(f => @@ -47,7 +68,13 @@ internal class JellyfinMigrationService var stage = new MigrationStage(f.Key); foreach (var item in f) { - stage.Add(new(item.Type, item.Metadata!)); + JellyfinMigrationBackupAttribute? backupMetadata = null; + if (item.Backup?.Any() == true) + { + backupMetadata = item.Backup.Aggregate(MergeBackupAttributes); + } + + stage.Add(new(item.Type, item.Metadata!, backupMetadata)); } return stage; @@ -57,14 +84,14 @@ internal class JellyfinMigrationService private interface IInternalMigration { - Task PerformAsync(ILogger logger); + Task PerformAsync(IStartupLogger logger); } private HashSet<MigrationStage> Migrations { get; set; } public async Task CheckFirstTimeRunOrMigration(IApplicationPaths appPaths) { - var logger = _loggerFactory.CreateLogger<JellyfinMigrationService>(); + var logger = _startupLogger.With(_loggerFactory.CreateLogger<JellyfinMigrationService>()).BeginGroup($"Migration Startup"); logger.LogInformation("Initialise Migration service."); var xmlSerializer = new MyXmlSerializer(); var serverConfig = File.Exists(appPaths.SystemConfigurationFilePath) @@ -112,11 +139,22 @@ internal class JellyfinMigrationService { var historyRepository = dbContext.GetService<IHistoryRepository>(); var appliedMigrations = await dbContext.Database.GetAppliedMigrationsAsync().ConfigureAwait(false); - var oldMigrations = Migrations + var lastOldAppliedMigration = Migrations .SelectMany(e => e.Where(e => e.Metadata.Key is not null)) // only consider migrations that have the key set as its the reference marker for legacy migrations. .Where(e => migrationOptions.Applied.Any(f => f.Id.Equals(e.Metadata.Key!.Value))) .Where(e => !appliedMigrations.Contains(e.BuildCodeMigrationId())) - .ToArray(); + .OrderBy(e => e.BuildCodeMigrationId()) + .Last(); // this is the latest migration applied in the old migration.xml + + IReadOnlyList<CodeMigration> oldMigrations = [ + .. Migrations + .SelectMany(e => e) + .OrderBy(e => e.BuildCodeMigrationId()) + .TakeWhile(e => e.BuildCodeMigrationId() != lastOldAppliedMigration.BuildCodeMigrationId()), + lastOldAppliedMigration + ]; + // those are all migrations that had to run in the old migration system, even if not noted in the migration.xml file. + var startupScripts = oldMigrations.Select(e => (Migration: e.Metadata, Script: historyRepository.GetInsertScript(new HistoryRow(e.BuildCodeMigrationId(), GetJellyfinVersion())))); foreach (var item in startupScripts) { @@ -139,8 +177,7 @@ internal class JellyfinMigrationService public async Task MigrateStepAsync(JellyfinMigrationStageTypes stage, IServiceProvider? serviceProvider) { - var logger = _loggerFactory.CreateLogger<JellyfinMigrationService>(); - logger.LogInformation("Migrate stage {Stage}.", stage); + var logger = _startupLogger.With(_loggerFactory.CreateLogger<JellyfinMigrationService>()).BeginGroup($"Migrate stage {stage}."); ICollection<CodeMigration> migrationStage = (Migrations.FirstOrDefault(e => e.Stage == stage) as ICollection<CodeMigration>) ?? []; var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); @@ -155,7 +192,7 @@ internal class JellyfinMigrationService .ToArray(); (string Key, InternalDatabaseMigration Migration)[] pendingDatabaseMigrations = []; - if (stage is JellyfinMigrationStageTypes.CoreInitialisaition) + if (stage is JellyfinMigrationStageTypes.CoreInitialisation) { pendingDatabaseMigrations = migrationsAssembly.Migrations.Where(f => appliedMigrations.All(e => e.MigrationId != f.Key)) .Select(e => (Key: e.Key, Migration: new InternalDatabaseMigration(e, dbContext))) @@ -168,15 +205,61 @@ internal class JellyfinMigrationService foreach (var item in migrations) { + var migrationLogger = logger.With(_loggerFactory.CreateLogger(item.Migration.GetType().Name)).BeginGroup($"{item.Key}"); try { - logger.LogInformation("Perform migration {Name}", item.Key); - await item.Migration.PerformAsync(_loggerFactory.CreateLogger(item.GetType().Name)).ConfigureAwait(false); - logger.LogInformation("Migration {Name} was successfully applied", item.Key); + migrationLogger.LogInformation("Perform migration {Name}", item.Key); + await item.Migration.PerformAsync(migrationLogger).ConfigureAwait(false); + migrationLogger.LogInformation("Migration {Name} was successfully applied", item.Key); } catch (Exception ex) { - logger.LogCritical(ex, "Migration {Name} failed", item.Key); + migrationLogger.LogCritical("Error: {Error}", ex.Message); + migrationLogger.LogError(ex, "Migration {Name} failed", item.Key); + + if (_backupKey != default && _backupService is not null && _jellyfinDatabaseProvider is not null) + { + if (_backupKey.LibraryDb is not null) + { + migrationLogger.LogInformation("Attempt to rollback librarydb."); + try + { + var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); + File.Move(_backupKey.LibraryDb, libraryDbPath, true); + } + catch (Exception inner) + { + migrationLogger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.LibraryDb); + } + } + + if (_backupKey.JellyfinDb is not null) + { + migrationLogger.LogInformation("Attempt to rollback JellyfinDb."); + try + { + await _jellyfinDatabaseProvider.RestoreBackupFast(_backupKey.JellyfinDb, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception inner) + { + migrationLogger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.JellyfinDb); + } + } + + if (_backupKey.FullBackup is not null) + { + migrationLogger.LogInformation("Attempt to rollback from backup."); + try + { + await _backupService.RestoreBackupAsync(_backupKey.FullBackup.Path).ConfigureAwait(false); + } + catch (Exception inner) + { + migrationLogger.LogCritical(inner, "Could not rollback from backup {Backup}. Manual intervention might be required to restore a operational state.", _backupKey.FullBackup.Path); + } + } + } + throw; } } @@ -188,6 +271,143 @@ internal class JellyfinMigrationService return Assembly.GetEntryAssembly()!.GetName().Version!.ToString(); } + public async Task CleanupSystemAfterMigration(ILogger logger) + { + if (_backupKey != default) + { + if (_backupKey.LibraryDb is not null) + { + logger.LogInformation("Attempt to cleanup librarydb backup."); + try + { + File.Delete(_backupKey.LibraryDb); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.LibraryDb); + } + } + + if (_backupKey.JellyfinDb is not null && _jellyfinDatabaseProvider is not null) + { + logger.LogInformation("Attempt to cleanup JellyfinDb backup."); + try + { + await _jellyfinDatabaseProvider.DeleteBackup(_backupKey.JellyfinDb).ConfigureAwait(false); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.JellyfinDb); + } + } + + if (_backupKey.FullBackup is not null) + { + logger.LogInformation("Attempt to cleanup from migration backup."); + try + { + File.Delete(_backupKey.FullBackup.Path); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup backup {Backup}.", _backupKey.FullBackup.Path); + } + } + } + } + + public async Task PrepareSystemForMigration(ILogger logger) + { + logger.LogInformation("Prepare system for possible migrations"); + JellyfinMigrationBackupAttribute backupInstruction; + IReadOnlyList<HistoryRow> appliedMigrations; + var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var historyRepository = dbContext.GetService<IHistoryRepository>(); + var migrationsAssembly = dbContext.GetService<IMigrationsAssembly>(); + appliedMigrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); + backupInstruction = new JellyfinMigrationBackupAttribute() + { + JellyfinDb = migrationsAssembly.Migrations.Any(f => appliedMigrations.All(e => e.MigrationId != f.Key)) + }; + } + + backupInstruction = Migrations.SelectMany(e => e) + .Where(e => appliedMigrations.All(f => f.MigrationId != e.BuildCodeMigrationId())) + .Select(e => e.BackupRequirements) + .Where(e => e is not null) + .Aggregate(backupInstruction, MergeBackupAttributes!); + + if (backupInstruction.LegacyLibraryDb) + { + logger.LogInformation("A migration will attempt to modify the library.db, will attempt to backup the file now."); + // for legacy migrations that still operates on the library.db + var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); + if (File.Exists(libraryDbPath)) + { + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", libraryDbPath, i); + if (!File.Exists(bakPath)) + { + try + { + logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); + File.Copy(libraryDbPath, bakPath); + _backupKey = (bakPath, _backupKey.JellyfinDb, _backupKey.FullBackup); + logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); + throw; + } + } + } + + logger.LogInformation("{Library} has been backed up as {BackupPath}", DbFilename, _backupKey.LibraryDb); + } + else + { + logger.LogError("Cannot make a backup of {Library} at path {BackupPath} because file could not be found at {LibraryPath}", DbFilename, libraryDbPath, _applicationPaths.DataPath); + } + } + + if (backupInstruction.JellyfinDb && _jellyfinDatabaseProvider != null) + { + logger.LogInformation("A migration will attempt to modify the jellyfin.db, will attempt to backup the file now."); + _backupKey = (_backupKey.LibraryDb, await _jellyfinDatabaseProvider.MigrationBackupFast(CancellationToken.None).ConfigureAwait(false), _backupKey.FullBackup); + logger.LogInformation("Jellyfin database has been backed up as {BackupPath}", _backupKey.JellyfinDb); + } + + if (_backupService is not null && (backupInstruction.Metadata || backupInstruction.Subtitles || backupInstruction.Trickplay)) + { + logger.LogInformation("A migration will attempt to modify system resources. Will attempt to create backup now."); + _backupKey = (_backupKey.LibraryDb, _backupKey.JellyfinDb, await _backupService.CreateBackupAsync(new BackupOptionsDto() + { + Metadata = backupInstruction.Metadata, + Subtitles = backupInstruction.Subtitles, + Trickplay = backupInstruction.Trickplay, + Database = false // database backups are explicitly handled by the provider itself as the backup service requires parity with the current model + }).ConfigureAwait(false)); + logger.LogInformation("Pre-Migration backup successfully created as {BackupKey}", _backupKey.FullBackup.Path); + } + } + + private static JellyfinMigrationBackupAttribute MergeBackupAttributes(JellyfinMigrationBackupAttribute left, JellyfinMigrationBackupAttribute right) + { + return new JellyfinMigrationBackupAttribute() + { + JellyfinDb = left!.JellyfinDb || right!.JellyfinDb, + LegacyLibraryDb = left.LegacyLibraryDb || right!.LegacyLibraryDb, + Metadata = left.Metadata || right!.Metadata, + Subtitles = left.Subtitles || right!.Subtitles, + Trickplay = left.Trickplay || right!.Trickplay + }; + } + private class InternalCodeMigration : IInternalMigration { private readonly CodeMigration _codeMigration; @@ -201,9 +421,9 @@ internal class JellyfinMigrationService _dbContext = dbContext; } - public async Task PerformAsync(ILogger logger) + public async Task PerformAsync(IStartupLogger logger) { - await _codeMigration.Perform(_serviceProvider, CancellationToken.None).ConfigureAwait(false); + await _codeMigration.Perform(_serviceProvider, logger, CancellationToken.None).ConfigureAwait(false); var historyRepository = _dbContext.GetService<IHistoryRepository>(); var createScript = historyRepository.GetInsertScript(new HistoryRow(_codeMigration.BuildCodeMigrationId(), GetJellyfinVersion())); @@ -222,7 +442,7 @@ internal class JellyfinMigrationService _jellyfinDbContext = jellyfinDbContext; } - public async Task PerformAsync(ILogger logger) + public async Task PerformAsync(IStartupLogger logger) { var migrator = _jellyfinDbContext.GetService<IMigrator>(); await migrator.MigrateAsync(_databaseMigrationInfo.Key).ConfigureAwait(false); diff --git a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs index af8787b95..05ded06ba 100644 --- a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs +++ b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs @@ -18,10 +18,10 @@ namespace Jellyfin.Server.Migrations.Routines /// </summary> #pragma warning disable CS0618 // Type or member is obsolete [JellyfinMigration("2025-04-20T18:00:00", nameof(FixAudioData), "CF6FABC2-9FBE-4933-84A5-FFE52EF22A58")] + [JellyfinMigrationBackup(LegacyLibraryDb = true)] internal class FixAudioData : IMigrationRoutine #pragma warning restore CS0618 // Type or member is obsolete { - private const string DbFilename = "library.db"; private readonly ILogger<FixAudioData> _logger; private readonly IServerApplicationPaths _applicationPaths; private readonly IItemRepository _itemRepository; @@ -39,29 +39,6 @@ namespace Jellyfin.Server.Migrations.Routines /// <inheritdoc/> public void Perform() { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) - { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - _logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); - File.Copy(dbPath, bakPath); - _logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } - } - _logger.LogInformation("Backfilling audio lyrics data to database."); var startIndex = 0; var records = _itemRepository.GetCount(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs index 03a521258..033045e63 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs @@ -9,6 +9,7 @@ using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions.Json; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using Microsoft.EntityFrameworkCore; @@ -22,7 +23,7 @@ namespace Jellyfin.Server.Migrations.Routines; [JellyfinMigration("2025-04-21T00:00:00", nameof(MigrateKeyframeData))] public class MigrateKeyframeData : IDatabaseMigrationRoutine { - private readonly ILogger<MigrateKeyframeData> _logger; + private readonly IStartupLogger _logger; private readonly IApplicationPaths _appPaths; private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; @@ -30,15 +31,15 @@ public class MigrateKeyframeData : IDatabaseMigrationRoutine /// <summary> /// Initializes a new instance of the <see cref="MigrateKeyframeData"/> class. /// </summary> - /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> /// <param name="dbProvider">The EFCore db factory.</param> public MigrateKeyframeData( - ILogger<MigrateKeyframeData> logger, + IStartupLogger startupLogger, IApplicationPaths appPaths, IDbContextFactory<JellyfinDbContext> dbProvider) { - _logger = logger; + _logger = startupLogger; _appPaths = appPaths; _dbProvider = dbProvider; } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs index c9d289940..521655a4f 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs @@ -14,6 +14,7 @@ using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions; using Jellyfin.Server.Implementations.Item; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; @@ -29,11 +30,12 @@ namespace Jellyfin.Server.Migrations.Routines; /// The migration routine for migrating the userdata database to EF Core. /// </summary> [JellyfinMigration("2025-04-20T20:00:00", nameof(MigrateLibraryDb))] +[JellyfinMigrationBackup(JellyfinDb = true, LegacyLibraryDb = true)] internal class MigrateLibraryDb : IDatabaseMigrationRoutine { private const string DbFilename = "library.db"; - private readonly ILogger<MigrateLibraryDb> _logger; + private readonly IStartupLogger _logger; private readonly IServerApplicationPaths _paths; private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; private readonly IDbContextFactory<JellyfinDbContext> _provider; @@ -41,19 +43,17 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine /// <summary> /// Initializes a new instance of the <see cref="MigrateLibraryDb"/> class. /// </summary> - /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> /// <param name="provider">The database provider.</param> /// <param name="paths">The server application paths.</param> /// <param name="jellyfinDatabaseProvider">The database provider for special access.</param> - /// <param name="serviceProvider">The Service provider.</param> public MigrateLibraryDb( - ILogger<MigrateLibraryDb> logger, + IStartupLogger startupLogger, IDbContextFactory<JellyfinDbContext> provider, IServerApplicationPaths paths, - IJellyfinDatabaseProvider jellyfinDatabaseProvider, - IServiceProvider serviceProvider) + IJellyfinDatabaseProvider jellyfinDatabaseProvider) { - _logger = logger; + _logger = startupLogger; _provider = provider; _paths = paths; _jellyfinDatabaseProvider = jellyfinDatabaseProvider; diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs new file mode 100644 index 000000000..2d5fc2a0d --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs @@ -0,0 +1,73 @@ +#pragma warning disable RS0030 // Do not use banned APIs + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Controller; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// The migration routine for checking if the current instance of Jellyfin is compatiable to be upgraded. +/// </summary> +[JellyfinMigration("2025-04-20T19:30:00", nameof(MigrateLibraryDbCompatibilityCheck))] +public class MigrateLibraryDbCompatibilityCheck : IAsyncMigrationRoutine +{ + private const string DbFilename = "library.db"; + private readonly IStartupLogger _logger; + private readonly IServerApplicationPaths _paths; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateLibraryDbCompatibilityCheck"/> class. + /// </summary> + /// <param name="startupLogger">The startup logger.</param> + /// <param name="paths">The Path service.</param> + public MigrateLibraryDbCompatibilityCheck(IStartupLogger startupLogger, IServerApplicationPaths paths) + { + _logger = startupLogger; + _paths = paths; + } + + /// <inheritdoc/> + public async Task PerformAsync(CancellationToken cancellationToken) + { + var dataPath = _paths.DataPath; + var libraryDbPath = Path.Combine(dataPath, DbFilename); + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate {LibraryDb} as it does not exist..", libraryDbPath); + return; + } + + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + CheckMigratableVersion(connection); + await connection.CloseAsync().ConfigureAwait(false); + } + + private static void CheckMigratableVersion(SqliteConnection connection) + { + CheckColumnExistance(connection, "TypedBaseItems", "lufs"); + CheckColumnExistance(connection, "TypedBaseItems", "normalizationgain"); + CheckColumnExistance(connection, "mediastreams", "dvversionmajor"); + + static void CheckColumnExistance(SqliteConnection connection, string table, string column) + { + using (var cmd = connection.CreateCommand()) + { +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + cmd.CommandText = $"Select COUNT(1) FROM pragma_table_xinfo('{table}') WHERE lower(name) = '{column}';"; +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities + var result = cmd.ExecuteScalar()!; + if (!result.Equals(1L)) + { + throw new InvalidOperationException("Your database does not meet the required standard. Only upgrades from server version 10.9.11 or above are supported. Please upgrade first to server version 10.10.7 before attempting to upgrade afterwards to 10.11"); + } + } + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs index 234965c0a..ae93557de 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using Jellyfin.Database.Implementations; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Model.Globalization; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -12,21 +13,22 @@ namespace Jellyfin.Server.Migrations.Routines; /// </summary> #pragma warning disable CS0618 // Type or member is obsolete [JellyfinMigration("2025-04-20T22:00:00", nameof(MigrateRatingLevels))] +[JellyfinMigrationBackup(JellyfinDb = true)] #pragma warning restore CS0618 // Type or member is obsolete internal class MigrateRatingLevels : IDatabaseMigrationRoutine { - private readonly ILogger<MigrateRatingLevels> _logger; + private readonly IStartupLogger _logger; private readonly IDbContextFactory<JellyfinDbContext> _provider; private readonly ILocalizationManager _localizationManager; public MigrateRatingLevels( IDbContextFactory<JellyfinDbContext> provider, - ILoggerFactory loggerFactory, + IStartupLogger logger, ILocalizationManager localizationManager) { _provider = provider; _localizationManager = localizationManager; - _logger = loggerFactory.CreateLogger<MigrateRatingLevels>(); + _logger = logger; } /// <inheritdoc/> diff --git a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs index 8b4abdfe5..6f650f731 100644 --- a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs +++ b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs @@ -8,13 +8,15 @@ using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; +using System.Threading; +using System.Threading.Tasks; using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller.IO; -using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -25,12 +27,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// Migration to move extracted files to the new directories. /// </summary> [JellyfinMigration("2025-04-20T21:00:00", nameof(MoveExtractedFiles))] -#pragma warning disable CS0618 // Type or member is obsolete -public class MoveExtractedFiles : IMigrationRoutine -#pragma warning restore CS0618 // Type or member is obsolete +public class MoveExtractedFiles : IAsyncMigrationRoutine { private readonly IApplicationPaths _appPaths; - private readonly ILogger<MoveExtractedFiles> _logger; + private readonly ILogger _logger; private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; private readonly IPathManager _pathManager; private readonly IFileSystem _fileSystem; @@ -40,18 +40,20 @@ public class MoveExtractedFiles : IMigrationRoutine /// </summary> /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> /// <param name="pathManager">Instance of the <see cref="IPathManager"/> interface.</param> /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> public MoveExtractedFiles( IApplicationPaths appPaths, ILogger<MoveExtractedFiles> logger, + IStartupLogger startupLogger, IPathManager pathManager, IFileSystem fileSystem, IDbContextFactory<JellyfinDbContext> dbProvider) { _appPaths = appPaths; - _logger = logger; + _logger = startupLogger.With(logger); _pathManager = pathManager; _fileSystem = fileSystem; _dbProvider = dbProvider; @@ -62,10 +64,10 @@ public class MoveExtractedFiles : IMigrationRoutine private string AttachmentCachePath => Path.Combine(_appPaths.DataPath, "attachments"); /// <inheritdoc /> - public void Perform() + public async Task PerformAsync(CancellationToken cancellationToken) { const int Limit = 5000; - int itemCount = 0, offset = 0; + int itemCount = 0; var sw = Stopwatch.StartNew(); @@ -76,32 +78,27 @@ public class MoveExtractedFiles : IMigrationRoutine // Make sure directories exist Directory.CreateDirectory(SubtitleCachePath); Directory.CreateDirectory(AttachmentCachePath); - do - { - var results = context.BaseItems - .Include(e => e.MediaStreams!.Where(s => s.StreamType == MediaStreamTypeEntity.Subtitle && !s.IsExternal)) - .Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder) - .OrderBy(e => e.Id) - .Skip(offset) - .Take(Limit) - .Select(b => new Tuple<Guid, string?, ICollection<MediaStreamInfo>?>(b.Id, b.Path, b.MediaStreams)).ToList(); - - foreach (var result in results) - { - if (MoveSubtitleAndAttachmentFiles(result.Item1, result.Item2, result.Item3, context)) - { - itemCount++; - } - } - offset += Limit; - if (offset > records) + await foreach (var result in context.BaseItems + .Include(e => e.MediaStreams!.Where(s => s.StreamType == MediaStreamTypeEntity.Subtitle && !s.IsExternal)) + .Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder) + .Select(b => new + { + b.Id, + b.Path, + b.MediaStreams + }) + .OrderBy(e => e.Id) + .WithPartitionProgress((partition) => _logger.LogInformation("Checked: {Count} - Moved: {Items} - Time: {Time}", partition * Limit, itemCount, sw.Elapsed)) + .PartitionEagerAsync(Limit, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + if (MoveSubtitleAndAttachmentFiles(result.Id, result.Path, result.MediaStreams, context)) { - offset = records; + itemCount++; } - - _logger.LogInformation("Checked: {Count} - Moved: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed); - } while (offset < records); + } _logger.LogInformation("Moved files for {Count} items in {Time}", itemCount, sw.Elapsed); diff --git a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs index 63b0614fd..a674aa928 100644 --- a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs +++ b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.IO; using System.Linq; using Jellyfin.Data.Enums; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Trickplay; @@ -23,7 +24,7 @@ public class MoveTrickplayFiles : IMigrationRoutine private readonly ITrickplayManager _trickplayManager; private readonly IFileSystem _fileSystem; private readonly ILibraryManager _libraryManager; - private readonly ILogger<MoveTrickplayFiles> _logger; + private readonly IStartupLogger _logger; /// <summary> /// Initializes a new instance of the <see cref="MoveTrickplayFiles"/> class. @@ -36,7 +37,7 @@ public class MoveTrickplayFiles : IMigrationRoutine ITrickplayManager trickplayManager, IFileSystem fileSystem, ILibraryManager libraryManager, - ILogger<MoveTrickplayFiles> logger) + IStartupLogger logger) { _trickplayManager = trickplayManager; _fileSystem = fileSystem; diff --git a/Jellyfin.Server/Migrations/Stages/CodeMigration.cs b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs index 1e4dfb237..47ed26965 100644 --- a/Jellyfin.Server/Migrations/Stages/CodeMigration.cs +++ b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs @@ -2,22 +2,50 @@ using System; using System.Globalization; using System.Threading; using System.Threading.Tasks; +using Jellyfin.Server.ServerSetupApp; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; namespace Jellyfin.Server.Migrations.Stages; -internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute metadata) +internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute metadata, JellyfinMigrationBackupAttribute? migrationBackupAttribute) { public Type MigrationType { get; } = migrationType; public JellyfinMigrationAttribute Metadata { get; } = metadata; + public JellyfinMigrationBackupAttribute? BackupRequirements { get; set; } = migrationBackupAttribute; + public string BuildCodeMigrationId() { - return Metadata.Order.ToString("yyyyMMddHHmmsss", CultureInfo.InvariantCulture) + "_" + MigrationType.Name!; + return Metadata.Order.ToString("yyyyMMddHHmmsss", CultureInfo.InvariantCulture) + "_" + Metadata.Name!; + } + + private ServiceCollection MigrationServices(IServiceProvider serviceProvider, IStartupLogger logger) + { + var childServiceCollection = new ServiceCollection(); + childServiceCollection.AddSingleton(serviceProvider); + childServiceCollection.AddSingleton(logger); + + foreach (ServiceDescriptor service in serviceProvider.GetRequiredService<IServiceCollection>()) + { + if (service.Lifetime == ServiceLifetime.Singleton && !service.ServiceType.IsGenericTypeDefinition) + { + object? serviceInstance = serviceProvider.GetService(service.ServiceType); + if (serviceInstance != null) + { + childServiceCollection.AddSingleton(service.ServiceType, serviceInstance); + continue; + } + } + + childServiceCollection.Add(service); + } + + return childServiceCollection; } - public async Task Perform(IServiceProvider? serviceProvider, CancellationToken cancellationToken) + public async Task Perform(IServiceProvider? serviceProvider, IStartupLogger logger, CancellationToken cancellationToken) { #pragma warning disable CS0618 // Type or member is obsolete if (typeof(IMigrationRoutine).IsAssignableFrom(MigrationType)) @@ -28,7 +56,8 @@ internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute meta } else { - ((IMigrationRoutine)ActivatorUtilities.CreateInstance(serviceProvider, MigrationType)).Perform(); + using var migrationServices = MigrationServices(serviceProvider, logger).BuildServiceProvider(); + ((IMigrationRoutine)ActivatorUtilities.CreateInstance(migrationServices, MigrationType)).Perform(); #pragma warning restore CS0618 // Type or member is obsolete } } @@ -40,7 +69,8 @@ internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute meta } else { - await ((IAsyncMigrationRoutine)ActivatorUtilities.CreateInstance(serviceProvider, MigrationType)).PerformAsync(cancellationToken).ConfigureAwait(false); + using var migrationServices = MigrationServices(serviceProvider, logger).BuildServiceProvider(); + await ((IAsyncMigrationRoutine)ActivatorUtilities.CreateInstance(migrationServices, MigrationType)).PerformAsync(cancellationToken).ConfigureAwait(false); } } else diff --git a/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs b/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs index d90ad3d9b..3d5ec233b 100644 --- a/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs +++ b/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs @@ -17,7 +17,7 @@ public enum JellyfinMigrationStageTypes /// Runs after the host has been configured and includes the database migrations. /// Allows the mix order of migrations that contain application code and database changes. /// </summary> - CoreInitialisaition = 2, + CoreInitialisation = 2, /// <summary> /// Runs after services has been registered and initialised. Last step before running the server. diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs index 4584b25bd..0b77d63ac 100644 --- a/Jellyfin.Server/Program.cs +++ b/Jellyfin.Server/Program.cs @@ -16,10 +16,10 @@ using Jellyfin.Server.Extensions; using Jellyfin.Server.Helpers; using Jellyfin.Server.Implementations.DatabaseConfiguration; using Jellyfin.Server.Implementations.Extensions; -using Jellyfin.Server.Implementations.FullSystemBackup; using Jellyfin.Server.Implementations.StorageHelpers; using Jellyfin.Server.Implementations.SystemBackupService; using Jellyfin.Server.Migrations; +using Jellyfin.Server.Migrations.Stages; using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; @@ -60,6 +60,7 @@ namespace Jellyfin.Server private static long _startTimestamp; private static ILogger _logger = NullLogger.Instance; private static bool _restartOnShutdown; + private static IStartupLogger? _migrationLogger; private static string? _restoreFromBackup; /// <summary> @@ -98,9 +99,9 @@ namespace Jellyfin.Server // Create an instance of the application configuration to use for application startup IConfiguration startupConfig = CreateAppConfiguration(options, appPaths); + StartupHelpers.InitializeLoggingFramework(startupConfig, appPaths); _setupServer = new SetupServer(static () => _jellyfinHost?.Services?.GetService<INetworkManager>(), appPaths, static () => _appHost, _loggerFactory, startupConfig); await _setupServer.RunAsync().ConfigureAwait(false); - StartupHelpers.InitializeLoggingFramework(startupConfig, appPaths); _logger = _loggerFactory.CreateLogger("Main"); // Use the logging framework for uncaught exceptions instead of std error @@ -131,7 +132,7 @@ namespace Jellyfin.Server } } - StorageHelper.TestCommonPathsForStorageCapacity(appPaths, _loggerFactory.CreateLogger<Startup>()); + StorageHelper.TestCommonPathsForStorageCapacity(appPaths, StartupLogger.Logger.With(_loggerFactory.CreateLogger<Startup>()).BeginGroup($"Storage Check")); StartupHelpers.PerformStaticInitialization(); @@ -160,6 +161,7 @@ namespace Jellyfin.Server options, startupConfig); _appHost = appHost; + var configurationCompleted = false; try { _jellyfinHost = Host.CreateDefaultBuilder() @@ -176,6 +178,7 @@ namespace Jellyfin.Server }) .ConfigureAppConfiguration(config => config.ConfigureAppConfiguration(options, appPaths, startupConfig)) .UseSerilog() + .ConfigureServices(e => e.AddTransient<IStartupLogger, StartupLogger>().AddSingleton<IServiceCollection>(e)) .Build(); // Re-use the host service provider in the app host since ASP.NET doesn't allow a custom service collection. @@ -190,14 +193,17 @@ namespace Jellyfin.Server return; } - await ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.CoreInitialisaition).ConfigureAwait(false); + var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(appHost.ServiceProvider); + await jellyfinMigrationService.PrepareSystemForMigration(_logger).ConfigureAwait(false); + await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.CoreInitialisation, appHost.ServiceProvider).ConfigureAwait(false); await appHost.InitializeServices(startupConfig).ConfigureAwait(false); - await ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.AppInitialisation).ConfigureAwait(false); - + await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.AppInitialisation, appHost.ServiceProvider).ConfigureAwait(false); + await jellyfinMigrationService.CleanupSystemAfterMigration(_logger).ConfigureAwait(false); try { + configurationCompleted = true; await _setupServer!.StopAsync().ConfigureAwait(false); await _jellyfinHost.StartAsync().ConfigureAwait(false); @@ -226,6 +232,12 @@ namespace Jellyfin.Server { _restartOnShutdown = false; _logger.LogCritical(ex, "Error while starting server"); + if (_setupServer!.IsAlive && !configurationCompleted) + { + _setupServer!.SoftStop(); + await Task.Delay(TimeSpan.FromMinutes(10)).ConfigureAwait(false); + await _setupServer!.StopAsync().ConfigureAwait(false); + } } finally { @@ -256,13 +268,17 @@ namespace Jellyfin.Server /// <returns>A task.</returns> public static async Task ApplyStartupMigrationAsync(ServerApplicationPaths appPaths, IConfiguration startupConfig) { + _migrationLogger = StartupLogger.Logger.BeginGroup($"Migration Service"); var startupConfigurationManager = new ServerConfigurationManager(appPaths, _loggerFactory, new MyXmlSerializer()); startupConfigurationManager.AddParts([new DatabaseConfigurationFactory()]); var migrationStartupServiceProvider = new ServiceCollection() .AddLogging(d => d.AddSerilog()) .AddJellyfinDbContext(startupConfigurationManager, startupConfig) .AddSingleton<IApplicationPaths>(appPaths) - .AddSingleton<ServerApplicationPaths>(appPaths); + .AddSingleton<ServerApplicationPaths>(appPaths) + .AddSingleton<IStartupLogger>(_migrationLogger); + + migrationStartupServiceProvider.AddSingleton(migrationStartupServiceProvider); var startupService = migrationStartupServiceProvider.BuildServiceProvider(); PrepareDatabaseProvider(startupService); @@ -283,7 +299,7 @@ namespace Jellyfin.Server /// <returns>A task.</returns> public static async Task ApplyCoreMigrationsAsync(IServiceProvider serviceProvider, Migrations.Stages.JellyfinMigrationStageTypes jellyfinMigrationStage) { - var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(serviceProvider); + var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(serviceProvider, _migrationLogger!); await jellyfinMigrationService.MigrateStepAsync(jellyfinMigrationStage, serviceProvider).ConfigureAwait(false); } diff --git a/Jellyfin.Server/ServerSetupApp/IStartupLogger.cs b/Jellyfin.Server/ServerSetupApp/IStartupLogger.cs new file mode 100644 index 000000000..2c2ef05f8 --- /dev/null +++ b/Jellyfin.Server/ServerSetupApp/IStartupLogger.cs @@ -0,0 +1,25 @@ +using System; +using Morestachio.Helper.Logging; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Jellyfin.Server.ServerSetupApp; + +/// <summary> +/// Defines the Startup Logger. This logger acts an an aggregate logger that will push though all log messages to both the attached logger as well as the startup UI. +/// </summary> +public interface IStartupLogger : ILogger +{ + /// <summary> + /// Adds another logger instance to this logger for combined logging. + /// </summary> + /// <param name="logger">Other logger to rely messages to.</param> + /// <returns>A combined logger.</returns> + IStartupLogger With(ILogger logger); + + /// <summary> + /// Opens a new Group logger within the parent logger. + /// </summary> + /// <param name="logEntry">Defines the log message that introduces the new group.</param> + /// <returns>A new logger that can write to the group.</returns> + IStartupLogger BeginGroup(FormattableString logEntry); +} diff --git a/Jellyfin.Server/ServerSetupApp/SetupServer.cs b/Jellyfin.Server/ServerSetupApp/SetupServer.cs index 7ab5defc8..d88dbee57 100644 --- a/Jellyfin.Server/ServerSetupApp/SetupServer.cs +++ b/Jellyfin.Server/ServerSetupApp/SetupServer.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Net; @@ -10,6 +13,7 @@ using Jellyfin.Networking.Manager; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller; +using MediaBrowser.Model.IO; using MediaBrowser.Model.System; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -20,6 +24,9 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Primitives; +using Morestachio; +using Morestachio.Framework.IO.SingleStream; +using Morestachio.Rendering; namespace Jellyfin.Server.ServerSetupApp; @@ -34,8 +41,10 @@ public sealed class SetupServer : IDisposable private readonly ILoggerFactory _loggerFactory; private readonly IConfiguration _startupConfiguration; private readonly ServerConfigurationManager _configurationManager; + private IRenderer? _startupUiRenderer; private IHost? _startupServer; private bool _disposed; + private bool _isUnhealthy; /// <summary> /// Initializes a new instance of the <see cref="SetupServer"/> class. @@ -62,13 +71,73 @@ public sealed class SetupServer : IDisposable _configurationManager.RegisterConfiguration<NetworkConfigurationFactory>(); } + internal static ConcurrentQueue<StartupLogEntry>? LogQueue { get; set; } = new(); + + /// <summary> + /// Gets a value indicating whether Startup server is currently running. + /// </summary> + public bool IsAlive { get; internal set; } + /// <summary> /// Starts the Bind-All Setup aspcore server to provide a reflection on the current core setup. /// </summary> /// <returns>A Task.</returns> public async Task RunAsync() { + var fileTemplate = await File.ReadAllTextAsync(Path.Combine(AppContext.BaseDirectory, "ServerSetupApp", "index.mstemplate.html")).ConfigureAwait(false); + _startupUiRenderer = (await ParserOptionsBuilder.New() + .WithTemplate(fileTemplate) + .WithFormatter( + (StartupLogEntry logEntry, IEnumerable<StartupLogEntry> children) => + { + if (children.Any()) + { + var maxLevel = logEntry.LogLevel; + var stack = new Stack<StartupLogEntry>(children); + + while (maxLevel != LogLevel.Error && stack.Count > 0 && (logEntry = stack.Pop()) != null) // error is the highest inherted error level. + { + maxLevel = maxLevel < logEntry.LogLevel ? logEntry.LogLevel : maxLevel; + foreach (var child in logEntry.Children) + { + stack.Push(child); + } + } + + return maxLevel; + } + + return logEntry.LogLevel; + }, + "FormatLogLevel") + .WithFormatter( + (LogLevel logLevel) => + { + switch (logLevel) + { + case LogLevel.Trace: + case LogLevel.Debug: + case LogLevel.None: + return "success"; + case LogLevel.Information: + return "info"; + case LogLevel.Warning: + return "warn"; + case LogLevel.Error: + return "danger"; + case LogLevel.Critical: + return "danger-strong"; + } + + return string.Empty; + }, + "ToString") + .BuildAndParseAsync() + .ConfigureAwait(false)) + .CreateCompiledRenderer(); + ThrowIfDisposed(); + var retryAfterValue = TimeSpan.FromSeconds(5); _startupServer = Host.CreateDefaultBuilder() .UseConsoleLifetime() .ConfigureServices(serv => @@ -140,7 +209,7 @@ public sealed class SetupServer : IDisposable if (jfApplicationHost is null) { context.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; - context.Response.Headers.RetryAfter = new StringValues("5"); + context.Response.Headers.RetryAfter = new StringValues(retryAfterValue.TotalSeconds.ToString("000", CultureInfo.InvariantCulture)); return; } @@ -158,24 +227,30 @@ public sealed class SetupServer : IDisposable }); }); - app.Run((context) => + app.Run(async (context) => { context.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; - context.Response.Headers.RetryAfter = new StringValues("5"); + context.Response.Headers.RetryAfter = new StringValues(retryAfterValue.TotalSeconds.ToString("000", CultureInfo.InvariantCulture)); context.Response.Headers.ContentType = new StringValues("text/html"); - context.Response.WriteAsync("<p>Jellyfin Server still starting. Please wait.</p>"); var networkManager = _networkManagerFactory(); - if (networkManager is not null && context.Connection.RemoteIpAddress is not null && networkManager.IsInLocalNetwork(context.Connection.RemoteIpAddress)) - { - context.Response.WriteAsync("<p>You can download the current logfiles <a href='/startup/logger'>here</a>.</p>"); - } - return Task.CompletedTask; + var startupLogEntries = LogQueue?.ToArray() ?? []; + await _startupUiRenderer.RenderAsync( + new Dictionary<string, object>() + { + { "isInReportingMode", _isUnhealthy }, + { "retryValue", retryAfterValue }, + { "logs", startupLogEntries }, + { "localNetworkRequest", networkManager is not null && context.Connection.RemoteIpAddress is not null && networkManager.IsInLocalNetwork(context.Connection.RemoteIpAddress) } + }, + new ByteCounterStream(context.Response.BodyWriter.AsStream(), IODefaults.FileStreamBufferSize, true, _startupUiRenderer.ParserOptions)) + .ConfigureAwait(false); }); }); }) .Build(); await _startupServer.StartAsync().ConfigureAwait(false); + IsAlive = true; } /// <summary> @@ -191,6 +266,7 @@ public sealed class SetupServer : IDisposable } await _startupServer.StopAsync().ConfigureAwait(false); + IsAlive = false; } /// <inheritdoc/> @@ -203,6 +279,9 @@ public sealed class SetupServer : IDisposable _disposed = true; _startupServer?.Dispose(); + IsAlive = false; + LogQueue?.Clear(); + LogQueue = null; } private void ThrowIfDisposed() @@ -210,11 +289,88 @@ public sealed class SetupServer : IDisposable ObjectDisposedException.ThrowIf(_disposed, this); } + internal void SoftStop() + { + _isUnhealthy = true; + } + private class SetupHealthcheck : IHealthCheck { + private readonly SetupServer _startupServer; + + public SetupHealthcheck(SetupServer startupServer) + { + _startupServer = startupServer; + } + public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { + if (_startupServer._isUnhealthy) + { + return Task.FromResult(HealthCheckResult.Unhealthy("Server is could not complete startup. Check logs.")); + } + return Task.FromResult(HealthCheckResult.Degraded("Server is still starting up.")); } } + + internal sealed class SetupLoggerFactory : ILoggerProvider, IDisposable + { + private bool _disposed; + + public ILogger CreateLogger(string categoryName) + { + return new CatchingSetupServerLogger(); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + } + } + + internal sealed class CatchingSetupServerLogger : ILogger + { + public IDisposable? BeginScope<TState>(TState state) + where TState : notnull + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) + { + return logLevel is LogLevel.Error or LogLevel.Critical; + } + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + + LogQueue?.Enqueue(new() + { + LogLevel = logLevel, + Content = formatter(state, exception), + DateOfCreation = DateTimeOffset.Now + }); + } + } + + internal class StartupLogEntry + { + public LogLevel LogLevel { get; set; } + + public string? Content { get; set; } + + public DateTimeOffset DateOfCreation { get; set; } + + public List<StartupLogEntry> Children { get; set; } = []; + } } diff --git a/Jellyfin.Server/ServerSetupApp/StartupLogger.cs b/Jellyfin.Server/ServerSetupApp/StartupLogger.cs new file mode 100644 index 000000000..2b86dc0c1 --- /dev/null +++ b/Jellyfin.Server/ServerSetupApp/StartupLogger.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Jellyfin.Server.Migrations.Routines; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.ServerSetupApp; + +/// <inheritdoc/> +public class StartupLogger : IStartupLogger +{ + private readonly SetupServer.StartupLogEntry? _groupEntry; + + /// <summary> + /// Initializes a new instance of the <see cref="StartupLogger"/> class. + /// </summary> + public StartupLogger() + { + Loggers = []; + } + + /// <summary> + /// Initializes a new instance of the <see cref="StartupLogger"/> class. + /// </summary> + private StartupLogger(SetupServer.StartupLogEntry? groupEntry) : this() + { + _groupEntry = groupEntry; + } + + internal static IStartupLogger Logger { get; } = new StartupLogger(); + + private List<ILogger> Loggers { get; set; } + + /// <inheritdoc/> + public IStartupLogger BeginGroup(FormattableString logEntry) + { + var startupEntry = new SetupServer.StartupLogEntry() + { + Content = logEntry.ToString(CultureInfo.InvariantCulture), + DateOfCreation = DateTimeOffset.Now + }; + + if (_groupEntry is null) + { + SetupServer.LogQueue?.Enqueue(startupEntry); + } + else + { + _groupEntry.Children.Add(startupEntry); + } + + return new StartupLogger(startupEntry); + } + + /// <inheritdoc/> + public IDisposable? BeginScope<TState>(TState state) + where TState : notnull + { + return null; + } + + /// <inheritdoc/> + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + /// <inheritdoc/> + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) + { + foreach (var item in Loggers.Where(e => e.IsEnabled(logLevel))) + { + item.Log(logLevel, eventId, state, exception, formatter); + } + + var startupEntry = new SetupServer.StartupLogEntry() + { + LogLevel = logLevel, + Content = formatter(state, exception), + DateOfCreation = DateTimeOffset.Now + }; + + if (_groupEntry is null) + { + SetupServer.LogQueue?.Enqueue(startupEntry); + } + else + { + _groupEntry.Children.Add(startupEntry); + } + } + + /// <inheritdoc/> + public IStartupLogger With(ILogger logger) + { + return new StartupLogger(_groupEntry) + { + Loggers = [.. Loggers, logger] + }; + } +} diff --git a/Jellyfin.Server/ServerSetupApp/index.mstemplate.html b/Jellyfin.Server/ServerSetupApp/index.mstemplate.html new file mode 100644 index 000000000..747835b2a --- /dev/null +++ b/Jellyfin.Server/ServerSetupApp/index.mstemplate.html @@ -0,0 +1,225 @@ +<!DOCTYPE html> +<html> + +<head> + <meta charset="UTF-8" /> + <title> + {{#IF isInReportingMode}} + ❌ + {{/IF}} + Jellyfin Startup + </title> + <style> + * { + font-family: sans-serif; + } + + .flex-row { + display: flex; + flex-direction: row; + flex-wrap: nowrap; + justify-content: center; + align-items: center; + align-content: normal; + } + + .flex-col { + display: flex; + flex-direction: column; + flex-wrap: nowrap; + justify-content: center; + align-items: center; + align-content: normal; + } + + header { + height: 5rem; + width: 100%; + } + + header svg { + height: 3rem; + width: 9rem; + margin-right: 1rem; + } + + /* ol.action-list { + list-style-type: none; + position: relative; + } */ + + ol.action-list * { + font-family: monospace; + font-weight: 300; + font-size: clamp(18px, 100vw / var(--width), 20px); + font-feature-settings: 'onum', 'pnum'; + line-height: 1.8; + -webkit-text-size-adjust: none; + } + + /* + ol.action-list li { + padding-top: .5rem; + } + + ol.action-list li::before { + position: absolute; + left: -0.8em; + font-size: 1.1em; + } */ + + /* Attribution as heavily inspired by: https://iamkate.com/code/tree-views/ */ + .action-list { + --spacing: 1.4rem; + --radius: 14px; + } + + .action-list li { + display: block; + position: relative; + padding-left: calc(2 * var(--spacing) - var(--radius) - 1px); + } + + .action-list ul { + margin-left: calc(var(--radius) - var(--spacing)); + padding-left: 0; + } + + .action-list ul li { + border-left: 2px solid #ddd; + } + + .action-list ul li:last-child { + border-color: transparent; + } + + .action-list ul li::before { + content: ''; + display: block; + position: absolute; + top: calc(var(--spacing) / -2); + left: -2px; + width: calc(var(--spacing) + 2px); + height: calc(var(--spacing) + 1px); + border: solid #ddd; + border-width: 0 0 2px 2px; + } + + .action-list summary { + display: block; + cursor: pointer; + } + + .action-list summary::marker, + .action-list summary::-webkit-details-marker { + display: none; + } + + .action-list summary:focus { + outline: none; + } + + .action-list summary:focus-visible { + outline: 1px dotted #000; + } + + .action-list li::after, + .action-list summary::before { + content: ''; + display: block; + position: absolute; + top: calc(var(--spacing) / 2 - var(--radius) + 4px); + left: calc(var(--spacing) - var(--radius) - -5px); + } + + .action-list summary::before { + z-index: 1; + /* background: #696 url('expand-collapse.svg') 0 0; */ + } + + .action-list details[open]>summary::before { + background-position: calc(-2 * var(--radius)) 0; + } + + .action-list li.danger-item::after, + .action-list li.danger-strong-item::after { + content: '❌'; + } + + ol.action-list li span.danger-strong-item { + text-decoration-style: solid; + text-decoration-color: red; + text-decoration-line: underline; + } + + ol.action-list li.warn-item::after { + content: '⚠️'; + } + + ol.action-list li.success-item::after { + content: '✅'; + } + + ol.action-list li.info-item::after { + content: '🔹'; + } + + /* End Attribution */ + </style> +</head> + +<body> + <div> + <header class="flex-row"> + + {{^IF isInReportingMode}} + <p>Jellyfin Server still starting. Please wait.</p> + {{#ELSE}} + <p>Jellyfin Server has encountered an error and was not able to start.</p> + {{/ELSE}} + {{/IF}} + + {{#IF localNetworkRequest}} + <p style="margin-left: 1rem;">You can download the current log file <a href='/startup/logger' + target="_blank">here</a>.</p> + {{/IF}} + </header> + + {{#DECLARE LogEntry |--}} + {{#LET children = Children}} + <li class="{{FormatLogLevel(children).ToString()}}-item"> + {{--| #IF children.Count > 0}} + <details open> + <summary>{{DateOfCreation}} - {{Content}}</summary> + <ul class="action-list"> + {{--| #EACH children.Reverse() |-}} + {{#IMPORT 'LogEntry'}} + {{--| /EACH |-}} + </ul> + </details> + {{--| #ELSE |-}} + <span class="{{FormatLogLevel(children).ToString()}}-item">{{DateOfCreation}} - {{Content}}</span> + {{--| /ELSE |--}} + {{--| /IF |-}} + </li> + {{--| /DECLARE}} + + <div class="flex-col"> + <ol class="action-list"> + {{#FOREACH log IN logs.Reverse()}} + {{#IMPORT 'LogEntry' #WITH log}} + {{/FOREACH}} + </ol> + </div> + </div> +</body> + +{{^IF isInReportingMode}} +<script> + setTimeout(() => { + window.location.reload(); + }, {{ retryValue.TotalMilliseconds }}); +</script> +{{/IF}} + +</html> diff --git a/Jellyfin.Server/Startup.cs b/Jellyfin.Server/Startup.cs index 688b16935..aa8f6dd1c 100644 --- a/Jellyfin.Server/Startup.cs +++ b/Jellyfin.Server/Startup.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Net; using System.Net.Http; using System.Net.Http.Headers; @@ -29,6 +30,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Primitives; using Prometheus; namespace Jellyfin.Server @@ -195,7 +197,14 @@ namespace Jellyfin.Server { FileProvider = new PhysicalFileProvider(_serverConfigurationManager.ApplicationPaths.WebPath), RequestPath = "/web", - ContentTypeProvider = extensionProvider + ContentTypeProvider = extensionProvider, + OnPrepareResponse = (context) => + { + if (Path.GetFileName(context.File.Name).Equals("index.html", StringComparison.Ordinal)) + { + context.Context.Response.Headers.CacheControl = new StringValues("no-cache"); + } + } }); mainApp.UseRobotsRedirection(); @@ -208,7 +217,6 @@ namespace Jellyfin.Server mainApp.UseRouting(); mainApp.UseAuthorization(); - mainApp.UseLanFiltering(); mainApp.UseIPBasedAccessValidation(); mainApp.UseWebSocketHandler(); mainApp.UseServerStartupMessage(); diff --git a/MediaBrowser.Common/Extensions/HttpContextExtensions.cs b/MediaBrowser.Common/Extensions/HttpContextExtensions.cs index a1056b7c8..739a53c7a 100644 --- a/MediaBrowser.Common/Extensions/HttpContextExtensions.cs +++ b/MediaBrowser.Common/Extensions/HttpContextExtensions.cs @@ -12,7 +12,7 @@ namespace MediaBrowser.Common.Extensions /// Checks the origin of the HTTP context. /// </summary> /// <param name="context">The incoming HTTP context.</param> - /// <returns><c>true</c> if the request is coming from LAN, <c>false</c> otherwise.</returns> + /// <returns><c>true</c> if the request is coming from the same machine as is running the server, <c>false</c> otherwise.</returns> public static bool IsLocal(this HttpContext context) { return (context.Connection.LocalIpAddress is null diff --git a/MediaBrowser.Common/Net/INetworkManager.cs b/MediaBrowser.Common/Net/INetworkManager.cs index d838144ff..bd785bcbc 100644 --- a/MediaBrowser.Common/Net/INetworkManager.cs +++ b/MediaBrowser.Common/Net/INetworkManager.cs @@ -127,7 +127,7 @@ namespace MediaBrowser.Common.Net /// Checks if <paramref name="remoteIP"/> has access to the server. /// </summary> /// <param name="remoteIP">IP address of the client.</param> - /// <returns><b>True</b> if it has access, otherwise <b>false</b>.</returns> - bool HasRemoteAccess(IPAddress remoteIP); + /// <returns>The result of evaluating the access policy, <c>Allow</c> if it should be allowed.</returns> + RemoteAccessPolicyResult ShouldAllowServerAccess(IPAddress remoteIP); } } diff --git a/MediaBrowser.Common/Net/RemoteAccessPolicyResult.cs b/MediaBrowser.Common/Net/RemoteAccessPolicyResult.cs new file mode 100644 index 000000000..193d37228 --- /dev/null +++ b/MediaBrowser.Common/Net/RemoteAccessPolicyResult.cs @@ -0,0 +1,29 @@ +using System; + +namespace MediaBrowser.Common.Net; + +/// <summary> +/// Result of <see cref="INetworkManager.ShouldAllowServerAccess" />. +/// </summary> +public enum RemoteAccessPolicyResult +{ + /// <summary> + /// The connection should be allowed. + /// </summary> + Allow, + + /// <summary> + /// The connection should be rejected since it is not from a local IP and remote access is disabled. + /// </summary> + RejectDueToRemoteAccessDisabled, + + /// <summary> + /// The connection should be rejected since it is from a blocklisted IP. + /// </summary> + RejectDueToIPBlocklist, + + /// <summary> + /// The connection should be rejected since it is from a remote IP that is not in the allowlist. + /// </summary> + RejectDueToNotAllowlistedRemoteIP, +} diff --git a/MediaBrowser.Controller/Entities/TV/Series.cs b/MediaBrowser.Controller/Entities/TV/Series.cs index b4ad05921..62c73d56f 100644 --- a/MediaBrowser.Controller/Entities/TV/Series.cs +++ b/MediaBrowser.Controller/Entities/TV/Series.cs @@ -214,7 +214,7 @@ namespace MediaBrowser.Controller.Entities.TV query.AncestorWithPresentationUniqueKey = null; query.SeriesPresentationUniqueKey = seriesKey; query.IncludeItemTypes = new[] { BaseItemKind.Season }; - query.OrderBy = new[] { (ItemSortBy.SortName, SortOrder.Ascending) }; + query.OrderBy = new[] { (ItemSortBy.IndexNumber, SortOrder.Ascending) }; if (user is not null && !user.DisplayMissingEpisodes) { @@ -247,10 +247,6 @@ namespace MediaBrowser.Controller.Entities.TV query.AncestorWithPresentationUniqueKey = null; query.SeriesPresentationUniqueKey = seriesKey; - if (query.OrderBy.Count == 0) - { - query.OrderBy = new[] { (ItemSortBy.SortName, SortOrder.Ascending) }; - } if (query.IncludeItemTypes.Length == 0) { diff --git a/MediaBrowser.Controller/Entities/UserViewBuilder.cs b/MediaBrowser.Controller/Entities/UserViewBuilder.cs index c2b4da32a..1eb3c8f50 100644 --- a/MediaBrowser.Controller/Entities/UserViewBuilder.cs +++ b/MediaBrowser.Controller/Entities/UserViewBuilder.cs @@ -924,6 +924,11 @@ namespace MediaBrowser.Controller.Entities } } + if (query.ExcludeItemIds.Contains(item.Id)) + { + return false; + } + return true; } diff --git a/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs b/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs index 228839a1d..fc5a109f1 100644 --- a/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs +++ b/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs @@ -21,4 +21,9 @@ public class BackupOptionsDto /// Gets or sets a value indicating whether the archive contains the Subtitle contents. /// </summary> public bool Subtitles { get; set; } + + /// <summary> + /// Gets or sets a value indicating whether the archive contains the Database contents. + /// </summary> + public bool Database { get; set; } = true; } diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index 5ea21fa63..8eae6af72 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -86,7 +86,7 @@ namespace MediaBrowser.Providers.Manager var itemOfType = (TItemType)item; var updateType = ItemUpdateType.None; var libraryOptions = LibraryManager.GetLibraryOptions(item); - var isFirstRefresh = item.DateLastRefreshed == default; + var isFirstRefresh = item.DateLastRefreshed.Date == DateTime.MinValue.Date; var hasRefreshedMetadata = true; var hasRefreshedImages = true; @@ -650,7 +650,7 @@ namespace MediaBrowser.Providers.Manager var dateLastImageRefresh = item.DateLastRefreshed; // Run all if either of these flags are true - var runAllProviders = options.ImageRefreshMode == MetadataRefreshMode.FullRefresh || dateLastImageRefresh == default(DateTime); + var runAllProviders = options.ImageRefreshMode == MetadataRefreshMode.FullRefresh || dateLastImageRefresh.Date == DateTime.MinValue.Date; if (!runAllProviders) { diff --git a/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs b/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs index 4310f93d4..81dcbf893 100644 --- a/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs +++ b/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs @@ -59,14 +59,14 @@ public class TrickplayImagesTask : IScheduledTask /// <inheritdoc /> public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() { - return new[] - { + return + [ new TaskTriggerInfo { Type = TaskTriggerInfoType.DailyTrigger, TimeOfDayTicks = TimeSpan.FromHours(3).Ticks } - }; + ]; } /// <inheritdoc /> @@ -74,8 +74,8 @@ public class TrickplayImagesTask : IScheduledTask { var query = new InternalItemsQuery { - MediaTypes = new[] { MediaType.Video }, - SourceTypes = new[] { SourceType.Library }, + MediaTypes = [MediaType.Video], + SourceTypes = [SourceType.Library], IsVirtualItem = false, IsFolder = false, Recursive = true, diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOption.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOption.cs new file mode 100644 index 000000000..fcb8f41b3 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOption.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; + +namespace Jellyfin.Database.Implementations.DbConfiguration; + +/// <summary> +/// The custom value option for custom database providers. +/// </summary> +public class CustomDatabaseOption +{ + /// <summary> + /// Gets or sets the key of the value. + /// </summary> + public required string Key { get; set; } + + /// <summary> + /// Gets or sets the value. + /// </summary> + public required string Value { get; set; } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOptions.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOptions.cs new file mode 100644 index 000000000..e2088704d --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOptions.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace Jellyfin.Database.Implementations.DbConfiguration; + +/// <summary> +/// Defines the options for a custom database connector. +/// </summary> +public class CustomDatabaseOptions +{ + /// <summary> + /// Gets or sets the Plugin name to search for database providers. + /// </summary> + public required string PluginName { get; set; } + + /// <summary> + /// Gets or sets the plugin assembly to search for providers. + /// </summary> + public required string PluginAssembly { get; set; } + + /// <summary> + /// Gets or sets the connection string for the custom database provider. + /// </summary> + public required string ConnectionString { get; set; } + + /// <summary> + /// Gets or sets the list of extra options for the custom provider. + /// </summary> +#pragma warning disable CA2227 // Collection properties should be read only + public Collection<CustomDatabaseOption> Options { get; set; } = []; +#pragma warning restore CA2227 // Collection properties should be read only +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs index b481a106f..bc0cacf3c 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; + namespace Jellyfin.Database.Implementations.DbConfiguration; /// <summary> @@ -9,4 +11,15 @@ public class DatabaseConfigurationOptions /// Gets or Sets the type of database jellyfin should use. /// </summary> public required string DatabaseType { get; set; } + + /// <summary> + /// Gets or sets the options required to use a custom database provider. + /// </summary> + public CustomDatabaseOptions? CustomProviderOptions { get; set; } + + /// <summary> + /// Gets or Sets the kind of locking behavior jellyfin should perform. Possible options are "NoLock", "Pessimistic", "Optimistic". + /// Defaults to "NoLock". + /// </summary> + public DatabaseLockingBehaviorTypes LockingBehavior { get; set; } } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseLockingBehaviorTypes.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseLockingBehaviorTypes.cs new file mode 100644 index 000000000..3b2a55802 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseLockingBehaviorTypes.cs @@ -0,0 +1,22 @@ +namespace Jellyfin.Database.Implementations.DbConfiguration; + +/// <summary> +/// Defines all possible methods for locking database access for concurrent queries. +/// </summary> +public enum DatabaseLockingBehaviorTypes +{ + /// <summary> + /// Defines that no explicit application level locking for reads and writes should be done and only provider specific locking should be relied on. + /// </summary> + NoLock = 0, + + /// <summary> + /// Defines a behavior that always blocks all reads while any one write is done. + /// </summary> + Pessimistic = 1, + + /// <summary> + /// Defines that all writes should be attempted and when fail should be retried. + /// </summary> + Optimistic = 2 +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs index b0dc98469..6b35810b2 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs @@ -65,6 +65,13 @@ public interface IJellyfinDatabaseProvider Task RestoreBackupFast(string key, CancellationToken cancellationToken); /// <summary> + /// Deletes a backup that has been previously created by <see cref="MigrationBackupFast(CancellationToken)"/>. + /// </summary> + /// <param name="key">The key to the backup which should be cleaned up.</param> + /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns> + Task DeleteBackup(string key); + + /// <summary> /// Removes all contents from the database. /// </summary> /// <param name="dbContext">The Database context.</param> diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj index 356f96fc9..28c4972d2 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj @@ -24,6 +24,7 @@ </PropertyGroup> <ItemGroup> + <PackageReference Include="Polly" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Design"> <PrivateAssets>all</PrivateAssets> diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs index 35ad461ec..5163bff8b 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs @@ -1,9 +1,14 @@ using System; +using System.Data.Common; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Database.Implementations.Entities.Security; using Jellyfin.Database.Implementations.Interfaces; +using Jellyfin.Database.Implementations.Locking; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Logging; namespace Jellyfin.Database.Implementations; @@ -15,7 +20,8 @@ namespace Jellyfin.Database.Implementations; /// <param name="options">The database context options.</param> /// <param name="logger">Logger.</param> /// <param name="jellyfinDatabaseProvider">The provider for the database engine specific operations.</param> -public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILogger<JellyfinDbContext> logger, IJellyfinDatabaseProvider jellyfinDatabaseProvider) : DbContext(options) +/// <param name="entityFrameworkCoreLocking">The locking behavior.</param> +public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILogger<JellyfinDbContext> logger, IJellyfinDatabaseProvider jellyfinDatabaseProvider, IEntityFrameworkCoreLockingBehavior entityFrameworkCoreLocking) : DbContext(options) { /// <summary> /// Gets the <see cref="DbSet{TEntity}"/> containing the access schedules. @@ -247,19 +253,41 @@ public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILog public DbSet<TrackMetadata> TrackMetadata => Set<TrackMetadata>();*/ /// <inheritdoc/> - public override int SaveChanges() + public override async Task<int> SaveChangesAsync( + bool acceptAllChangesOnSuccess, + CancellationToken cancellationToken = default) { - foreach (var saveEntity in ChangeTracker.Entries() - .Where(e => e.State == EntityState.Modified) - .Select(entry => entry.Entity) - .OfType<IHasConcurrencyToken>()) + HandleConcurrencyToken(); + + try { - saveEntity.OnSavingChanges(); + var result = -1; + await entityFrameworkCoreLocking.OnSaveChangesAsync(this, async () => + { + result = await base.SaveChangesAsync(acceptAllChangesOnSuccess, cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + return result; } + catch (Exception e) + { + logger.LogError(e, "Error trying to save changes."); + throw; + } + } + + /// <inheritdoc/> + public override int SaveChanges(bool acceptAllChangesOnSuccess) // SaveChanges(bool) is beeing called by SaveChanges() with default to false. + { + HandleConcurrencyToken(); try { - return base.SaveChanges(); + var result = -1; + entityFrameworkCoreLocking.OnSaveChanges(this, () => + { + result = base.SaveChanges(acceptAllChangesOnSuccess); + }); + return result; } catch (Exception e) { @@ -268,6 +296,17 @@ public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILog } } + private void HandleConcurrencyToken() + { + foreach (var saveEntity in ChangeTracker.Entries() + .Where(e => e.State == EntityState.Modified) + .Select(entry => entry.Entity) + .OfType<IHasConcurrencyToken>()) + { + saveEntity.OnSavingChanges(); + } + } + /// <inheritdoc /> protected override void OnModelCreating(ModelBuilder modelBuilder) { diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/IEntityFrameworkCoreLockingBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/IEntityFrameworkCoreLockingBehavior.cs new file mode 100644 index 000000000..465c31212 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/IEntityFrameworkCoreLockingBehavior.cs @@ -0,0 +1,32 @@ +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// Defines a jellyfin locking behavior that can be configured. +/// </summary> +public interface IEntityFrameworkCoreLockingBehavior +{ + /// <summary> + /// Provides access to the builder to setup any connection related locking behavior. + /// </summary> + /// <param name="optionsBuilder">The options builder.</param> + void Initialise(DbContextOptionsBuilder optionsBuilder); + + /// <summary> + /// Will be invoked when changes should be saved in the current locking behavior. + /// </summary> + /// <param name="context">The database context invoking the action.</param> + /// <param name="saveChanges">Callback for performing the actual save changes.</param> + void OnSaveChanges(JellyfinDbContext context, Action saveChanges); + + /// <summary> + /// Will be invoked when changes should be saved in the current locking behavior. + /// </summary> + /// <param name="context">The database context invoking the action.</param> + /// <param name="saveChanges">Callback for performing the actual save changes.</param> + /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> + Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges); +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/NoLockBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/NoLockBehavior.cs new file mode 100644 index 000000000..3b654f4c4 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/NoLockBehavior.cs @@ -0,0 +1,41 @@ +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// Default lock behavior. Defines no explicit application locking behavior. +/// </summary> +public class NoLockBehavior : IEntityFrameworkCoreLockingBehavior +{ + private readonly ILogger<NoLockBehavior> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="NoLockBehavior"/> class. + /// </summary> + /// <param name="logger">The Application logger.</param> + public NoLockBehavior(ILogger<NoLockBehavior> logger) + { + _logger = logger; + } + + /// <inheritdoc/> + public void OnSaveChanges(JellyfinDbContext context, Action saveChanges) + { + saveChanges(); + } + + /// <inheritdoc/> + public void Initialise(DbContextOptionsBuilder optionsBuilder) + { + _logger.LogInformation("The database locking mode has been set to: NoLock."); + } + + /// <inheritdoc/> + public async Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges) + { + await saveChanges().ConfigureAwait(false); + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs new file mode 100644 index 000000000..9395b2e2d --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs @@ -0,0 +1,137 @@ +using System; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; +using Polly; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// Defines a locking mechanism that will retry any write operation for a few times. +/// </summary> +public class OptimisticLockBehavior : IEntityFrameworkCoreLockingBehavior +{ + private readonly Policy _writePolicy; + private readonly AsyncPolicy _writeAsyncPolicy; + private readonly ILogger<OptimisticLockBehavior> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="OptimisticLockBehavior"/> class. + /// </summary> + /// <param name="logger">The application logger.</param> + public OptimisticLockBehavior(ILogger<OptimisticLockBehavior> logger) + { + TimeSpan[] sleepDurations = [ + TimeSpan.FromMilliseconds(50), + TimeSpan.FromMilliseconds(50), + TimeSpan.FromMilliseconds(250), + TimeSpan.FromMilliseconds(150), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromSeconds(3) + ]; + _logger = logger; + _writePolicy = Policy.HandleInner<Exception>(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)).WaitAndRetry(sleepDurations, RetryHandle); + _writeAsyncPolicy = Policy.HandleInner<Exception>(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)).WaitAndRetryAsync(sleepDurations, RetryHandle); + + void RetryHandle(Exception exception, TimeSpan timespan, int retryNo, Context context) + { + if (retryNo < sleepDurations.Length) + { + _logger.LogWarning("Operation failed retry {RetryNo}", retryNo); + } + else + { + _logger.LogError(exception, "Operation failed retry {RetryNo}", retryNo); + } + } + } + + /// <inheritdoc/> + public void Initialise(DbContextOptionsBuilder optionsBuilder) + { + _logger.LogInformation("The database locking mode has been set to: Optimistic."); + optionsBuilder.AddInterceptors(new RetryInterceptor(_writeAsyncPolicy, _writePolicy)); + optionsBuilder.AddInterceptors(new TransactionLockingInterceptor(_writeAsyncPolicy, _writePolicy)); + } + + /// <inheritdoc/> + public void OnSaveChanges(JellyfinDbContext context, Action saveChanges) + { + _writePolicy.ExecuteAndCapture(saveChanges); + } + + /// <inheritdoc/> + public async Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges) + { + await _writeAsyncPolicy.ExecuteAndCaptureAsync(saveChanges).ConfigureAwait(false); + } + + private sealed class TransactionLockingInterceptor : DbTransactionInterceptor + { + private readonly AsyncPolicy _asyncRetryPolicy; + private readonly Policy _retryPolicy; + + public TransactionLockingInterceptor(AsyncPolicy asyncRetryPolicy, Policy retryPolicy) + { + _asyncRetryPolicy = asyncRetryPolicy; + _retryPolicy = retryPolicy; + } + + public override InterceptionResult<DbTransaction> TransactionStarting(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result) + { + return InterceptionResult<DbTransaction>.SuppressWithResult(_retryPolicy.Execute(() => connection.BeginTransaction(eventData.IsolationLevel))); + } + + public override async ValueTask<InterceptionResult<DbTransaction>> TransactionStartingAsync(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<DbTransaction>.SuppressWithResult(await _asyncRetryPolicy.ExecuteAsync(async () => await connection.BeginTransactionAsync(eventData.IsolationLevel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false)); + } + } + + private sealed class RetryInterceptor : DbCommandInterceptor + { + private readonly AsyncPolicy _asyncRetryPolicy; + private readonly Policy _retryPolicy; + + public RetryInterceptor(AsyncPolicy asyncRetryPolicy, Policy retryPolicy) + { + _asyncRetryPolicy = asyncRetryPolicy; + _retryPolicy = retryPolicy; + } + + public override InterceptionResult<int> NonQueryExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<int> result) + { + return InterceptionResult<int>.SuppressWithResult(_retryPolicy.Execute(command.ExecuteNonQuery)); + } + + public override async ValueTask<InterceptionResult<int>> NonQueryExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<int> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<int>.SuppressWithResult(await _asyncRetryPolicy.ExecuteAsync(async () => await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false)); + } + + public override InterceptionResult<object> ScalarExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<object> result) + { + return InterceptionResult<object>.SuppressWithResult(_retryPolicy.Execute(() => command.ExecuteScalar()!)); + } + + public override async ValueTask<InterceptionResult<object>> ScalarExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<object> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<object>.SuppressWithResult((await _asyncRetryPolicy.ExecuteAsync(async () => await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false)!).ConfigureAwait(false))!); + } + + public override InterceptionResult<DbDataReader> ReaderExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(_retryPolicy.Execute(command.ExecuteReader)); + } + + public override async ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(await _asyncRetryPolicy.ExecuteAsync(async () => await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false)); + } + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/PessimisticLockBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/PessimisticLockBehavior.cs new file mode 100644 index 000000000..2d6bc6902 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/PessimisticLockBehavior.cs @@ -0,0 +1,296 @@ +#pragma warning disable MT1013 // Releasing lock without guarantee of execution +#pragma warning disable MT1012 // Acquiring lock without guarantee of releasing + +using System; +using System.Data; +using System.Data.Common; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// A locking behavior that will always block any operation while a write is requested. Mimicks the old SqliteRepository behavior. +/// </summary> +public class PessimisticLockBehavior : IEntityFrameworkCoreLockingBehavior +{ + private readonly ILogger<PessimisticLockBehavior> _logger; + private readonly ILoggerFactory _loggerFactory; + + /// <summary> + /// Initializes a new instance of the <see cref="PessimisticLockBehavior"/> class. + /// </summary> + /// <param name="logger">The application logger.</param> + /// <param name="loggerFactory">The logger factory.</param> + public PessimisticLockBehavior(ILogger<PessimisticLockBehavior> logger, ILoggerFactory loggerFactory) + { + _logger = logger; + _loggerFactory = loggerFactory; + } + + private static ReaderWriterLockSlim DatabaseLock { get; } = new(LockRecursionPolicy.SupportsRecursion); + + /// <inheritdoc/> + public void OnSaveChanges(JellyfinDbContext context, Action saveChanges) + { + using (DbLock.EnterWrite(_logger)) + { + saveChanges(); + } + } + + /// <inheritdoc/> + public void Initialise(DbContextOptionsBuilder optionsBuilder) + { + _logger.LogInformation("The database locking mode has been set to: Pessimistic."); + optionsBuilder.AddInterceptors(new CommandLockingInterceptor(_loggerFactory.CreateLogger<CommandLockingInterceptor>())); + optionsBuilder.AddInterceptors(new TransactionLockingInterceptor(_loggerFactory.CreateLogger<TransactionLockingInterceptor>())); + } + + /// <inheritdoc/> + public async Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges) + { + using (DbLock.EnterWrite(_logger)) + { + await saveChanges().ConfigureAwait(false); + } + } + + private sealed class TransactionLockingInterceptor : DbTransactionInterceptor + { + private readonly ILogger _logger; + + public TransactionLockingInterceptor(ILogger logger) + { + _logger = logger; + } + + public override InterceptionResult<DbTransaction> TransactionStarting(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result) + { + DbLock.BeginWriteLock(_logger); + + return base.TransactionStarting(connection, eventData, result); + } + + public override ValueTask<InterceptionResult<DbTransaction>> TransactionStartingAsync(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result, CancellationToken cancellationToken = default) + { + DbLock.BeginWriteLock(_logger); + + return base.TransactionStartingAsync(connection, eventData, result, cancellationToken); + } + + public override void TransactionCommitted(DbTransaction transaction, TransactionEndEventData eventData) + { + DbLock.EndWriteLock(_logger); + + base.TransactionCommitted(transaction, eventData); + } + + public override Task TransactionCommittedAsync(DbTransaction transaction, TransactionEndEventData eventData, CancellationToken cancellationToken = default) + { + DbLock.EndWriteLock(_logger); + + return base.TransactionCommittedAsync(transaction, eventData, cancellationToken); + } + + public override void TransactionFailed(DbTransaction transaction, TransactionErrorEventData eventData) + { + DbLock.EndWriteLock(_logger); + + base.TransactionFailed(transaction, eventData); + } + + public override Task TransactionFailedAsync(DbTransaction transaction, TransactionErrorEventData eventData, CancellationToken cancellationToken = default) + { + DbLock.EndWriteLock(_logger); + + return base.TransactionFailedAsync(transaction, eventData, cancellationToken); + } + + public override void TransactionRolledBack(DbTransaction transaction, TransactionEndEventData eventData) + { + DbLock.EndWriteLock(_logger); + + base.TransactionRolledBack(transaction, eventData); + } + + public override Task TransactionRolledBackAsync(DbTransaction transaction, TransactionEndEventData eventData, CancellationToken cancellationToken = default) + { + DbLock.EndWriteLock(_logger); + + return base.TransactionRolledBackAsync(transaction, eventData, cancellationToken); + } + } + + /// <summary> + /// Adds strict read/write locking. + /// </summary> + private sealed class CommandLockingInterceptor : DbCommandInterceptor + { + private readonly ILogger _logger; + + public CommandLockingInterceptor(ILogger logger) + { + _logger = logger; + } + + public override InterceptionResult<int> NonQueryExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<int> result) + { + using (DbLock.EnterWrite(_logger, command)) + { + return InterceptionResult<int>.SuppressWithResult(command.ExecuteNonQuery()); + } + } + + public override async ValueTask<InterceptionResult<int>> NonQueryExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<int> result, CancellationToken cancellationToken = default) + { + using (DbLock.EnterWrite(_logger, command)) + { + return InterceptionResult<int>.SuppressWithResult(await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false)); + } + } + + public override InterceptionResult<object> ScalarExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<object> result) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<object>.SuppressWithResult(command.ExecuteScalar()!); + } + } + + public override async ValueTask<InterceptionResult<object>> ScalarExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<object> result, CancellationToken cancellationToken = default) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<object>.SuppressWithResult((await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!); + } + } + + public override InterceptionResult<DbDataReader> ReaderExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(command.ExecuteReader()!); + } + } + + public override async ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result, CancellationToken cancellationToken = default) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)); + } + } + } + + private sealed class DbLock : IDisposable + { + private readonly Action? _action; + private bool _disposed; + + private static readonly IDisposable _noLock = new DbLock(null) { _disposed = true }; + private static (string Command, Guid Id, DateTimeOffset QueryDate, bool Printed) _blockQuery; + + public DbLock(Action? action = null) + { + _action = action; + } + +#pragma warning disable IDISP015 // Member should not return created and cached instance + public static IDisposable EnterWrite(ILogger logger, IDbCommand? command = null, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) +#pragma warning restore IDISP015 // Member should not return created and cached instance + { + logger.LogTrace("Enter Write for {Caller}:{Line}", callerMemberName, callerNo); + if (DatabaseLock.IsWriteLockHeld) + { + logger.LogTrace("Write Held {Caller}:{Line}", callerMemberName, callerNo); + return _noLock; + } + + BeginWriteLock(logger, command, callerMemberName, callerNo); + return new DbLock(() => + { + EndWriteLock(logger, callerMemberName, callerNo); + }); + } + +#pragma warning disable IDISP015 // Member should not return created and cached instance + public static IDisposable EnterRead(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) +#pragma warning restore IDISP015 // Member should not return created and cached instance + { + logger.LogTrace("Enter Read {Caller}:{Line}", callerMemberName, callerNo); + if (DatabaseLock.IsWriteLockHeld) + { + logger.LogTrace("Write Held {Caller}:{Line}", callerMemberName, callerNo); + return _noLock; + } + + BeginReadLock(logger, callerMemberName, callerNo); + return new DbLock(() => + { + ExitReadLock(logger, callerMemberName, callerNo); + }); + } + + public static void BeginWriteLock(ILogger logger, IDbCommand? command = null, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Aquire Write {Caller}:{Line}", callerMemberName, callerNo); + if (!DatabaseLock.TryEnterWriteLock(TimeSpan.FromMilliseconds(1000))) + { + var blockingQuery = _blockQuery; + if (!blockingQuery.Printed) + { + _blockQuery = (blockingQuery.Command, blockingQuery.Id, blockingQuery.QueryDate, true); + logger.LogInformation("QueryLock: {Id} --- {Query}", blockingQuery.Id, blockingQuery.Command); + } + + logger.LogInformation("Query congestion detected: '{Id}' since '{Date}'", blockingQuery.Id, blockingQuery.QueryDate); + + DatabaseLock.EnterWriteLock(); + + logger.LogInformation("Query congestion cleared: '{Id}' for '{Date}'", blockingQuery.Id, DateTimeOffset.Now - blockingQuery.QueryDate); + } + + _blockQuery = (command?.CommandText ?? "Transaction", Guid.NewGuid(), DateTimeOffset.Now, false); + + logger.LogTrace("Write Aquired {Caller}:{Line}", callerMemberName, callerNo); + } + + public static void BeginReadLock(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Aquire Write {Caller}:{Line}", callerMemberName, callerNo); + DatabaseLock.EnterReadLock(); + logger.LogTrace("Read Aquired {Caller}:{Line}", callerMemberName, callerNo); + } + + public static void EndWriteLock(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Release Write {Caller}:{Line}", callerMemberName, callerNo); + DatabaseLock.ExitWriteLock(); + } + + public static void ExitReadLock(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Release Read {Caller}:{Line}", callerMemberName, callerNo); + DatabaseLock.ExitReadLock(); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + if (_action is not null) + { + _action(); + } + } + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ProgressablePartitionReporting.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ProgressablePartitionReporting.cs new file mode 100644 index 000000000..7654dd3c5 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ProgressablePartitionReporting.cs @@ -0,0 +1,55 @@ +using System; +using System.Diagnostics; +using System.Linq; + +namespace Jellyfin.Database.Implementations; + +/// <summary> +/// Wrapper for progress reporting on Partition helpers. +/// </summary> +/// <typeparam name="TEntity">The entity to load.</typeparam> +public class ProgressablePartitionReporting<TEntity> +{ + private readonly IOrderedQueryable<TEntity> _source; + + private readonly Stopwatch _partitionTime = new(); + + private readonly Stopwatch _itemTime = new(); + + internal ProgressablePartitionReporting(IOrderedQueryable<TEntity> source) + { + _source = source; + } + + internal Action<TEntity, int, int>? OnBeginItem { get; set; } + + internal Action<int>? OnBeginPartition { get; set; } + + internal Action<TEntity, int, int, TimeSpan>? OnEndItem { get; set; } + + internal Action<int, TimeSpan>? OnEndPartition { get; set; } + + internal IOrderedQueryable<TEntity> Source => _source; + + internal void BeginItem(TEntity entity, int iteration, int itemIndex) + { + _itemTime.Restart(); + OnBeginItem?.Invoke(entity, iteration, itemIndex); + } + + internal void BeginPartition(int iteration) + { + _partitionTime.Restart(); + OnBeginPartition?.Invoke(iteration); + } + + internal void EndItem(TEntity entity, int iteration, int itemIndex) + { + OnEndItem?.Invoke(entity, iteration, itemIndex, _itemTime.Elapsed); + } + + internal void EndPartition(int iteration) + { + OnEndPartition?.Invoke(iteration, _partitionTime.Elapsed); + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/QueryPartitionHelpers.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/QueryPartitionHelpers.cs new file mode 100644 index 000000000..bb66bddca --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/QueryPartitionHelpers.cs @@ -0,0 +1,215 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Jellyfin.Database.Implementations; + +/// <summary> +/// Contains helpers to partition EFCore queries. +/// </summary> +public static class QueryPartitionHelpers +{ + /// <summary> + /// Adds a callback to any directly following calls of Partition for every partition thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="beginPartition">The callback invoked for partition before enumerating items.</param> + /// <param name="endPartition">The callback invoked for partition after enumerating items.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithPartitionProgress<TEntity>(this IOrderedQueryable<TEntity> query, Action<int>? beginPartition = null, Action<int, TimeSpan>? endPartition = null) + { + var progressable = new ProgressablePartitionReporting<TEntity>(query); + progressable.OnBeginPartition = beginPartition; + progressable.OnEndPartition = endPartition; + return progressable; + } + + /// <summary> + /// Adds a callback to any directly following calls of Partition for every item thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="beginItem">The callback invoked for each item before processing.</param> + /// <param name="endItem">The callback invoked for each item after processing.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithItemProgress<TEntity>(this IOrderedQueryable<TEntity> query, Action<TEntity, int, int>? beginItem = null, Action<TEntity, int, int, TimeSpan>? endItem = null) + { + var progressable = new ProgressablePartitionReporting<TEntity>(query); + progressable.OnBeginItem = beginItem; + progressable.OnEndItem = endItem; + return progressable; + } + + /// <summary> + /// Adds a callback to any directly following calls of Partition for every partition thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="progressable">The source query.</param> + /// <param name="beginPartition">The callback invoked for partition before enumerating items.</param> + /// <param name="endPartition">The callback invoked for partition after enumerating items.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithPartitionProgress<TEntity>(this ProgressablePartitionReporting<TEntity> progressable, Action<int>? beginPartition = null, Action<int, TimeSpan>? endPartition = null) + { + progressable.OnBeginPartition = beginPartition; + progressable.OnEndPartition = endPartition; + return progressable; + } + + /// <summary> + /// Adds a callback to any directly following calls of Partition for every item thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="progressable">The source query.</param> + /// <param name="beginItem">The callback invoked for each item before processing.</param> + /// <param name="endItem">The callback invoked for each item after processing.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithItemProgress<TEntity>(this ProgressablePartitionReporting<TEntity> progressable, Action<TEntity, int, int>? beginItem = null, Action<TEntity, int, int, TimeSpan>? endItem = null) + { + progressable.OnBeginItem = beginItem; + progressable.OnEndItem = endItem; + return progressable; + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions in a lazy manner reading each item from the database as its requested. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="partitionInfo">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionAsync<TEntity>(this ProgressablePartitionReporting<TEntity> partitionInfo, int partitionSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var item in partitionInfo.Source.PartitionAsync(partitionSize, partitionInfo, cancellationToken).ConfigureAwait(false)) + { + yield return item; + } + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions directly into memory. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="partitionInfo">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionEagerAsync<TEntity>(this ProgressablePartitionReporting<TEntity> partitionInfo, int partitionSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var item in partitionInfo.Source.PartitionEagerAsync(partitionSize, partitionInfo, cancellationToken).ConfigureAwait(false)) + { + yield return item; + } + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions in a lazy manner reading each item from the database as its requested. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="progressablePartition">Reporting helper.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionAsync<TEntity>( + this IOrderedQueryable<TEntity> query, + int partitionSize, + ProgressablePartitionReporting<TEntity>? progressablePartition = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var iterator = 0; + int itemCounter; + do + { + progressablePartition?.BeginPartition(iterator); + itemCounter = 0; + await foreach (var item in query + .Skip(partitionSize * iterator) + .Take(partitionSize) + .AsAsyncEnumerable() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + progressablePartition?.BeginItem(item, iterator, itemCounter); + yield return item; + progressablePartition?.EndItem(item, iterator, itemCounter); + itemCounter++; + } + + progressablePartition?.EndPartition(iterator); + iterator++; + } while (itemCounter == partitionSize && !cancellationToken.IsCancellationRequested); + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions directly into memory. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="progressablePartition">Reporting helper.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionEagerAsync<TEntity>( + this IOrderedQueryable<TEntity> query, + int partitionSize, + ProgressablePartitionReporting<TEntity>? progressablePartition = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var iterator = 0; + int itemCounter; + var items = ArrayPool<TEntity>.Shared.Rent(partitionSize); + try + { + do + { + progressablePartition?.BeginPartition(iterator); + itemCounter = 0; + await foreach (var item in query + .Skip(partitionSize * iterator) + .Take(partitionSize) + .AsAsyncEnumerable() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + items[itemCounter++] = item; + } + + for (int i = 0; i < itemCounter; i++) + { + progressablePartition?.BeginItem(items[i], iterator, itemCounter); + yield return items[i]; + progressablePartition?.EndItem(items[i], iterator, itemCounter); + } + + progressablePartition?.EndPartition(iterator); + iterator++; + } while (itemCounter == partitionSize && !cancellationToken.IsCancellationRequested); + } + finally + { + ArrayPool<TEntity>.Shared.Return(items); + } + } + + /// <summary> + /// Adds an Index to the enumeration of the async enumerable. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <returns>The source list with an index added.</returns> + public static async IAsyncEnumerable<(TEntity Item, int Index)> WithIndex<TEntity>(this IAsyncEnumerable<TEntity> query) + { + var index = 0; + await foreach (var item in query.ConfigureAwait(false)) + { + yield return (item, index++); + } + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/SqliteDesignTimeJellyfinDbFactory.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/SqliteDesignTimeJellyfinDbFactory.cs index 78815c311..4d420bf8c 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/SqliteDesignTimeJellyfinDbFactory.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/SqliteDesignTimeJellyfinDbFactory.cs @@ -1,4 +1,5 @@ using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Locking; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using Microsoft.Extensions.Logging.Abstractions; @@ -19,7 +20,8 @@ namespace Jellyfin.Database.Providers.Sqlite.Migrations return new JellyfinDbContext( optionsBuilder.Options, NullLogger<JellyfinDbContext>.Instance, - new SqliteDatabaseProvider(null!, NullLogger<SqliteDatabaseProvider>.Instance)); + new SqliteDatabaseProvider(null!, NullLogger<SqliteDatabaseProvider>.Instance), + new NoLockBehavior(NullLogger<NoLockBehavior>.Instance)); } } } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs index 519584003..dda1ca075 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs @@ -129,6 +129,21 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider return Task.CompletedTask; } + /// <inheritdoc /> + public Task DeleteBackup(string key) + { + var backupFile = Path.Combine(_applicationPaths.DataPath, BackupFolderName, $"{key}_jellyfin.db"); + + if (!File.Exists(backupFile)) + { + _logger.LogCritical("Tried to delete a backup that does not exist: {Key}", key); + return Task.CompletedTask; + } + + File.Delete(backupFile); + return Task.CompletedTask; + } + /// <inheritdoc/> public async Task PurgeDatabase(JellyfinDbContext dbContext, IEnumerable<string>? tableNames) { diff --git a/src/Jellyfin.Networking/Manager/NetworkManager.cs b/src/Jellyfin.Networking/Manager/NetworkManager.cs index 80a5741df..126d9f15c 100644 --- a/src/Jellyfin.Networking/Manager/NetworkManager.cs +++ b/src/Jellyfin.Networking/Manager/NetworkManager.cs @@ -690,33 +690,42 @@ public class NetworkManager : INetworkManager, IDisposable } /// <inheritdoc/> - public bool HasRemoteAccess(IPAddress remoteIP) + public RemoteAccessPolicyResult ShouldAllowServerAccess(IPAddress remoteIP) { var config = _configurationManager.GetNetworkConfiguration(); - if (config.EnableRemoteAccess) + if (IsInLocalNetwork(remoteIP)) { - // Comma separated list of IP addresses or IP/netmask entries for networks that will be allowed to connect remotely. - // If left blank, all remote addresses will be allowed. - if (_remoteAddressFilter.Any() && !IsInLocalNetwork(remoteIP)) - { - // remoteAddressFilter is a whitelist or blacklist. - var matches = _remoteAddressFilter.Count(remoteNetwork => NetworkUtils.SubnetContainsAddress(remoteNetwork, remoteIP)); - if ((!config.IsRemoteIPFilterBlacklist && matches > 0) - || (config.IsRemoteIPFilterBlacklist && matches == 0)) - { - return true; - } - - return false; - } + return RemoteAccessPolicyResult.Allow; } - else if (!IsInLocalNetwork(remoteIP)) + + if (!config.EnableRemoteAccess) { // Remote not enabled. So everyone should be LAN. - return false; + return RemoteAccessPolicyResult.RejectDueToRemoteAccessDisabled; } - return true; + if (!_remoteAddressFilter.Any()) + { + // No filter on remote addresses, allow any of them. + return RemoteAccessPolicyResult.Allow; + } + + // Comma separated list of IP addresses or IP/netmask entries for networks that will be allowed to connect remotely. + // If left blank, all remote addresses will be allowed. + + // remoteAddressFilter is a whitelist or blacklist. + var anyMatches = _remoteAddressFilter.Any(remoteNetwork => NetworkUtils.SubnetContainsAddress(remoteNetwork, remoteIP)); + if (config.IsRemoteIPFilterBlacklist) + { + return anyMatches + ? RemoteAccessPolicyResult.RejectDueToIPBlocklist + : RemoteAccessPolicyResult.Allow; + } + + // Allow-list + return anyMatches + ? RemoteAccessPolicyResult.Allow + : RemoteAccessPolicyResult.RejectDueToNotAllowlistedRemoteIP; } /// <inheritdoc/> diff --git a/tests/Jellyfin.LiveTv.Tests/Listings/XmlTvListingsProviderTests.cs b/tests/Jellyfin.LiveTv.Tests/Listings/XmlTvListingsProviderTests.cs index 0fb7894e5..b71dc1520 100644 --- a/tests/Jellyfin.LiveTv.Tests/Listings/XmlTvListingsProviderTests.cs +++ b/tests/Jellyfin.LiveTv.Tests/Listings/XmlTvListingsProviderTests.cs @@ -54,7 +54,7 @@ public class XmlTvListingsProviderTests Path = path }; - var startDate = new DateTime(2022, 11, 4); + var startDate = new DateTime(2022, 11, 4, 0, 0, 0, DateTimeKind.Utc); var programs = await _xmlTvListingsProvider.GetProgramsAsync(info, "3297", startDate, startDate.AddDays(1), CancellationToken.None); var programsList = programs.ToList(); Assert.Single(programsList); @@ -78,7 +78,7 @@ public class XmlTvListingsProviderTests Path = path }; - var startDate = new DateTime(2022, 11, 4); + var startDate = new DateTime(2022, 11, 4, 0, 0, 0, DateTimeKind.Utc); var programs = await _xmlTvListingsProvider.GetProgramsAsync(info, "3297", startDate, startDate.AddDays(1), CancellationToken.None); var programsList = programs.ToList(); Assert.Single(programsList); diff --git a/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/emptycategory.xml b/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/emptycategory.xml index dd4aa8977..487b02893 100644 --- a/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/emptycategory.xml +++ b/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/emptycategory.xml @@ -1,5 +1,5 @@ <tv date="20221104"> - <programme channel="3297" start="20221104130000 -0400" stop="20221105235959 -0400"> + <programme channel="3297" start="20221104130000 +0000" stop="20221105235959 +0000"> <category lang="en" /> <category lang="en">sports</category> </programme> diff --git a/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/notitle.xml b/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/notitle.xml index 5a5be7997..b3cef41f3 100644 --- a/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/notitle.xml +++ b/tests/Jellyfin.LiveTv.Tests/Test Data/LiveTv/Listings/XmlTv/notitle.xml @@ -1,5 +1,5 @@ <tv date="20221104"> - <programme channel="3297" start="20221104130000 -0400" stop="20221105235959 -0400"> + <programme channel="3297" start="20221104130000 +0000" stop="20221105235959 +0000"> <category lang="en">sports</category> <episode-num system="original-air-date">2022-11-04 13:00:00</episode-num> <icon height="" src="https://domain.tld/image.png" width=""/> diff --git a/tests/Jellyfin.Networking.Tests/NetworkParseTests.cs b/tests/Jellyfin.Networking.Tests/NetworkParseTests.cs index ef87e46a7..38208476f 100644 --- a/tests/Jellyfin.Networking.Tests/NetworkParseTests.cs +++ b/tests/Jellyfin.Networking.Tests/NetworkParseTests.cs @@ -281,11 +281,11 @@ namespace Jellyfin.Networking.Tests } [Theory] - [InlineData("185.10.10.10,200.200.200.200", "79.2.3.4", true)] - [InlineData("185.10.10.10", "185.10.10.10", false)] - [InlineData("", "100.100.100.100", false)] + [InlineData("185.10.10.10,200.200.200.200", "79.2.3.4", RemoteAccessPolicyResult.RejectDueToNotAllowlistedRemoteIP)] + [InlineData("185.10.10.10", "185.10.10.10", RemoteAccessPolicyResult.Allow)] + [InlineData("", "100.100.100.100", RemoteAccessPolicyResult.Allow)] - public void HasRemoteAccess_GivenWhitelist_AllowsOnlyIPsInWhitelist(string addresses, string remoteIP, bool denied) + public void HasRemoteAccess_GivenWhitelist_AllowsOnlyIPsInWhitelist(string addresses, string remoteIP, RemoteAccessPolicyResult expectedResult) { // Comma separated list of IP addresses or IP/netmask entries for networks that will be allowed to connect remotely. // If left blank, all remote addresses will be allowed. @@ -299,15 +299,38 @@ namespace Jellyfin.Networking.Tests var startupConf = new Mock<IConfiguration>(); using var nm = new NetworkManager(NetworkParseTests.GetMockConfig(conf), startupConf.Object, new NullLogger<NetworkManager>()); - Assert.NotEqual(nm.HasRemoteAccess(IPAddress.Parse(remoteIP)), denied); + Assert.Equal(expectedResult, nm.ShouldAllowServerAccess(IPAddress.Parse(remoteIP))); } [Theory] - [InlineData("185.10.10.10", "79.2.3.4", false)] - [InlineData("185.10.10.10", "185.10.10.10", true)] - [InlineData("", "100.100.100.100", false)] + [InlineData("185.10.10.10,200.200.200.200", "79.2.3.4", RemoteAccessPolicyResult.RejectDueToRemoteAccessDisabled)] + [InlineData("185.10.10.10", "127.0.0.1", RemoteAccessPolicyResult.Allow)] + [InlineData("", "100.100.100.100", RemoteAccessPolicyResult.RejectDueToRemoteAccessDisabled)] - public void HasRemoteAccess_GivenBlacklist_BlacklistTheIPs(string addresses, string remoteIP, bool denied) + public void HasRemoteAccess_GivenRemoteAccessDisabled_IgnoresAllowlist(string addresses, string remoteIP, RemoteAccessPolicyResult expectedResult) + { + // Comma separated list of IP addresses or IP/netmask entries for networks that will be allowed to connect remotely. + // If left blank, all remote addresses will be allowed. + var conf = new NetworkConfiguration() + { + EnableIPv4 = true, + EnableRemoteAccess = false, + RemoteIPFilter = addresses.Split(','), + IsRemoteIPFilterBlacklist = false + }; + + var startupConf = new Mock<IConfiguration>(); + using var nm = new NetworkManager(NetworkParseTests.GetMockConfig(conf), startupConf.Object, new NullLogger<NetworkManager>()); + + Assert.Equal(expectedResult, nm.ShouldAllowServerAccess(IPAddress.Parse(remoteIP))); + } + + [Theory] + [InlineData("185.10.10.10", "79.2.3.4", RemoteAccessPolicyResult.Allow)] + [InlineData("185.10.10.10", "185.10.10.10", RemoteAccessPolicyResult.RejectDueToIPBlocklist)] + [InlineData("", "100.100.100.100", RemoteAccessPolicyResult.Allow)] + + public void HasRemoteAccess_GivenBlacklist_BlacklistTheIPs(string addresses, string remoteIP, RemoteAccessPolicyResult expectedResult) { // Comma separated list of IP addresses or IP/netmask entries for networks that will be allowed to connect remotely. // If left blank, all remote addresses will be allowed. @@ -321,7 +344,7 @@ namespace Jellyfin.Networking.Tests var startupConf = new Mock<IConfiguration>(); using var nm = new NetworkManager(NetworkParseTests.GetMockConfig(conf), startupConf.Object, new NullLogger<NetworkManager>()); - Assert.NotEqual(nm.HasRemoteAccess(IPAddress.Parse(remoteIP)), denied); + Assert.Equal(expectedResult, nm.ShouldAllowServerAccess(IPAddress.Parse(remoteIP))); } [Theory] diff --git a/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs b/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs index c09bce52d..725e359d7 100644 --- a/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs +++ b/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs @@ -5,6 +5,7 @@ using System.IO; using Emby.Server.Implementations; using Jellyfin.Server.Extensions; using Jellyfin.Server.Helpers; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common; using MediaBrowser.Common.Configuration; using Microsoft.AspNetCore.Hosting; @@ -16,6 +17,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Moq; using Serilog; +using Serilog.Core; using Serilog.Extensions.Logging; namespace Jellyfin.Server.Integration.Tests @@ -95,7 +97,8 @@ namespace Jellyfin.Server.Integration.Tests .AddInMemoryCollection(ConfigurationOptions.DefaultConfiguration) .AddEnvironmentVariables("JELLYFIN_") .AddInMemoryCollection(commandLineOpts.ConvertToConfig()); - }); + }) + .ConfigureServices(e => e.AddSingleton<IStartupLogger, NullStartupLogger>().AddSingleton(e)); } /// <inheritdoc/> @@ -106,7 +109,7 @@ namespace Jellyfin.Server.Integration.Tests appHost.ServiceProvider = host.Services; var applicationPaths = appHost.ServiceProvider.GetRequiredService<IApplicationPaths>(); Program.ApplyStartupMigrationAsync((ServerApplicationPaths)applicationPaths, appHost.ServiceProvider.GetRequiredService<IConfiguration>()).GetAwaiter().GetResult(); - Program.ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.CoreInitialisaition).GetAwaiter().GetResult(); + Program.ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult(); appHost.InitializeServices(Mock.Of<IConfiguration>()).GetAwaiter().GetResult(); Program.ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult(); host.Start(); @@ -128,5 +131,39 @@ namespace Jellyfin.Server.Integration.Tests base.Dispose(disposing); } + + private sealed class NullStartupLogger : IStartupLogger + { + public IStartupLogger BeginGroup(FormattableString logEntry) + { + return this; + } + + public IDisposable? BeginScope<TState>(TState state) + where TState : notnull + { + return NullLogger.Instance.BeginScope(state); + } + + public bool IsEnabled(LogLevel logLevel) + { + return NullLogger.Instance.IsEnabled(logLevel); + } + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) + { + NullLogger.Instance.Log(logLevel, eventId, state, exception, formatter); + } + + public Microsoft.Extensions.Logging.ILogger With(Microsoft.Extensions.Logging.ILogger logger) + { + return this; + } + + IStartupLogger IStartupLogger.With(Microsoft.Extensions.Logging.ILogger logger) + { + return this; + } + } } } |